From 6447d3ac2d48e7e2b396b62a30bdaaf918a06011 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 11:55:48 -0400 Subject: [PATCH 01/91] Add feature flag for Nested Domains work --- .../linkedin/datahub/graphql/featureflags/FeatureFlags.java | 1 + .../datahub/graphql/resolvers/config/AppConfigResolver.java | 1 + datahub-graphql-core/src/main/resources/app.graphql | 6 ++++++ datahub-web-react/src/app/useAppConfig.ts | 5 +++++ datahub-web-react/src/appConfigContext.tsx | 1 + datahub-web-react/src/graphql/app.graphql | 1 + .../configuration/src/main/resources/application.yml | 1 + 7 files changed, 16 insertions(+) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index f813562945378..2b158f3b6c08a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -15,4 +15,5 @@ public class FeatureFlags { private boolean showBrowseV2 = false; private PreProcessHooks preProcessHooks; private boolean showAcrylInfo = false; + private boolean nestedDomainsEnabled = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 2c55bc79fe501..21adcbdede604 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -163,6 +163,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled()) .setShowBrowseV2(_featureFlags.isShowBrowseV2()) .setShowAcrylInfo(_featureFlags.isShowAcrylInfo()) + .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 761242a6711c1..5aa2e3f8e330a 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -426,6 +426,12 @@ type FeatureFlagsConfig { Whether we should show CTAs in the UI related to moving to Managed DataHub by Acryl. """ showAcrylInfo: Boolean! + + """ + Enables the nested Domains feature that allows users to have sub-Domains. + If this is off, Domains appear "flat" again. + """ + nestedDomainsEnabled: Boolean! } """ diff --git a/datahub-web-react/src/app/useAppConfig.ts b/datahub-web-react/src/app/useAppConfig.ts index cdc8f92210a0d..821d00b9017c3 100644 --- a/datahub-web-react/src/app/useAppConfig.ts +++ b/datahub-web-react/src/app/useAppConfig.ts @@ -12,3 +12,8 @@ export function useIsShowAcrylInfoEnabled() { const appConfig = useAppConfig(); return appConfig.config.featureFlags.showAcrylInfo; } + +export function useIsNestedDomainsEnabled() { + const appConfig = useAppConfig(); + return appConfig.config.featureFlags.nestedDomainsEnabled; +} diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 3b34b108ecc93..f18e91304d388 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -45,6 +45,7 @@ export const DEFAULT_APP_CONFIG = { showSearchFiltersV2: true, showBrowseV2: true, showAcrylInfo: false, + nestedDomainsEnabled: false, }, }; diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 4b1295f1024a2..516acc4cfbdf0 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -60,6 +60,7 @@ query appConfig { showSearchFiltersV2 showBrowseV2 showAcrylInfo + nestedDomainsEnabled } } } diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 82cf9e8fdc8a7..dcc6675fa39e0 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -297,6 +297,7 @@ featureFlags: preProcessHooks: uiEnabled: ${PRE_PROCESS_HOOKS_UI_ENABLED:true} # Circumvents Kafka for processing index updates for UI changes sourced from GraphQL to avoid processing delays showAcrylInfo: ${SHOW_ACRYL_INFO:false} # Show different CTAs within DataHub around moving to Managed DataHub. Set to true for the demo site. + nestedDomainsEnabled: ${NESTED_DOMAINS_ENABLED:true} # Enables the nested Domains feature that allows users to have sub-Domains. If this is off, Domains appear "flat" again entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} From 2be1631f8f9cb4f9abbb1220e77be062d3cad654 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 24 Aug 2023 10:04:08 -0700 Subject: [PATCH 02/91] feat(nested-domains): parentDomain model change --- .../com/linkedin/domain/DomainProperties.pdl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl index 5c8c8a4912e4c..89f44a433b7ba 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl @@ -1,6 +1,7 @@ namespace com.linkedin.domain import com.linkedin.common.AuditStamp +import com.linkedin.common.Urn /** * Information about a Domain @@ -36,4 +37,18 @@ record DomainProperties { } } created: optional AuditStamp + + /** + * Optional: Parent of the domain + */ + @Relationship = { + "name": "IsPartOf", + "entityTypes": [ "domain" ], + } + @Searchable = { + "fieldName": "parentDomain", + "fieldType": "URN", + "hasValuesFieldName": "hasParentDomain" + } + parentDomain: optional Urn } From 9814f021dfa53d93f2ddce3cee628c79166dd0de Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 15:12:25 -0400 Subject: [PATCH 03/91] set up skeleton for new manage domains page --- datahub-web-react/src/app/SearchRoutes.tsx | 9 +++-- .../src/app/domain/DomainRoutes.tsx | 33 +++++++++++++++++++ .../nestedDomains/ManageDomainsPageV2.tsx | 5 +++ .../nestedDomains/ManageDomainsSidebar.tsx | 5 +++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 datahub-web-react/src/app/domain/DomainRoutes.tsx create mode 100644 datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx create mode 100644 datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index 82606befd2663..516a21c04d916 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -8,16 +8,20 @@ import { EntityPage } from './entity/EntityPage'; import { BrowseResultsPage } from './browse/BrowseResultsPage'; import { SearchPage } from './search/SearchPage'; import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage'; -import { ManageDomainsPage } from './domain/ManageDomainsPage'; import { ManageIngestionPage } from './ingest/ManageIngestionPage'; import GlossaryRoutes from './glossary/GlossaryRoutes'; import { SettingsPage } from './settings/SettingsPage'; +import DomainRoutes from './domain/DomainRoutes'; +import { useIsNestedDomainsEnabled } from './useAppConfig'; +import { ManageDomainsPage } from './domain/ManageDomainsPage'; /** * Container for all searchable page routes */ export const SearchRoutes = (): JSX.Element => { const entityRegistry = useEntityRegistry(); + const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); + return ( @@ -38,7 +42,8 @@ export const SearchRoutes = (): JSX.Element => { /> } /> } /> - } /> + {isNestedDomainsEnabled && } />} + {!isNestedDomainsEnabled && } />} } /> } /> } /> diff --git a/datahub-web-react/src/app/domain/DomainRoutes.tsx b/datahub-web-react/src/app/domain/DomainRoutes.tsx new file mode 100644 index 0000000000000..236d6576b8845 --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainRoutes.tsx @@ -0,0 +1,33 @@ +import React from 'react'; +import styled from 'styled-components/macro'; +import { Switch, Route } from 'react-router-dom'; +import { PageRoutes } from '../../conf/Global'; +import { EntityPage } from '../entity/EntityPage'; +import { useEntityRegistry } from '../useEntityRegistry'; +import ManageDomainsPageV2 from './nestedDomains/ManageDomainsPageV2'; +import { EntityType } from '../../types.generated'; +import ManageDomainsSidebar from './nestedDomains/ManageDomainsSidebar'; + +const ContentWrapper = styled.div` + display: flex; + flex: 1; + overflow: hidden; +`; + +export default function DomainRoutes() { + const entityRegistry = useEntityRegistry(); + + return ( + + + + } + /> + } /> + + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx new file mode 100644 index 0000000000000..5468e67625b64 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -0,0 +1,5 @@ +import React from 'react'; + +export default function ManageDomainsPageV2() { + return
; +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx new file mode 100644 index 0000000000000..d9b53055bfece --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx @@ -0,0 +1,5 @@ +import React from 'react'; + +export default function ManageDomainsSidebar() { + return
; +} From a179c68ca0fc604d285aa2128296b61b3692baa0 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 15:41:24 -0400 Subject: [PATCH 04/91] get sidebar and header of page together --- .../src/app/domain/DomainIcon.tsx | 7 +++++ .../nestedDomains/DomainsSidebarHeader.tsx | 30 +++++++++++++++++++ .../app/domain/nestedDomains/DomainsTitle.tsx | 18 +++++++++++ .../nestedDomains/ManageDomainsPageV2.tsx | 28 ++++++++++++++++- .../nestedDomains/ManageDomainsSidebar.tsx | 22 ++++++++++++-- .../src/app/glossary/GlossarySidebar.tsx | 2 +- 6 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 datahub-web-react/src/app/domain/DomainIcon.tsx create mode 100644 datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx create mode 100644 datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx diff --git a/datahub-web-react/src/app/domain/DomainIcon.tsx b/datahub-web-react/src/app/domain/DomainIcon.tsx new file mode 100644 index 0000000000000..4d39a6b17d3aa --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainIcon.tsx @@ -0,0 +1,7 @@ +import Icon from '@ant-design/icons/lib/components/Icon'; +import React from 'react'; +import { ReactComponent as DomainsIcon } from '../../images/domain.svg'; + +export default function DomainIcon() { + return ; +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx new file mode 100644 index 0000000000000..ff6d11ae4d7ee --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx @@ -0,0 +1,30 @@ +import { PlusOutlined } from '@ant-design/icons'; +import { Button } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import { ANTD_GRAY, ANTD_GRAY_V2 } from '../../entity/shared/constants'; +import DomainsTitle from './DomainsTitle'; + +const HeaderWrapper = styled.div` + border-bottom: 1px solid ${ANTD_GRAY[4]}; + padding: 16px; + font-size: 20px; + display: flex; + align-items: center; + justify-content: space-between; +`; + +const StyledButton = styled(Button)` + box-shadow: none; + border-color: ${ANTD_GRAY_V2[6]}; +`; + +export default function DomainsSidebarHeader() { + return ( + + + {/* TODO - give functionality to this button */} + } /> + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx b/datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx new file mode 100644 index 0000000000000..3aa7c8330d079 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx @@ -0,0 +1,18 @@ +import React from 'react'; +import styled from 'styled-components'; +import DomainIcon from '../DomainIcon'; + +const IconWrapper = styled.span` + margin-right: 10px; +`; + +export default function DomainsTitle() { + return ( + + + + + Domains + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index 5468e67625b64..541adab1c7987 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -1,5 +1,31 @@ +import { Button } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; import React from 'react'; +import styled from 'styled-components/macro'; +import DomainsTitle from './DomainsTitle'; + +const PageWrapper = styled.div` + background-color: #f8f9fa; + flex: 1; +`; + +const Header = styled.div` + display: flex; + justify-content: space-between; + padding: 32px 24px; + font-size: 30px; + align-items: center; +`; export default function ManageDomainsPageV2() { - return
; + return ( + +
+ + +
+ + ); } diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx index d9b53055bfece..71da9b52678a1 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx @@ -1,5 +1,23 @@ -import React from 'react'; +import React, { useState } from 'react'; +import { BrowserWrapper, MAX_BROWSER_WIDTH, MIN_BROWSWER_WIDTH } from '../../glossary/BusinessGlossaryPage'; +import { ProfileSidebarResizer } from '../../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; +import DomainsSidebarHeader from './DomainsSidebarHeader'; export default function ManageDomainsSidebar() { - return
; + const [browserWidth, setBrowserWith] = useState(window.innerWidth * 0.2); + + return ( + <> + + + + + setBrowserWith(Math.min(Math.max(width, MIN_BROWSWER_WIDTH), MAX_BROWSER_WIDTH)) + } + initialSize={browserWidth} + isSidebarOnLeft + /> + + ); } diff --git a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx index 0bdcbf707ce09..52d9ea859f5e1 100644 --- a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx +++ b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx @@ -4,7 +4,7 @@ import GlossarySearch from './GlossarySearch'; import GlossaryBrowser from './GlossaryBrowser/GlossaryBrowser'; import { ProfileSidebarResizer } from '../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; -const BrowserWrapper = styled.div<{ width: number }>` +export const BrowserWrapper = styled.div<{ width: number }>` max-height: 100%; width: ${(props) => props.width}px; min-width: ${(props) => props.width}px; From 90cdea86c70d186182f6a74430ca309c25fc2dc5 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 16:58:47 -0400 Subject: [PATCH 05/91] show root domains --- .../nestedDomains/ManageDomainsPageV2.tsx | 8 +++- .../app/domain/nestedDomains/RootDomains.tsx | 41 +++++++++++++++++++ .../src/app/search/SearchResultList.tsx | 3 +- datahub-web-react/src/appConfigContext.tsx | 2 +- datahub-web-react/src/graphql/domain.graphql | 1 + 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index 541adab1c7987..ad041c2e114b5 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -3,6 +3,9 @@ import { PlusOutlined } from '@ant-design/icons'; import React from 'react'; import styled from 'styled-components/macro'; import DomainsTitle from './DomainsTitle'; +import RootDomains from './RootDomains'; +import { DOMAINS_CREATE_DOMAIN_ID, DOMAINS_INTRO_ID } from '../../onboarding/config/DomainsOnboardingConfig'; +import { OnboardingTour } from '../../onboarding/OnboardingTour'; const PageWrapper = styled.div` background-color: #f8f9fa; @@ -20,12 +23,15 @@ const Header = styled.div` export default function ManageDomainsPageV2() { return ( +
-
+ ); } diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx new file mode 100644 index 0000000000000..8b8e844f4f7a0 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -0,0 +1,41 @@ +import React from 'react'; +import styled from 'styled-components'; +import { useListDomainsQuery } from '../../../graphql/domain.generated'; +import { Message } from '../../shared/Message'; +import { ResultWrapper } from '../../search/SearchResultList'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { EntityType } from '../../../types.generated'; + +const RootDomainsWrapper = styled.div` + padding: 0 28px; +`; + +const RootDomainsHeader = styled.div` + font-size: 20px; + margin-bottom: 18px; +`; + +export default function RootDomains() { + const entityRegistry = useEntityRegistry(); + const { loading, error, data } = useListDomainsQuery({ + variables: { + input: { + start: 0, + count: 1000, // don't paginate the home page, get all root level domains + }, + }, + }); + + return ( + + Your Domains + {!data && loading && } + {error && } + {data?.listDomains?.domains.map((domain) => ( + + {entityRegistry.renderSearchResult(EntityType.Domain, { entity: domain, matchedFields: [] })} + + ))} + + ); +} diff --git a/datahub-web-react/src/app/search/SearchResultList.tsx b/datahub-web-react/src/app/search/SearchResultList.tsx index 6e2d5c923c6e2..ff439096aa2f0 100644 --- a/datahub-web-react/src/app/search/SearchResultList.tsx +++ b/datahub-web-react/src/app/search/SearchResultList.tsx @@ -40,7 +40,7 @@ const ThinDivider = styled(Divider)` margin-bottom: 16px; `; -const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` +export const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` ${(props) => props.showUpdatedStyles && ` @@ -48,7 +48,6 @@ const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` border-radius: 5px; margin: 0 auto 8px auto; padding: 8px 16px; - max-width: 1200px; border-bottom: 1px solid ${ANTD_GRAY[5]}; `} `; diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index f18e91304d388..f08cdb2bed6e4 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -45,7 +45,7 @@ export const DEFAULT_APP_CONFIG = { showSearchFiltersV2: true, showBrowseV2: true, showAcrylInfo: false, - nestedDomainsEnabled: false, + nestedDomainsEnabled: true, }, }; diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index d72ff336bf9e7..dfaa7dddbdccb 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -33,6 +33,7 @@ query listDomains($input: ListDomainsInput!) { total domains { urn + type properties { name description From 0e1fbdfe6a6ac028aa89dca0accae6ed6f2d8660 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 17:05:31 -0400 Subject: [PATCH 06/91] display domains sidebar on domain profile page --- datahub-web-react/src/app/SearchRoutes.tsx | 4 ++-- datahub-web-react/src/app/entity/EntityRegistry.tsx | 6 ++++-- datahub-web-react/src/conf/Global.ts | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index 516a21c04d916..a435565da4cd6 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -25,7 +25,7 @@ export const SearchRoutes = (): JSX.Element => { return ( - {entityRegistry.getNonGlossaryEntities().map((entity) => ( + {entityRegistry.getEntitiesForSearchRoutes().map((entity) => ( { /> } /> } /> - {isNestedDomainsEnabled && } />} + {isNestedDomainsEnabled && } />} {!isNestedDomainsEnabled && } />} } /> } /> diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index a07fd02841197..93f50dcf98dd5 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -43,8 +43,10 @@ export default class EntityRegistry { return this.entities; } - getNonGlossaryEntities(): Array> { - return this.entities.filter((entity) => !GLOSSARY_ENTITY_TYPES.includes(entity.type)); + getEntitiesForSearchRoutes(): Array> { + return this.entities.filter( + (entity) => !GLOSSARY_ENTITY_TYPES.includes(entity.type) && entity.type !== EntityType.Domain, + ); } getGlossaryEntities(): Array> { diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts index e1220b8c81b53..82378bb621427 100644 --- a/datahub-web-react/src/conf/Global.ts +++ b/datahub-web-react/src/conf/Global.ts @@ -24,6 +24,7 @@ export enum PageRoutes { INGESTION = '/ingestion', SETTINGS = '/settings', DOMAINS = '/domains', + DOMAIN = '/domain', GLOSSARY = '/glossary', SETTINGS_VIEWS = '/settings/views', EMBED = '/embed', From c45803c5b89c1595680e46becc60e56567ac7f7e Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 17:21:40 -0400 Subject: [PATCH 07/91] update styling --- .../nestedDomains/DomainsSidebarHeader.tsx | 14 +++++++++- .../nestedDomains/ManageDomainsPageV2.tsx | 2 ++ .../app/domain/nestedDomains/RootDomains.tsx | 26 +++++++++++-------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx index ff6d11ae4d7ee..f70b7e1d28d33 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx @@ -1,9 +1,11 @@ import { PlusOutlined } from '@ant-design/icons'; import { Button } from 'antd'; import React from 'react'; +import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { ANTD_GRAY, ANTD_GRAY_V2 } from '../../entity/shared/constants'; import DomainsTitle from './DomainsTitle'; +import { PageRoutes } from '../../../conf/Global'; const HeaderWrapper = styled.div` border-bottom: 1px solid ${ANTD_GRAY[4]}; @@ -19,10 +21,20 @@ const StyledButton = styled(Button)` border-color: ${ANTD_GRAY_V2[6]}; `; +const StyledLink = styled(Link)` + color: inherit; + + &:hover { + color: inherit; + } +`; + export default function DomainsSidebarHeader() { return ( - + + + {/* TODO - give functionality to this button */} } /> diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index ad041c2e114b5..d124c44a11bea 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -10,6 +10,8 @@ import { OnboardingTour } from '../../onboarding/OnboardingTour'; const PageWrapper = styled.div` background-color: #f8f9fa; flex: 1; + display: flex; + flex-direction: column; `; const Header = styled.div` diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx index 8b8e844f4f7a0..a40404fd05ad7 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -6,13 +6,15 @@ import { ResultWrapper } from '../../search/SearchResultList'; import { useEntityRegistry } from '../../useEntityRegistry'; import { EntityType } from '../../../types.generated'; -const RootDomainsWrapper = styled.div` - padding: 0 28px; -`; - const RootDomainsHeader = styled.div` font-size: 20px; margin-bottom: 18px; + padding: 0 28px; +`; + +const DomainsWrapper = styled.div` + overflow: auto; + padding: 0 28px 16px 28px; `; export default function RootDomains() { @@ -27,15 +29,17 @@ export default function RootDomains() { }); return ( - + <> Your Domains {!data && loading && } {error && } - {data?.listDomains?.domains.map((domain) => ( - - {entityRegistry.renderSearchResult(EntityType.Domain, { entity: domain, matchedFields: [] })} - - ))} - + + {data?.listDomains?.domains.map((domain) => ( + + {entityRegistry.renderSearchResult(EntityType.Domain, { entity: domain, matchedFields: [] })} + + ))} + + ); } From 339a9476d9f28a91b29e61f6b546dd7b6ca8d1ab Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 24 Aug 2023 17:41:24 -0400 Subject: [PATCH 08/91] fix routing situation when flag is off --- datahub-web-react/src/app/SearchRoutes.tsx | 5 ++++- datahub-web-react/src/app/entity/EntityRegistry.tsx | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index a435565da4cd6..d2ad4ab6f4db1 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -21,11 +21,14 @@ import { ManageDomainsPage } from './domain/ManageDomainsPage'; export const SearchRoutes = (): JSX.Element => { const entityRegistry = useEntityRegistry(); const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); + const entities = isNestedDomainsEnabled + ? entityRegistry.getEntitiesForSearchRoutes() + : entityRegistry.getNonGlossaryEntities(); return ( - {entityRegistry.getEntitiesForSearchRoutes().map((entity) => ( + {entities.map((entity) => ( > { + return this.entities.filter((entity) => !GLOSSARY_ENTITY_TYPES.includes(entity.type)); + } + getGlossaryEntities(): Array> { return this.entities.filter((entity) => GLOSSARY_ENTITY_TYPES.includes(entity.type)); } From 0893ee442185367e20e34231eb19c5ce5a927bdd Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 24 Aug 2023 16:17:03 -0700 Subject: [PATCH 09/91] feat(nested-domains): parentDomain in listDomains resolver --- .../resolvers/domain/ListDomainsResolver.java | 22 +++++++++++++++++-- .../src/main/resources/entity.graphql | 5 +++++ .../domain/ListDomainsResolverTest.java | 7 ++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index 6ed8639592d6e..c36f3287ea493 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; @@ -11,12 +12,17 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.utils.QueryUtils; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; + +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -31,6 +37,8 @@ */ public class ListDomainsResolver implements DataFetcher> { + protected static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; + private static final Integer DEFAULT_START = 0; private static final Integer DEFAULT_COUNT = 20; private static final String DEFAULT_QUERY = ""; @@ -53,13 +61,15 @@ public CompletableFuture get(final DataFetchingEnvironment en final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + final Urn parentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; + final Filter filter = buildParentDomainFilter(parentDomainUrn); try { - // First, get all group Urns. + // First, get all domain Urns. final SearchResult gmsResult = _entityClient.search( Constants.DOMAIN_ENTITY_NAME, query, - null, + filter, new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING), start, count, @@ -94,4 +104,12 @@ private List mapUnresolvedDomains(final List entityUrns) { } return results; } + + private Filter buildParentDomainFilter(@Nullable final Urn parentDomainUrn) { + return parentDomainUrn != null ? ( + QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, parentDomainUrn.toString(), Condition.EQUAL)) + ) : ( + QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL)) + ); + } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index b1f9d57300177..7aaf4267b4019 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -9668,6 +9668,11 @@ input ListDomainsInput { Optional search query """ query: String + + """ + Optional parent domain + """ + parentDomain: String } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java index c143f3480fcff..6e6992e2c04f2 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java @@ -8,11 +8,13 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -20,6 +22,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver.PARENT_DOMAIN_INDEX_FIELD_NAME; import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertThrows; @@ -30,7 +33,7 @@ public class ListDomainsResolverTest { private static final Urn TEST_DOMAIN_URN = Urn.createFromTuple("domain", "test-id"); private static final ListDomainsInput TEST_INPUT = new ListDomainsInput( - 0, 20, null + 0, 20, null, null ); @Test @@ -41,7 +44,7 @@ public void testGetSuccess() throws Exception { Mockito.when(mockClient.search( Mockito.eq(Constants.DOMAIN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(null), + Mockito.eq(QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL))), Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), Mockito.eq(0), Mockito.eq(20), From 3c552e284cef4b4fd993bdc2bf7125933db50286 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 24 Aug 2023 17:12:33 -0700 Subject: [PATCH 10/91] add parentDomain input to create domain resolver --- .../resolvers/domain/CreateDomainResolver.java | 11 +++++++++++ .../src/main/resources/entity.graphql | 5 +++++ datahub-web-react/src/graphql/domain.graphql | 1 + 3 files changed, 17 insertions(+) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 39aa1ea28da20..226f7c56b6188 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; @@ -19,6 +20,8 @@ import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; + +import java.net.URISyntaxException; import java.util.UUID; import java.util.concurrent.CompletableFuture; import lombok.RequiredArgsConstructor; @@ -45,6 +48,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final QueryContext context = environment.getContext(); final CreateDomainInput input = bindArgument(environment.getArgument("input"), CreateDomainInput.class); + final Urn parentDomain = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; return CompletableFuture.supplyAsync(() -> { @@ -89,6 +93,13 @@ private DomainProperties mapDomainProperties(final CreateDomainInput input, fina result.setName(input.getName()); result.setDescription(input.getDescription(), SetMode.IGNORE_NULL); result.setCreated(new AuditStamp().setActor(UrnUtils.getUrn(context.getActorUrn())).setTime(System.currentTimeMillis())); + if (input.getParentDomain() != null) { + try { + result.setParentDomain(Urn.createFromString(input.getParentDomain())); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to create Domain Urn from string: %s", input.getParentDomain()), e); + } + } return result; } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 7aaf4267b4019..d5ff02b39776e 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -9648,6 +9648,11 @@ input CreateDomainInput { Optional description for the Domain """ description: String + + """ + Optional parent domain urn for the domain + """ + parentDomain: String } """ diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index d72ff336bf9e7..b60e0a6331973 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -33,6 +33,7 @@ query listDomains($input: ListDomainsInput!) { total domains { urn + parentDomain properties { name description From 4decf3e8aa3eb1c7f74adf849e85b641b95f460c Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 24 Aug 2023 17:21:40 -0700 Subject: [PATCH 11/91] back out a gql field --- datahub-web-react/src/graphql/domain.graphql | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index b60e0a6331973..d72ff336bf9e7 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -33,7 +33,6 @@ query listDomains($input: ListDomainsInput!) { total domains { urn - parentDomain properties { name description From 56644fcaca10f70dec06ccb272c80bbcdf9cf3f1 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 24 Aug 2023 17:22:46 -0700 Subject: [PATCH 12/91] add todos --- .../datahub/graphql/resolvers/domain/CreateDomainResolver.java | 1 + .../datahub/graphql/resolvers/domain/ListDomainsResolver.java | 1 + 2 files changed, 2 insertions(+) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 226f7c56b6188..470512e36863d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -51,6 +51,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final Urn parentDomain = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; return CompletableFuture.supplyAsync(() -> { + // todo - implement parentDomain permissions if (!AuthorizationUtils.canCreateDomains(context)) { throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index c36f3287ea493..598420f5dc4e4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -55,6 +55,7 @@ public CompletableFuture get(final DataFetchingEnvironment en final QueryContext context = environment.getContext(); return CompletableFuture.supplyAsync(() -> { + // todo - implement parentDomain permissions if (AuthorizationUtils.canCreateDomains(context)) { final ListDomainsInput input = bindArgument(environment.getArgument("input"), ListDomainsInput.class); From 2c84014d02115527fc5e27160449888d2a93a1c5 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 24 Aug 2023 17:26:20 -0700 Subject: [PATCH 13/91] get test to pass --- .../graphql/resolvers/domain/CreateDomainResolverTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 9343e5d772826..e2153e89b7bf2 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -29,7 +29,8 @@ public class CreateDomainResolverTest { private static final CreateDomainInput TEST_INPUT = new CreateDomainInput( "test-id", "test-name", - "test-description" + "test-description", + null ); private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test"); private static final String TEST_ENTITY_URN = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; From cd1a62771e187cb3069f7551d9460fb8c208a0a6 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Fri, 25 Aug 2023 12:21:16 -0400 Subject: [PATCH 14/91] PR edits --- .../app/domain/nestedDomains/ManageDomainsPageV2.tsx | 3 ++- .../domain/nestedDomains/ManageDomainsSidebar.tsx | 7 ++++--- datahub-web-react/src/app/entity/shared/constants.ts | 1 + .../src/app/glossary/BusinessGlossaryPage.tsx | 6 ------ .../src/app/glossary/GlossarySidebar.tsx | 12 +++--------- datahub-web-react/src/app/search/SearchResults.tsx | 3 ++- .../src/app/search/sidebar/BrowseSidebar.tsx | 3 ++- .../src/app/shared/sidebar/components.tsx | 7 +++++++ 8 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 datahub-web-react/src/app/shared/sidebar/components.tsx diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index d124c44a11bea..2994ea37f1082 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -6,9 +6,10 @@ import DomainsTitle from './DomainsTitle'; import RootDomains from './RootDomains'; import { DOMAINS_CREATE_DOMAIN_ID, DOMAINS_INTRO_ID } from '../../onboarding/config/DomainsOnboardingConfig'; import { OnboardingTour } from '../../onboarding/OnboardingTour'; +import { ANTD_GRAY_V2 } from '../../entity/shared/constants'; const PageWrapper = styled.div` - background-color: #f8f9fa; + background-color: ${ANTD_GRAY_V2[1]}; flex: 1; display: flex; flex-direction: column; diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx index 71da9b52678a1..c7ebe5f3a8358 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx @@ -1,16 +1,17 @@ import React, { useState } from 'react'; -import { BrowserWrapper, MAX_BROWSER_WIDTH, MIN_BROWSWER_WIDTH } from '../../glossary/BusinessGlossaryPage'; +import { MAX_BROWSER_WIDTH, MIN_BROWSWER_WIDTH } from '../../glossary/BusinessGlossaryPage'; import { ProfileSidebarResizer } from '../../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; import DomainsSidebarHeader from './DomainsSidebarHeader'; +import { SidebarWrapper } from '../../shared/sidebar/components'; export default function ManageDomainsSidebar() { const [browserWidth, setBrowserWith] = useState(window.innerWidth * 0.2); return ( <> - + - + setBrowserWith(Math.min(Math.max(width, MIN_BROWSWER_WIDTH), MAX_BROWSER_WIDTH)) diff --git a/datahub-web-react/src/app/entity/shared/constants.ts b/datahub-web-react/src/app/entity/shared/constants.ts index 447780fb0d641..9df5923d18542 100644 --- a/datahub-web-react/src/app/entity/shared/constants.ts +++ b/datahub-web-react/src/app/entity/shared/constants.ts @@ -21,6 +21,7 @@ export const ANTD_GRAY = { }; export const ANTD_GRAY_V2 = { + 1: '#F8F9Fa', 2: '#F3F5F6', 5: '#DDE0E4', 6: '#B2B8BD', diff --git a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx index 2adeb6b1684dc..11f54cb5078e6 100644 --- a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx +++ b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx @@ -38,12 +38,6 @@ const MainContentWrapper = styled.div` flex-direction: column; `; -export const BrowserWrapper = styled.div<{ width: number }>` - max-height: 100%; - width: ${(props) => props.width}px; - min-width: ${(props) => props.width}px; -`; - export const MAX_BROWSER_WIDTH = 500; export const MIN_BROWSWER_WIDTH = 200; diff --git a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx index 52d9ea859f5e1..2d620fb06df38 100644 --- a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx +++ b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx @@ -1,14 +1,8 @@ import React, { useState } from 'react'; -import styled from 'styled-components/macro'; import GlossarySearch from './GlossarySearch'; import GlossaryBrowser from './GlossaryBrowser/GlossaryBrowser'; import { ProfileSidebarResizer } from '../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; - -export const BrowserWrapper = styled.div<{ width: number }>` - max-height: 100%; - width: ${(props) => props.width}px; - min-width: ${(props) => props.width}px; -`; +import { SidebarWrapper } from '../shared/sidebar/components'; export const MAX_BROWSER_WIDTH = 500; export const MIN_BROWSWER_WIDTH = 200; @@ -18,10 +12,10 @@ export default function GlossarySidebar() { return ( <> - + - + setBrowserWith(Math.min(Math.max(width, MIN_BROWSWER_WIDTH), MAX_BROWSER_WIDTH)) diff --git a/datahub-web-react/src/app/search/SearchResults.tsx b/datahub-web-react/src/app/search/SearchResults.tsx index 19f762c1c6cf2..d60465eeb73e2 100644 --- a/datahub-web-react/src/app/search/SearchResults.tsx +++ b/datahub-web-react/src/app/search/SearchResults.tsx @@ -26,6 +26,7 @@ import { useIsBrowseV2, useIsSearchV2 } from './useSearchAndBrowseVersion'; import useToggleSidebar from './useToggleSidebar'; import SearchSortSelect from './sorting/SearchSortSelect'; import { combineSiblingsInSearchResults } from './utils/combineSiblingsInSearchResults'; +import { ANTD_GRAY_V2 } from '../entity/shared/constants'; const SearchResultsWrapper = styled.div<{ v2Styles: boolean }>` display: flex; @@ -54,7 +55,7 @@ const ResultContainer = styled.div<{ v2Styles: boolean }>` ? ` display: flex; flex-direction: column; - background-color: #F8F9FA; + background-color: ${ANTD_GRAY_V2[1]}; ` : ` max-width: calc(100% - 260px); diff --git a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx index b5e9272cc5273..0d3d40c4a71af 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx @@ -6,13 +6,14 @@ import { BrowseProvider } from './BrowseContext'; import SidebarLoadingError from './SidebarLoadingError'; import { SEARCH_RESULTS_BROWSE_SIDEBAR_ID } from '../../onboarding/config/SearchOnboardingConfig'; import useSidebarEntities from './useSidebarEntities'; +import { ANTD_GRAY_V2 } from '../../entity/shared/constants'; const Sidebar = styled.div<{ visible: boolean; width: number }>` height: 100%; width: ${(props) => (props.visible ? `${props.width}px` : '0')}; transition: width 250ms ease-in-out; border-right: 1px solid ${(props) => props.theme.styles['border-color-base']}; - background-color: #f8f9fa; + background-color: ${ANTD_GRAY_V2[1]}; background: white; `; diff --git a/datahub-web-react/src/app/shared/sidebar/components.tsx b/datahub-web-react/src/app/shared/sidebar/components.tsx new file mode 100644 index 0000000000000..2a5915050a25c --- /dev/null +++ b/datahub-web-react/src/app/shared/sidebar/components.tsx @@ -0,0 +1,7 @@ +import styled from 'styled-components'; + +export const SidebarWrapper = styled.div<{ width: number }>` + max-height: 100%; + width: ${(props) => props.width}px; + min-width: ${(props) => props.width}px; +`; From 00d9c3eb51d8e21b555b15558a3c141c9282d5f4 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 25 Aug 2023 09:24:01 -0700 Subject: [PATCH 15/91] chore(nested-domains): list and create tests --- .../resolvers/domain/ListDomainsResolver.java | 1 - .../domain/CreateDomainResolverTest.java | 49 +++++++++++++++++-- .../domain/ListDomainsResolverTest.java | 44 ++++++++++++++++- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index 598420f5dc4e4..c36f3287ea493 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -55,7 +55,6 @@ public CompletableFuture get(final DataFetchingEnvironment en final QueryContext context = environment.getContext(); return CompletableFuture.supplyAsync(() -> { - // todo - implement parentDomain permissions if (AuthorizationUtils.canCreateDomains(context)) { final ListDomainsInput input = bindArgument(environment.getArgument("input"), ListDomainsInput.class); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index e2153e89b7bf2..0c9ec81e61465 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -27,15 +27,21 @@ public class CreateDomainResolverTest { private static final CreateDomainInput TEST_INPUT = new CreateDomainInput( + "test-id", + "test-name", + "test-description", + "urn:li:domain:test-id-parent" + ); + + private static final CreateDomainInput TEST_INPUT_NO_PARENT_DOMAIN = new CreateDomainInput( "test-id", "test-name", "test-description", null ); + private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test"); - private static final String TEST_ENTITY_URN = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; - private static final String TEST_TAG_1_URN = "urn:li:tag:test-id-1"; - private static final String TEST_TAG_2_URN = "urn:li:tag:test-id-2"; + @Test public void testGetSuccess() throws Exception { @@ -52,6 +58,43 @@ public void testGetSuccess() throws Exception { resolver.get(mockEnv).get(); + final DomainKey key = new DomainKey(); + key.setId("test-id"); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME); + DomainProperties props = new DomainProperties(); + props.setDescription("test-description"); + props.setName("test-name"); + props.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)); + props.setParentDomain(Urn.createFromString("urn:li:domain:test-id-parent")); + proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(props)); + proposal.setChangeType(ChangeType.UPSERT); + + // Not ideal to match against "any", but we don't know the auto-generated execution request id + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.argThat(new CreateDomainProposalMatcher(proposal)), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + } + + @Test + public void testGetSuccessNoParentNode() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NO_PARENT_DOMAIN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + final DomainKey key = new DomainKey(); key.setId("test-id"); final MetadataChangeProposal proposal = new MetadataChangeProposal(); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java index 6e6992e2c04f2..f673ab4cee2a4 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java @@ -33,6 +33,10 @@ public class ListDomainsResolverTest { private static final Urn TEST_DOMAIN_URN = Urn.createFromTuple("domain", "test-id"); private static final ListDomainsInput TEST_INPUT = new ListDomainsInput( + 0, 20, null, "urn:li:domain:test-id-parent" + ); + + private static final ListDomainsInput TEST_INPUT_NO_PARENT_DOMAIN = new ListDomainsInput( 0, 20, null, null ); @@ -44,7 +48,7 @@ public void testGetSuccess() throws Exception { Mockito.when(mockClient.search( Mockito.eq(Constants.DOMAIN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL))), + Mockito.eq(QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "urn:li:domain:test-id-parent", Condition.EQUAL))), Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), Mockito.eq(0), Mockito.eq(20), @@ -74,6 +78,44 @@ public void testGetSuccess() throws Exception { assertEquals(resolver.get(mockEnv).get().getDomains().get(0).getUrn(), TEST_DOMAIN_URN.toString()); } + @Test + public void testGetSuccessNoParentDomain() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.search( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(""), + Mockito.eq(QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL))), + Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), + Mockito.eq(0), + Mockito.eq(20), + Mockito.any(Authentication.class), + Mockito.eq(new SearchFlags().setFulltext(true)) + )).thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities(new SearchEntityArray(ImmutableSet.of(new SearchEntity().setEntity(TEST_DOMAIN_URN)))) + ); + + ListDomainsResolver resolver = new ListDomainsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NO_PARENT_DOMAIN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Data Assertions + assertEquals((int) resolver.get(mockEnv).get().getStart(), 0); + assertEquals((int) resolver.get(mockEnv).get().getCount(), 1); + assertEquals((int) resolver.get(mockEnv).get().getTotal(), 1); + assertEquals(resolver.get(mockEnv).get().getDomains().size(), 1); + assertEquals(resolver.get(mockEnv).get().getDomains().get(0).getUrn(), TEST_DOMAIN_URN.toString()); + } + @Test public void testGetUnauthorized() throws Exception { // Create resolver From 0fa4c3f5b69ddaf1f5fe3d8e3ab0f3358bb81c2a Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 25 Aug 2023 10:58:40 -0700 Subject: [PATCH 16/91] feat(nested-domains): support children relationship query --- .../resolvers/domain/ListDomainsResolver.java | 15 ++++++++------- datahub-web-react/src/graphql/domain.graphql | 10 ++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index c36f3287ea493..4a39bb4859067 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -22,6 +22,7 @@ import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; @@ -62,7 +63,7 @@ public CompletableFuture get(final DataFetchingEnvironment en final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); final Urn parentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; - final Filter filter = buildParentDomainFilter(parentDomainUrn); + final Filter filter = parentDomainUrn != null ? buildParentDomainFilter(parentDomainUrn) : buildNoParentDomainFilter(); try { // First, get all domain Urns. @@ -105,11 +106,11 @@ private List mapUnresolvedDomains(final List entityUrns) { return results; } - private Filter buildParentDomainFilter(@Nullable final Urn parentDomainUrn) { - return parentDomainUrn != null ? ( - QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, parentDomainUrn.toString(), Condition.EQUAL)) - ) : ( - QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL)) - ); + private Filter buildParentDomainFilter(@Nonnull final Urn parentDomainUrn) { + return QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, parentDomainUrn.toString(), Condition.EQUAL)); + } + + private Filter buildNoParentDomainFilter() { + return QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL)); } } diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index d72ff336bf9e7..4202bcb10ad00 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -43,6 +43,16 @@ query listDomains($input: ListDomainsInput!) { entities(input: { start: 0, count: 1 }) { total } + children: relationships( + input: { + types: ["IsPartOf"] + direction: INCOMING + start: 0 + count: 0 + } + ) { + total + } } } } From 2eb31ab89c50bce77662e378656e2d0590594425 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 25 Aug 2023 15:49:17 -0700 Subject: [PATCH 17/91] feat(nested-domains): validate domain names --- .../domain/CreateDomainResolver.java | 5 +- .../resolvers/domain/ListDomainsResolver.java | 18 +--- .../resolvers/mutate/util/DomainUtils.java | 98 ++++++++++++++++++ .../domain/CreateDomainResolverTest.java | 99 +++++++++++++++++-- .../domain/ListDomainsResolverTest.java | 11 +-- 5 files changed, 202 insertions(+), 29 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 470512e36863d..3df9913ad88aa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -10,6 +10,7 @@ import com.linkedin.datahub.graphql.generated.CreateDomainInput; import com.linkedin.datahub.graphql.generated.OwnerEntityType; import com.linkedin.datahub.graphql.generated.OwnershipType; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; @@ -24,6 +25,7 @@ import java.net.URISyntaxException; import java.util.UUID; import java.util.concurrent.CompletableFuture; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -52,11 +54,12 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws return CompletableFuture.supplyAsync(() -> { // todo - implement parentDomain permissions - if (!AuthorizationUtils.canCreateDomains(context)) { throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } + DomainUtils.validateDomainName(input.getName(), parentDomain, context, _entityClient); + try { // Create the Domain Key final DomainKey key = new DomainKey(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index 4a39bb4859067..2736c2844b892 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -9,21 +9,18 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.ListDomainsInput; import com.linkedin.datahub.graphql.generated.ListDomainsResult; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.SearchFlags; -import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.search.utils.QueryUtils; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -37,9 +34,6 @@ * Resolver used for listing all Domains defined within DataHub. Requires the MANAGE_DOMAINS platform privilege. */ public class ListDomainsResolver implements DataFetcher> { - - protected static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; - private static final Integer DEFAULT_START = 0; private static final Integer DEFAULT_COUNT = 20; private static final String DEFAULT_QUERY = ""; @@ -63,7 +57,7 @@ public CompletableFuture get(final DataFetchingEnvironment en final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); final Urn parentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; - final Filter filter = parentDomainUrn != null ? buildParentDomainFilter(parentDomainUrn) : buildNoParentDomainFilter(); + final Filter filter = DomainUtils.buildParentDomainFilter(parentDomainUrn); try { // First, get all domain Urns. @@ -105,12 +99,4 @@ private List mapUnresolvedDomains(final List entityUrns) { } return results; } - - private Filter buildParentDomainFilter(@Nonnull final Urn parentDomainUrn) { - return QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, parentDomainUrn.toString(), Condition.EQUAL)); - } - - private Filter buildNoParentDomainFilter() { - return QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL)); - } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index b57160be09d32..f1d8dd2a094da 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -5,29 +5,53 @@ import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.domain.DomainProperties; import com.linkedin.domain.Domains; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.mxe.MetadataChangeProposal; + +import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; + +import com.linkedin.r2.RemoteInvocationException; import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.DOMAIN_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DOMAIN_PROPERTIES_ASPECT_NAME; // TODO: Move to consuming from DomainService. @Slf4j public class DomainUtils { + public static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; + public static final String NAME_INDEX_FIELD_NAME = "name.keyword"; + private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() )); @@ -85,4 +109,78 @@ public static void validateDomain(Urn domainUrn, EntityService entityService) { throw new IllegalArgumentException(String.format("Failed to validate Domain with urn %s. Urn does not exist.", domainUrn)); } } + + private static Criterion buildParentDomainCriterion(@Nullable final Urn parentDomainUrn) { + return QueryUtils.newCriterion( + PARENT_DOMAIN_INDEX_FIELD_NAME, + parentDomainUrn == null ? "" : parentDomainUrn.toString(), + parentDomainUrn == null ? Condition.IS_NULL : Condition.EQUAL + ); + } + + private static Criterion buildNameCriterion(@Nonnull final String name) { + return QueryUtils.newCriterion(NAME_INDEX_FIELD_NAME, name, Condition.EQUAL); + } + + public static Filter buildParentDomainFilter(@Nullable final Urn parentDomainUrn) { + return QueryUtils.newFilter(buildParentDomainCriterion(parentDomainUrn)); + } + + public static Filter buildNameAndParentDomainFilter(@Nonnull final String name, @Nullable final Urn parentDomainUrn) { + final Criterion parentDomainCriterion = buildParentDomainCriterion(parentDomainUrn); + final Criterion nameCriterion = buildNameCriterion(name); + return QueryUtils.getFilterFromCriteria(List.of(parentDomainCriterion, nameCriterion)); + } + + private static Map getDomainsByNameAndParent( + @Nonnull final String name, + @Nullable final Urn parentDomainUrn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) { + try { + final Filter filter = buildNameAndParentDomainFilter(name, parentDomainUrn); + + final SearchResult searchResult = entityClient.filter( + DOMAIN_ENTITY_NAME, + filter, + null, + 0, + 1000, + context.getAuthentication()); + + final Set domainUrns = searchResult.getEntities() + .stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toSet()); + + return entityClient.batchGetV2( + DOMAIN_ENTITY_NAME, + domainUrns, + Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), + context.getAuthentication()); + } catch (Exception e) { + throw new RuntimeException("Failed fetching Domains by name and parent", e); + } + } + + public static void validateDomainName( + @Nonnull final String name, + @Nullable final Urn parentDomainUrn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) { + final Map entities = getDomainsByNameAndParent(name, parentDomainUrn, context, entityClient); + + // Even though we searched by name, do one more pass to check the name is unique + entities.forEach((urn, entityResponse) -> { + if (entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { + DataMap dataMap = entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data(); + DomainProperties domainProperties = new DomainProperties(dataMap); + if (domainProperties.hasName() && domainProperties.getName().equals(name)) { + throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain"); + } + } + }); + } } \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 0c9ec81e61465..8906e41a6dc7f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -1,36 +1,55 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateDomainInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DomainKey; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; + +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CompletionException; import org.mockito.Mockito; import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.DOMAIN_PROPERTIES_ASPECT_NAME; import static org.testng.Assert.*; public class CreateDomainResolverTest { + private static final Urn TEST_PARENT_DOMAIN_URN = Urn.createFromTuple("domain", "test-parent-id"); + private static final CreateDomainInput TEST_INPUT = new CreateDomainInput( "test-id", "test-name", "test-description", - "urn:li:domain:test-id-parent" + TEST_PARENT_DOMAIN_URN.toString() ); private static final CreateDomainInput TEST_INPUT_NO_PARENT_DOMAIN = new CreateDomainInput( @@ -56,6 +75,22 @@ public void testGetSuccess() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), TEST_PARENT_DOMAIN_URN)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + + Mockito.when(mockClient.batchGetV2( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.any(), + Mockito.any(), + Mockito.any(Authentication.class) + )).thenReturn(new HashMap<>()); + resolver.get(mockEnv).get(); final DomainKey key = new DomainKey(); @@ -67,7 +102,7 @@ public void testGetSuccess() throws Exception { props.setDescription("test-description"); props.setName("test-name"); props.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)); - props.setParentDomain(Urn.createFromString("urn:li:domain:test-id-parent")); + props.setParentDomain(TEST_PARENT_DOMAIN_URN); proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); proposal.setAspect(GenericRecordUtils.serializeAspect(props)); proposal.setChangeType(ChangeType.UPSERT); @@ -81,18 +116,25 @@ public void testGetSuccess() throws Exception { } @Test - public void testGetSuccessNoParentNode() throws Exception { - // Create resolver + public void testGetSuccessNoParentDomain() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); - // Execute resolver QueryContext mockContext = getMockAllowContext(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NO_PARENT_DOMAIN); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), null)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + resolver.get(mockEnv).get(); final DomainKey key = new DomainKey(); @@ -108,7 +150,6 @@ public void testGetSuccessNoParentNode() throws Exception { proposal.setAspect(GenericRecordUtils.serializeAspect(props)); proposal.setChangeType(ChangeType.UPSERT); - // Not ideal to match against "any", but we don't know the auto-generated execution request id Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.argThat(new CreateDomainProposalMatcher(proposal)), Mockito.any(Authentication.class), @@ -116,6 +157,52 @@ public void testGetSuccessNoParentNode() throws Exception { ); } + @Test + public void testGetNameConflict() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), TEST_PARENT_DOMAIN_URN)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities( + new SearchEntityArray(new SearchEntity().setEntity(UrnUtils.getUrn("urn:li:domain:test-id"))) + )); + + DomainProperties domainProperties = new DomainProperties(); + domainProperties.setDescription(TEST_INPUT.getDescription()); + domainProperties.setName(TEST_INPUT.getName()); + domainProperties.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)); + domainProperties.setParentDomain(TEST_PARENT_DOMAIN_URN); + + EntityResponse entityResponse = new EntityResponse(); + EnvelopedAspectMap envelopedAspectMap = new EnvelopedAspectMap(); + envelopedAspectMap.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(domainProperties.data()))); + entityResponse.setAspects(envelopedAspectMap); + + Map entityResponseMap = new HashMap<>(); + entityResponseMap.put(UrnUtils.getUrn("urn:li:domain:test-id"), entityResponse); + + Mockito.when(mockClient.batchGetV2( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.any(), + Mockito.any(), + Mockito.any(Authentication.class) + )).thenReturn(entityResponseMap); + + Mockito.when(resolver.get(mockEnv)).thenThrow(IllegalArgumentException.class); + } + @Test public void testGetUnauthorized() throws Exception { // Create resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java index f673ab4cee2a4..bd8a8f98de497 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java @@ -5,16 +5,15 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.ListDomainsInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.SearchFlags; -import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -22,7 +21,6 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; -import static com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver.PARENT_DOMAIN_INDEX_FIELD_NAME; import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertThrows; @@ -31,9 +29,10 @@ public class ListDomainsResolverTest { private static final Urn TEST_DOMAIN_URN = Urn.createFromTuple("domain", "test-id"); + private static final Urn TEST_PARENT_DOMAIN_URN = Urn.createFromTuple("domain", "test-parent-id"); private static final ListDomainsInput TEST_INPUT = new ListDomainsInput( - 0, 20, null, "urn:li:domain:test-id-parent" + 0, 20, null, TEST_PARENT_DOMAIN_URN.toString() ); private static final ListDomainsInput TEST_INPUT_NO_PARENT_DOMAIN = new ListDomainsInput( @@ -48,7 +47,7 @@ public void testGetSuccess() throws Exception { Mockito.when(mockClient.search( Mockito.eq(Constants.DOMAIN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "urn:li:domain:test-id-parent", Condition.EQUAL))), + Mockito.eq(DomainUtils.buildParentDomainFilter(TEST_PARENT_DOMAIN_URN)), Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), Mockito.eq(0), Mockito.eq(20), @@ -86,7 +85,7 @@ public void testGetSuccessNoParentDomain() throws Exception { Mockito.when(mockClient.search( Mockito.eq(Constants.DOMAIN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(QueryUtils.newFilter(QueryUtils.newCriterion(PARENT_DOMAIN_INDEX_FIELD_NAME, "", Condition.IS_NULL))), + Mockito.eq(DomainUtils.buildParentDomainFilter(null)), Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), Mockito.eq(0), Mockito.eq(20), From 8da12db186becbd5cba4033edcacca4d933dd9f5 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 25 Aug 2023 16:30:44 -0700 Subject: [PATCH 18/91] improve validation --- .../domain/CreateDomainResolver.java | 11 ++- .../resolvers/mutate/util/DomainUtils.java | 14 ++-- .../domain/CreateDomainResolverTest.java | 70 +++++++++++++++++-- 3 files changed, 78 insertions(+), 17 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 3df9913ad88aa..0a99cbe8619c7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -58,8 +58,6 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } - DomainUtils.validateDomainName(input.getName(), parentDomain, context, _entityClient); - try { // Create the Domain Key final DomainKey key = new DomainKey(); @@ -72,6 +70,15 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws throw new IllegalArgumentException("This Domain already exists!"); } + if (parentDomain != null && !_entityClient.exists(parentDomain, context.getAuthentication())) { + throw new IllegalArgumentException("Parent Domain does not exist!"); + } + + // todo - I'm unsure if we actually need this if we have the above check for globally unique domain urn... + if (DomainUtils.hasNameConflict(input.getName(), parentDomain, context, _entityClient)) { + throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain!"); + } + // Create the MCP final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, DOMAIN_ENTITY_NAME, DOMAIN_PROPERTIES_ASPECT_NAME, mapDomainProperties(input, context)); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index f1d8dd2a094da..d18bbc9e27c6d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -27,10 +27,8 @@ import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.mxe.MetadataChangeProposal; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,7 +36,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import com.linkedin.r2.RemoteInvocationException; import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; @@ -50,7 +47,7 @@ @Slf4j public class DomainUtils { public static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; - public static final String NAME_INDEX_FIELD_NAME = "name.keyword"; + public static final String NAME_INDEX_FIELD_NAME = "name"; private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() @@ -164,7 +161,7 @@ private static Map getDomainsByNameAndParent( } } - public static void validateDomainName( + public static boolean hasNameConflict( @Nonnull final String name, @Nullable final Urn parentDomainUrn, @Nonnull final QueryContext context, @@ -173,14 +170,13 @@ public static void validateDomainName( final Map entities = getDomainsByNameAndParent(name, parentDomainUrn, context, entityClient); // Even though we searched by name, do one more pass to check the name is unique - entities.forEach((urn, entityResponse) -> { + return entities.values().stream().anyMatch(entityResponse -> { if (entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { DataMap dataMap = entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data(); DomainProperties domainProperties = new DomainProperties(dataMap); - if (domainProperties.hasName() && domainProperties.getName().equals(name)) { - throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain"); - } + return (domainProperties.hasName() && domainProperties.getName().equals(name)); } + return false; }); } } \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 8906e41a6dc7f..da380d1e0d01e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -1,11 +1,9 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.datahub.authentication.Authentication; -import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; -import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateDomainInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; @@ -18,9 +16,6 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DomainKey; -import com.linkedin.metadata.query.SearchFlags; -import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; @@ -43,6 +38,7 @@ public class CreateDomainResolverTest { + private static final Urn TEST_DOMAIN_URN = Urn.createFromTuple("domain", "test-id"); private static final Urn TEST_PARENT_DOMAIN_URN = Urn.createFromTuple("domain", "test-parent-id"); private static final CreateDomainInput TEST_INPUT = new CreateDomainInput( @@ -69,6 +65,16 @@ public void testGetSuccess() throws Exception { EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_PARENT_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(true); + // Execute resolver QueryContext mockContext = getMockAllowContext(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); @@ -121,6 +127,11 @@ public void testGetSuccessNoParentDomain() throws Exception { EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + QueryContext mockContext = getMockAllowContext(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NO_PARENT_DOMAIN); @@ -157,12 +168,57 @@ public void testGetSuccessNoParentDomain() throws Exception { ); } + @Test + public void testGetInvalidParent() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + + Mockito.when(mockClient.exists( + Mockito.eq(UrnUtils.getUrn("urn:li:domain:test-id")), + Mockito.any(Authentication.class) + )).thenReturn(false); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_PARENT_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), TEST_PARENT_DOMAIN_URN)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities( + new SearchEntityArray(new SearchEntity().setEntity(UrnUtils.getUrn("urn:li:domain:test-id"))) + )); + + Mockito.when(resolver.get(mockEnv)).thenThrow(new IllegalArgumentException("Parent Domain does not exist!")); + } + @Test public void testGetNameConflict() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_PARENT_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(true); + QueryContext mockContext = getMockAllowContext(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); @@ -200,7 +256,9 @@ public void testGetNameConflict() throws Exception { Mockito.any(Authentication.class) )).thenReturn(entityResponseMap); - Mockito.when(resolver.get(mockEnv)).thenThrow(IllegalArgumentException.class); + Mockito.when(resolver.get(mockEnv)).thenThrow( + new IllegalArgumentException("Domain with this name already exists at this level of the Domain!") + ); } @Test From d851e870064c007a7bbd0861b72e4073838a351b Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 25 Aug 2023 16:33:06 -0700 Subject: [PATCH 19/91] plug in more usage of static const --- .../resolvers/domain/CreateDomainResolverTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index da380d1e0d01e..a79593ece8ffa 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -175,7 +175,7 @@ public void testGetInvalidParent() throws Exception { CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); Mockito.when(mockClient.exists( - Mockito.eq(UrnUtils.getUrn("urn:li:domain:test-id")), + Mockito.eq(TEST_DOMAIN_URN), Mockito.any(Authentication.class) )).thenReturn(false); @@ -197,7 +197,7 @@ public void testGetInvalidParent() throws Exception { Mockito.any(Integer.class), Mockito.any(Authentication.class) )).thenReturn(new SearchResult().setEntities( - new SearchEntityArray(new SearchEntity().setEntity(UrnUtils.getUrn("urn:li:domain:test-id"))) + new SearchEntityArray(new SearchEntity().setEntity(TEST_DOMAIN_URN)) )); Mockito.when(resolver.get(mockEnv)).thenThrow(new IllegalArgumentException("Parent Domain does not exist!")); @@ -232,7 +232,7 @@ public void testGetNameConflict() throws Exception { Mockito.any(Integer.class), Mockito.any(Authentication.class) )).thenReturn(new SearchResult().setEntities( - new SearchEntityArray(new SearchEntity().setEntity(UrnUtils.getUrn("urn:li:domain:test-id"))) + new SearchEntityArray(new SearchEntity().setEntity(TEST_DOMAIN_URN)) )); DomainProperties domainProperties = new DomainProperties(); @@ -247,7 +247,7 @@ public void testGetNameConflict() throws Exception { entityResponse.setAspects(envelopedAspectMap); Map entityResponseMap = new HashMap<>(); - entityResponseMap.put(UrnUtils.getUrn("urn:li:domain:test-id"), entityResponse); + entityResponseMap.put(TEST_DOMAIN_URN, entityResponse); Mockito.when(mockClient.batchGetV2( Mockito.eq(Constants.DOMAIN_ENTITY_NAME), From d4b4e95c83ade641e427d4a9fcc24156072efa09 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 25 Aug 2023 16:41:58 -0700 Subject: [PATCH 20/91] remove todo --- .../datahub/graphql/resolvers/domain/CreateDomainResolver.java | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 0a99cbe8619c7..fe2906e6d5184 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -74,7 +74,6 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws throw new IllegalArgumentException("Parent Domain does not exist!"); } - // todo - I'm unsure if we actually need this if we have the above check for globally unique domain urn... if (DomainUtils.hasNameConflict(input.getName(), parentDomain, context, _entityClient)) { throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain!"); } From 297f26e8174e75c1f978ef1a84571cdc9a494ef3 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 10:25:14 -0400 Subject: [PATCH 21/91] add DomainNavigator and allow exploration of domains --- .../nestedDomains/ManageDomainsSidebar.tsx | 2 + .../app/domain/nestedDomains/RootDomains.tsx | 11 +-- .../domainNavigator/DomainNavigator.tsx | 26 +++++++ .../domainNavigator/DomainNode.tsx | 68 +++++++++++++++++++ .../src/app/domain/useListDomains.tsx | 21 ++++++ .../src/app/search/sidebar/ExpandableNode.tsx | 30 +------- .../src/app/shared/components.tsx | 29 ++++++++ .../src/app/shared/sidebar/components.tsx | 16 +++++ datahub-web-react/src/graphql/domain.graphql | 1 + 9 files changed, 167 insertions(+), 37 deletions(-) create mode 100644 datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx create mode 100644 datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx create mode 100644 datahub-web-react/src/app/domain/useListDomains.tsx create mode 100644 datahub-web-react/src/app/shared/components.tsx diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx index c7ebe5f3a8358..4db2f516023e5 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx @@ -3,6 +3,7 @@ import { MAX_BROWSER_WIDTH, MIN_BROWSWER_WIDTH } from '../../glossary/BusinessGl import { ProfileSidebarResizer } from '../../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; import DomainsSidebarHeader from './DomainsSidebarHeader'; import { SidebarWrapper } from '../../shared/sidebar/components'; +import DomainNavigator from './domainNavigator/DomainNavigator'; export default function ManageDomainsSidebar() { const [browserWidth, setBrowserWith] = useState(window.innerWidth * 0.2); @@ -11,6 +12,7 @@ export default function ManageDomainsSidebar() { <> + diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx index a40404fd05ad7..b0bca1cea82ee 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -1,10 +1,10 @@ import React from 'react'; import styled from 'styled-components'; -import { useListDomainsQuery } from '../../../graphql/domain.generated'; import { Message } from '../../shared/Message'; import { ResultWrapper } from '../../search/SearchResultList'; import { useEntityRegistry } from '../../useEntityRegistry'; import { EntityType } from '../../../types.generated'; +import useListDomains from '../useListDomains'; const RootDomainsHeader = styled.div` font-size: 20px; @@ -19,14 +19,7 @@ const DomainsWrapper = styled.div` export default function RootDomains() { const entityRegistry = useEntityRegistry(); - const { loading, error, data } = useListDomainsQuery({ - variables: { - input: { - start: 0, - count: 1000, // don't paginate the home page, get all root level domains - }, - }, - }); + const { loading, error, data } = useListDomains({}); return ( <> diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx new file mode 100644 index 0000000000000..6c383cf8709f2 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -0,0 +1,26 @@ +import { Alert } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import useListDomains from '../../useListDomains'; +import DomainNode from './DomainNode'; +import { Domain } from '../../../../types.generated'; + +const NavigatorWrapper = styled.div` + font-size: 14px; + max-height: calc(100% - 47px); + padding: 10px 20px 20px 20px; + overflow: auto; +`; + +export default function DomainNavigator() { + const { data, error } = useListDomains({}); + + return ( + + {error && } + {data?.listDomains?.domains.map((domain) => ( + + ))} + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx new file mode 100644 index 0000000000000..f5417fe858c95 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -0,0 +1,68 @@ +import { Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import { Domain } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { RotatingTriangle } from '../../../shared/sidebar/components'; +import DomainIcon from '../../DomainIcon'; +import useListDomains from '../../useListDomains'; +import useToggle from '../../../shared/useToggle'; +import { BodyContainer, BodyGridExpander } from '../../../shared/components'; + +const RowWrapper = styled.div` + align-items: center; + display: flex; + padding: 3px 2px 4px 0; + overflow: hidden; +`; + +const NameWrapper = styled(Typography.Text)` + flex: 1; + overflow: hidden; + svg { + margin-right: 6px; + } +`; + +const ButtonWrapper = styled.span` + margin-right: 4px; +`; + +const StyledExpander = styled(BodyGridExpander)` + padding-left: 24px; +`; + +interface Props { + domain: Domain; +} + +export default function DomainNode({ domain }: Props) { + const entityRegistry = useEntityRegistry(); + const { isOpen, isClosing, toggle } = useToggle({ + initialValue: false, + closeDelay: 250, + }); + const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); + + return ( + <> + + {/* TODO: only show this triangle if we know there are child domains */} + + + + + + {entityRegistry.getDisplayName(domain.type, domain)} + + + + + {data?.listDomains?.domains.map((childDomain) => ( + + ))} + + + + ); +} diff --git a/datahub-web-react/src/app/domain/useListDomains.tsx b/datahub-web-react/src/app/domain/useListDomains.tsx new file mode 100644 index 0000000000000..2daea2b5bfa93 --- /dev/null +++ b/datahub-web-react/src/app/domain/useListDomains.tsx @@ -0,0 +1,21 @@ +import { useListDomainsQuery } from '../../graphql/domain.generated'; + +interface Props { + parentDomain?: string; + skip?: boolean; +} + +export default function useListDomains({ parentDomain, skip }: Props) { + const { data, error, loading, refetch } = useListDomainsQuery({ + skip, + variables: { + input: { + start: 0, + count: 1000, // don't paginate the home page, get all root level domains + parentDomain, + }, + }, + }); + + return { data, error, loading, refetch }; +} diff --git a/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx b/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx index 32d2c4af948ef..ba93cf94fba2b 100644 --- a/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx @@ -1,9 +1,10 @@ import React, { MouseEventHandler, ReactNode } from 'react'; import styled from 'styled-components'; import { VscTriangleRight } from 'react-icons/vsc'; -import { Button, Typography } from 'antd'; +import { Typography } from 'antd'; import { UpCircleOutlined } from '@ant-design/icons'; import { ANTD_GRAY } from '../../entity/shared/constants'; +import { BaseButton, BodyContainer, BodyGridExpander, RotatingButton } from '../../shared/components'; const Layout = styled.div` margin-left: 8px; @@ -11,17 +12,6 @@ const Layout = styled.div` const HeaderContainer = styled.div``; -const BodyGridExpander = styled.div<{ isOpen: boolean }>` - display: grid; - grid-template-rows: ${(props) => (props.isOpen ? '1fr' : '0fr')}; - transition: grid-template-rows 250ms; - overflow: hidden; -`; - -const BodyContainer = styled.div` - min-height: 0; -`; - type ExpandableNodeProps = { isOpen: boolean; header: ReactNode; @@ -68,22 +58,6 @@ ExpandableNode.HeaderLeft = styled.div` align-items: center; `; -const BaseButton = styled(Button)` - &&& { - display: flex; - align-items: center; - justify-content: center; - border: none; - box-shadow: none; - border-radius: 50%; - } -`; - -const RotatingButton = styled(BaseButton)<{ deg: number }>` - transform: rotate(${(props) => props.deg}deg); - transition: transform 250ms; -`; - ExpandableNode.StaticButton = ({ icon, onClick }: { icon: JSX.Element; onClick?: () => void }) => { const onClickButton: MouseEventHandler = (e) => { e.stopPropagation(); diff --git a/datahub-web-react/src/app/shared/components.tsx b/datahub-web-react/src/app/shared/components.tsx new file mode 100644 index 0000000000000..425fa0577da02 --- /dev/null +++ b/datahub-web-react/src/app/shared/components.tsx @@ -0,0 +1,29 @@ +import { Button } from 'antd'; +import styled from 'styled-components'; + +export const BaseButton = styled(Button)` + &&& { + display: flex; + align-items: center; + justify-content: center; + border: none; + box-shadow: none; + border-radius: 50%; + } +`; + +export const RotatingButton = styled(BaseButton)<{ deg: number }>` + transform: rotate(${(props) => props.deg}deg); + transition: transform 250ms; +`; + +export const BodyGridExpander = styled.div<{ isOpen: boolean }>` + display: grid; + grid-template-rows: ${(props) => (props.isOpen ? '1fr' : '0fr')}; + transition: grid-template-rows 250ms; + overflow: hidden; +`; + +export const BodyContainer = styled.div` + min-height: 0; +`; diff --git a/datahub-web-react/src/app/shared/sidebar/components.tsx b/datahub-web-react/src/app/shared/sidebar/components.tsx index 2a5915050a25c..5d123d6022790 100644 --- a/datahub-web-react/src/app/shared/sidebar/components.tsx +++ b/datahub-web-react/src/app/shared/sidebar/components.tsx @@ -1,7 +1,23 @@ +import React from 'react'; +import { RightOutlined } from '@ant-design/icons'; import styled from 'styled-components'; +import { RotatingButton } from '../components'; export const SidebarWrapper = styled.div<{ width: number }>` max-height: 100%; width: ${(props) => props.width}px; min-width: ${(props) => props.width}px; `; + +export function RotatingTriangle({ isOpen, onClick }: { isOpen: boolean; onClick?: () => void }) { + return ( + } + onClick={onClick} + /> + ); +} diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index de46fb8eee52b..081c2104012cf 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -33,6 +33,7 @@ query listDomains($input: ListDomainsInput!) { total domains { urn + id type properties { name From 4638adae720144d2a20c463ace095ecb8e22d913 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 11:43:00 -0400 Subject: [PATCH 22/91] set entity data and show styled selected domain in sidebar --- .../src/app/domain/DomainRoutes.tsx | 29 ++++++++------- .../src/app/domain/DomainsContext.tsx | 17 +++++++++ .../domainNavigator/DomainNavigator.tsx | 4 +-- .../domainNavigator/DomainNode.tsx | 35 ++++++++++++++++--- datahub-web-react/src/app/domain/utils.ts | 17 +++++++++ .../containers/profile/EntityProfile.tsx | 2 ++ .../src/app/shared/styleUtils.ts | 7 ++++ 7 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 datahub-web-react/src/app/domain/DomainsContext.tsx create mode 100644 datahub-web-react/src/app/shared/styleUtils.ts diff --git a/datahub-web-react/src/app/domain/DomainRoutes.tsx b/datahub-web-react/src/app/domain/DomainRoutes.tsx index 236d6576b8845..d4e048789dbdd 100644 --- a/datahub-web-react/src/app/domain/DomainRoutes.tsx +++ b/datahub-web-react/src/app/domain/DomainRoutes.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useState } from 'react'; import styled from 'styled-components/macro'; import { Switch, Route } from 'react-router-dom'; import { PageRoutes } from '../../conf/Global'; @@ -7,6 +7,8 @@ import { useEntityRegistry } from '../useEntityRegistry'; import ManageDomainsPageV2 from './nestedDomains/ManageDomainsPageV2'; import { EntityType } from '../../types.generated'; import ManageDomainsSidebar from './nestedDomains/ManageDomainsSidebar'; +import { DomainsContext } from './DomainsContext'; +import { GenericEntityProperties } from '../entity/shared/types'; const ContentWrapper = styled.div` display: flex; @@ -16,18 +18,21 @@ const ContentWrapper = styled.div` export default function DomainRoutes() { const entityRegistry = useEntityRegistry(); + const [entityData, setEntityData] = useState(null); return ( - - - - } - /> - } /> - - + + + + + } + /> + } /> + + + ); } diff --git a/datahub-web-react/src/app/domain/DomainsContext.tsx b/datahub-web-react/src/app/domain/DomainsContext.tsx new file mode 100644 index 0000000000000..3a6523e31a001 --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainsContext.tsx @@ -0,0 +1,17 @@ +import React, { useContext } from 'react'; +import { GenericEntityProperties } from '../entity/shared/types'; + +export interface DomainsContextType { + entityData: GenericEntityProperties | null; + setEntityData: (entityData: GenericEntityProperties | null) => void; +} + +export const DomainsContext = React.createContext({ + entityData: null, + setEntityData: () => {}, +}); + +export const useDomainsContext = () => { + const { entityData, setEntityData } = useContext(DomainsContext); + return { entityData, setEntityData }; +}; diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx index 6c383cf8709f2..33345b8d42d2d 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -7,8 +7,8 @@ import { Domain } from '../../../../types.generated'; const NavigatorWrapper = styled.div` font-size: 14px; - max-height: calc(100% - 47px); - padding: 10px 20px 20px 20px; + max-height: calc(100% - 65px); + padding: 10px 10px 20px 20px; overflow: auto; `; diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index f5417fe858c95..e98b5cb3aad47 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -1,5 +1,6 @@ import { Typography } from 'antd'; import React from 'react'; +import { useHistory } from 'react-router'; import styled from 'styled-components'; import { Domain } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; @@ -8,17 +9,29 @@ import DomainIcon from '../../DomainIcon'; import useListDomains from '../../useListDomains'; import useToggle from '../../../shared/useToggle'; import { BodyContainer, BodyGridExpander } from '../../../shared/components'; +import { ANTD_GRAY_V2 } from '../../../entity/shared/constants'; +import { useDomainsContext } from '../../DomainsContext'; +import { applyOpacity } from '../../../shared/styleUtils'; const RowWrapper = styled.div` align-items: center; display: flex; - padding: 3px 2px 4px 0; + padding: 2px 2px 4px 0; overflow: hidden; `; -const NameWrapper = styled(Typography.Text)` +const NameWrapper = styled(Typography.Text)<{ isSelected: boolean }>` flex: 1; overflow: hidden; + padding: 2px; + ${(props) => + props.isSelected && `background-color: ${applyOpacity(props.theme.styles['primary-color'] || '', 10)};`} + + &:hover { + ${(props) => !props.isSelected && `background-color: ${ANTD_GRAY_V2[1]};`} + cursor: pointer; + } + svg { margin-right: 6px; } @@ -37,12 +50,20 @@ interface Props { } export default function DomainNode({ domain }: Props) { + const history = useHistory(); const entityRegistry = useEntityRegistry(); + const { entityData } = useDomainsContext(); const { isOpen, isClosing, toggle } = useToggle({ initialValue: false, closeDelay: 250, }); const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); + const isOnEntityPage = entityData && entityData.urn === domain.urn; + const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); + + function handleSelectDomain() { + history.push(entityRegistry.getEntityUrl(domain.type, domain.urn)); + } return ( <> @@ -51,13 +72,17 @@ export default function DomainNode({ domain }: Props) { - + - {entityRegistry.getDisplayName(domain.type, domain)} + {displayName} - + {data?.listDomains?.domains.map((childDomain) => ( ))} diff --git a/datahub-web-react/src/app/domain/utils.ts b/datahub-web-react/src/app/domain/utils.ts index 3af161bc44565..2bf22db5cca71 100644 --- a/datahub-web-react/src/app/domain/utils.ts +++ b/datahub-web-react/src/app/domain/utils.ts @@ -1,4 +1,10 @@ +import { useEffect } from 'react'; +import { isEqual } from 'lodash'; import { ListDomainsDocument, ListDomainsQuery } from '../../graphql/domain.generated'; +import { EntityType } from '../../types.generated'; +import { GenericEntityProperties } from '../entity/shared/types'; +import usePrevious from '../shared/usePrevious'; +import { useDomainsContext } from './DomainsContext'; /** * Add an entry to the list domains cache. @@ -75,3 +81,14 @@ export const removeFromListDomainsCache = (client, urn, page, pageSize) => { }, }); }; + +export function useUpdateDomainEntityDataOnChange(entityData: GenericEntityProperties | null, entityType: EntityType) { + const { setEntityData } = useDomainsContext(); + const previousEntityData = usePrevious(entityData); + + useEffect(() => { + if (EntityType.Domain === entityType && !isEqual(entityData, previousEntityData)) { + setEntityData(entityData); + } + }); +} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx index 8a559013c892c..5384eb94429ed 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx @@ -45,6 +45,7 @@ import { LINEAGE_GRAPH_TIME_FILTER_ID, } from '../../../../onboarding/config/LineageGraphOnboardingConfig'; import { useAppConfig } from '../../../../useAppConfig'; +import { useUpdateDomainEntityDataOnChange } from '../../../../domain/utils'; type Props = { urn: string; @@ -212,6 +213,7 @@ export const EntityProfile = ({ useGetDataForProfile({ urn, entityType, useEntityQuery, getOverrideProperties }); useUpdateGlossaryEntityDataOnChange(entityData, entityType); + useUpdateDomainEntityDataOnChange(entityData, entityType); const maybeUpdateEntity = useUpdateQuery?.({ onCompleted: () => refetch(), diff --git a/datahub-web-react/src/app/shared/styleUtils.ts b/datahub-web-react/src/app/shared/styleUtils.ts new file mode 100644 index 0000000000000..21bc866218cb8 --- /dev/null +++ b/datahub-web-react/src/app/shared/styleUtils.ts @@ -0,0 +1,7 @@ +export function applyOpacity(hexColor: string, opacity: number) { + if (hexColor.length !== 7) return hexColor; + + const updatedOpacity = Math.round(opacity * 2.55); + + return hexColor + updatedOpacity.toString(16).padStart(2, '0'); +} From 2a94f291613afb815e83062dca8b4d4449e53600 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Mon, 28 Aug 2023 08:58:42 -0700 Subject: [PATCH 23/91] remove a filter mock we don't need --- .../resolvers/domain/CreateDomainResolverTest.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index a79593ece8ffa..7ef0495bd998e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -189,17 +189,6 @@ public void testGetInvalidParent() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); - Mockito.when(mockClient.filter( - Mockito.eq(Constants.DOMAIN_ENTITY_NAME), - Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), TEST_PARENT_DOMAIN_URN)), - Mockito.eq(null), - Mockito.any(Integer.class), - Mockito.any(Integer.class), - Mockito.any(Authentication.class) - )).thenReturn(new SearchResult().setEntities( - new SearchEntityArray(new SearchEntity().setEntity(TEST_DOMAIN_URN)) - )); - Mockito.when(resolver.get(mockEnv)).thenThrow(new IllegalArgumentException("Parent Domain does not exist!")); } From d1113943614d02e154f5b2822a0df331c0a8000f Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Mon, 28 Aug 2023 09:22:45 -0700 Subject: [PATCH 24/91] feat(nested-domains): validate name is unique per level --- .../resolvers/mutate/UpdateNameResolver.java | 4 ++++ .../resolvers/domain/CreateDomainResolverTest.java | 7 ------- .../resolvers/glossary/UpdateNameResolverTest.java | 13 +++++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java index 225bee54142c4..1246821712ca9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.UpdateNameInput; import com.linkedin.datahub.graphql.resolvers.dataproduct.DataProductAuthorizationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils; import com.linkedin.dataproduct.DataProductProperties; import com.linkedin.domain.DomainProperties; @@ -127,6 +128,9 @@ private Boolean updateDomainName( if (domainProperties == null) { throw new IllegalArgumentException("Domain does not exist"); } + if (DomainUtils.hasNameConflict(input.getName(), domainProperties.getParentDomain(), context, _entityClient)) { + throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain!"); + } domainProperties.setName(input.getName()); Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); persistAspect(targetUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, domainProperties, actor, _entityService); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 7ef0495bd998e..01b9a8b6e1878 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -90,13 +90,6 @@ public void testGetSuccess() throws Exception { Mockito.any(Authentication.class) )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); - Mockito.when(mockClient.batchGetV2( - Mockito.eq(Constants.DOMAIN_ENTITY_NAME), - Mockito.any(), - Mockito.any(), - Mockito.any(Authentication.class) - )).thenReturn(new HashMap<>()); - resolver.get(mockEnv).get(); final DomainKey key = new DomainKey(); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java index 36f909bd7ebe5..09da1f09732a3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java @@ -8,17 +8,21 @@ import com.linkedin.datahub.graphql.generated.UpdateNameInput; import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; import org.testng.annotations.Test; +import java.util.HashMap; import java.util.concurrent.CompletionException; import static com.linkedin.datahub.graphql.TestUtils.*; @@ -121,6 +125,15 @@ public void testGetSuccessForDomain() throws Exception { 0)) .thenReturn(new DomainProperties().setName(name)); + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(INPUT_FOR_DOMAIN.getName(), null)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + DomainProperties properties = new DomainProperties(); properties.setName(NEW_NAME); final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(DOMAIN_URN), From 9c8b342b76df551a0dd148810027755bfa25f6b7 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 15:20:31 -0400 Subject: [PATCH 25/91] hide toggle if no domain children --- .../domainNavigator/DomainNavigator.tsx | 8 +++-- .../domainNavigator/DomainNode.tsx | 35 +++++++++++++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx index 33345b8d42d2d..03c91f166a731 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -8,7 +8,7 @@ import { Domain } from '../../../../types.generated'; const NavigatorWrapper = styled.div` font-size: 14px; max-height: calc(100% - 65px); - padding: 10px 10px 20px 20px; + padding: 8px 8px 16px 16px; overflow: auto; `; @@ -19,7 +19,11 @@ export default function DomainNavigator() { {error && } {data?.listDomains?.domains.map((domain) => ( - + ))} ); diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index e98b5cb3aad47..5188d871ae521 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -20,12 +20,13 @@ const RowWrapper = styled.div` overflow: hidden; `; -const NameWrapper = styled(Typography.Text)<{ isSelected: boolean }>` +const NameWrapper = styled(Typography.Text)<{ isSelected: boolean; addLeftPadding: boolean }>` flex: 1; overflow: hidden; padding: 2px; ${(props) => props.isSelected && `background-color: ${applyOpacity(props.theme.styles['primary-color'] || '', 10)};`} + ${(props) => props.addLeftPadding && 'padding-left: 22px;'} &:hover { ${(props) => !props.isSelected && `background-color: ${ANTD_GRAY_V2[1]};`} @@ -39,6 +40,19 @@ const NameWrapper = styled(Typography.Text)<{ isSelected: boolean }>` const ButtonWrapper = styled.span` margin-right: 4px; + font-size: 16px; + height: 16px; + width: 16px; + + svg { + height: 10px; + width: 10px; + } + + .ant-btn { + height: 16px; + width: 16px; + } `; const StyledExpander = styled(BodyGridExpander)` @@ -47,9 +61,10 @@ const StyledExpander = styled(BodyGridExpander)` interface Props { domain: Domain; + numDomainChildren: number; } -export default function DomainNode({ domain }: Props) { +export default function DomainNode({ domain, numDomainChildren }: Props) { const history = useHistory(); const entityRegistry = useEntityRegistry(); const { entityData } = useDomainsContext(); @@ -68,14 +83,16 @@ export default function DomainNode({ domain }: Props) { return ( <> - {/* TODO: only show this triangle if we know there are child domains */} - - - + {!!numDomainChildren && ( + + + + )} {displayName} @@ -84,7 +101,11 @@ export default function DomainNode({ domain }: Props) { {data?.listDomains?.domains.map((childDomain) => ( - + ))} From 4dfe08497cf798777f6414f7b5c205575d298fb4 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Mon, 28 Aug 2023 17:11:07 -0300 Subject: [PATCH 26/91] Do not allow deleting a domain with child domains --- .../domain/DeleteDomainResolver.java | 6 ++++ .../resolvers/mutate/util/DomainUtils.java | 27 ++++++++++++++++++ .../domain/DeleteDomainResolverTest.java | 28 +++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java index 60a03fcddcc4d..9ab90e8b4ff72 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.entity.client.EntityClient; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -32,6 +33,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) if (AuthorizationUtils.canManageDomains(context) || AuthorizationUtils.canDeleteEntity(urn, context)) { try { + // Make sure there are no child domains + if (DomainUtils.hasChildDomains(urn, context, _entityClient)) { + throw new RuntimeException(String.format("Cannot delete domain %s which has child domains", domainUrn)); + } + _entityClient.deleteEntity(urn, context.getAuthentication()); log.info(String.format("I've successfully deleted the entity %s with urn", domainUrn)); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index d18bbc9e27c6d..bbe80543ae567 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -27,6 +27,7 @@ import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -36,6 +37,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import javax.management.Query; import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; @@ -129,6 +131,31 @@ public static Filter buildNameAndParentDomainFilter(@Nonnull final String name, return QueryUtils.getFilterFromCriteria(List.of(parentDomainCriterion, nameCriterion)); } + /** + * Check if a domain has any child domains + * @param domainUrn the URN of the domain to check + * @param context query context (includes authorization context to authorize the request) + * @param entityClient client used to perform the check + * @return true if the domain has any child domains, false if it does not + */ + public static boolean hasChildDomains( + @Nonnull final Urn domainUrn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) throws RemoteInvocationException { + Filter parentDomainFilter = buildParentDomainFilter(domainUrn); + // Search for entities matching parent domain + // Limit count to 1 for existence check + final SearchResult searchResult = entityClient.filter( + DOMAIN_ENTITY_NAME, + parentDomainFilter, + null, + 0, + 1, + context.getAuthentication()); + return (searchResult.getNumEntities() > 0); + } + private static Map getDomainsByNameAndParent( @Nonnull final String name, @Nullable final Urn parentDomainUrn, diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java index 1c450b0e85424..f191136e4d378 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java @@ -1,9 +1,11 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.datahub.authentication.Authentication; +import com.datastax.dse.driver.api.core.graph.predicates.Search; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.search.SearchResult; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; import org.mockito.Mockito; @@ -28,6 +30,10 @@ public void testGetSuccess() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + // Domain has 0 child domains + Mockito.when(mockClient.filter(Mockito.eq("domain"), Mockito.any(), Mockito.any(), Mockito.eq(0), Mockito.eq(1), Mockito.any())) + .thenReturn(new SearchResult().setNumEntities(0)); + assertTrue(resolver.get(mockEnv).get()); Mockito.verify(mockClient, Mockito.times(1)).deleteEntity( @@ -36,6 +42,28 @@ public void testGetSuccess() throws Exception { ); } + @Test + public void testDeleteWithChildDomains() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + DeleteDomainResolver resolver = new DeleteDomainResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Domain has child domains + Mockito.when(mockClient.filter(Mockito.eq("domain"), Mockito.any(), Mockito.any(), Mockito.eq(0), Mockito.eq(1), Mockito.any())) + .thenReturn(new SearchResult().setNumEntities(1)); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + Mockito.verify(mockClient, Mockito.times(0)).deleteEntity( + Mockito.any(), + Mockito.any(Authentication.class)); + } + @Test public void testGetUnauthorized() throws Exception { // Create resolver From a8d8c15d488f6afeb44cf8509070b159c7b5acb7 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Mon, 28 Aug 2023 14:22:14 -0700 Subject: [PATCH 27/91] feat(nested-domains): ParentDomainsResolver --- .../datahub/graphql/GmsGraphQLEngine.java | 5 +- .../resolvers/domain/ListDomainsResolver.java | 5 - .../domain/ParentDomainsResolver.java | 57 ++++++++ .../resolvers/mutate/util/DomainUtils.java | 44 ++++++- .../src/main/resources/entity.graphql | 20 ++- .../domain/ParentDomainsResolverTest.java | 124 ++++++++++++++++++ datahub-web-react/src/graphql/domain.graphql | 7 + 7 files changed, 251 insertions(+), 11 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d6dd2de6d31e3..fa1ede2ce40d5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -123,6 +123,7 @@ import com.linkedin.datahub.graphql.resolvers.domain.DeleteDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.DomainEntitiesResolver; import com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver; +import com.linkedin.datahub.graphql.resolvers.domain.ParentDomainsResolver; import com.linkedin.datahub.graphql.resolvers.domain.SetDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.UnsetDomainResolver; import com.linkedin.datahub.graphql.resolvers.embed.UpdateEmbedResolver; @@ -1670,8 +1671,8 @@ private void configureGlossaryRelationshipResolvers(final RuntimeWiring.Builder private void configureDomainResolvers(final RuntimeWiring.Builder builder) { builder.type("Domain", typeWiring -> typeWiring .dataFetcher("entities", new DomainEntitiesResolver(this.entityClient)) - .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient) - ) + .dataFetcher("parentDomains", new ParentDomainsResolver(this.entityClient)) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) ); builder.type("DomainAssociation", typeWiring -> typeWiring .dataFetcher("domain", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index 2736c2844b892..3a751e502eb10 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -3,8 +3,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; -import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.ListDomainsInput; @@ -51,7 +49,6 @@ public CompletableFuture get(final DataFetchingEnvironment en return CompletableFuture.supplyAsync(() -> { - if (AuthorizationUtils.canCreateDomains(context)) { final ListDomainsInput input = bindArgument(environment.getArgument("input"), ListDomainsInput.class); final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); @@ -83,8 +80,6 @@ public CompletableFuture get(final DataFetchingEnvironment en } catch (Exception e) { throw new RuntimeException("Failed to list domains", e); } - } - throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java new file mode 100644 index 0000000000000..dcbb1eae09725 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java @@ -0,0 +1,57 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.ParentDomainsResult; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.metadata.Constants.DOMAIN_ENTITY_NAME; + +public class ParentDomainsResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public ParentDomainsResolver(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + final QueryContext context = environment.getContext(); + final Urn urn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); + final List domains = new ArrayList<>(); + + + if (!DOMAIN_ENTITY_NAME.equals(urn.getEntityType())) { + throw new IllegalArgumentException(String.format("Failed to resolve parents for entity type %s", urn)); + } + + return CompletableFuture.supplyAsync(() -> { + try { + Domain parentDomain = DomainUtils.getParentDomain(urn.toString(), context, _entityClient); + + while (parentDomain != null) { + domains.add(parentDomain); + parentDomain = DomainUtils.getParentDomain(parentDomain.getUrn(), context, _entityClient); + } + + final ParentDomainsResult result = new ParentDomainsResult(); + result.setCount(domains.size()); + result.setDomains(domains); + return result; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to load parent domains for entity %s", urn), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index d18bbc9e27c6d..1f77769191b9e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -10,7 +10,9 @@ import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.types.domain.DomainMapper; import com.linkedin.domain.DomainProperties; import com.linkedin.domain.Domains; import com.linkedin.entity.EntityResponse; @@ -39,8 +41,7 @@ import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; -import static com.linkedin.metadata.Constants.DOMAIN_ENTITY_NAME; -import static com.linkedin.metadata.Constants.DOMAIN_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.*; // TODO: Move to consuming from DomainService. @@ -179,4 +180,43 @@ public static boolean hasNameConflict( return false; }); } + + @Nullable + public static Domain getParentDomain( + @Nonnull final String urnStr, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) { + try { + Urn urn = Urn.createFromString(urnStr); + EntityResponse entityResponse = entityClient.getV2( + urn.getEntityType(), + urn, + Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), + context.getAuthentication() + ); + + if (entityResponse != null && entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { + DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); + if (properties.hasParentDomain()) { + Urn parentDomainUrn = properties.getParentDomain(); + if (parentDomainUrn != null) { + EntityResponse parentResponse = entityClient.getV2( + parentDomainUrn.getEntityType(), + parentDomainUrn, + Collections.singleton(DOMAIN_KEY_ASPECT_NAME), + context.getAuthentication() + ); + if (parentResponse != null) { + return DomainMapper.map(parentResponse); + } + } + } + } + + return null; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to retrieve parent domain for entity %s", urnStr), e); + } + } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index d5ff02b39776e..c1b64c0bc8057 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -9580,15 +9580,31 @@ type Domain implements Entity { """ entities(input: DomainEntitiesInput): SearchResults + """ + Recursively get the lineage of parent domains for this entity + """ + parentDomains: ParentDomainsResult + """ Edges extending from this entity """ relationships(input: RelationshipsInput!): EntityRelationshipsResult } +""" +All of the parent domains starting from a single Domain through all of its ancestors +""" +type ParentDomainsResult { + """ + The number of parent domains bubbling up for this entity + """ + count: Int! - - + """ + A list of parent domains in order from direct parent, to parent's parent etc. If there are no parents, return an empty list + """ + domains: [Domain!]! +} """ Properties about a domain diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java new file mode 100644 index 0000000000000..f8aae698b7010 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java @@ -0,0 +1,124 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.ParentDomainsResult; +import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.key.DomainKey; +import graphql.schema.DataFetchingEnvironment; +import lombok.extern.slf4j.Slf4j; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.assertEquals; + +@Slf4j +public class ParentDomainsResolverTest { + @Test + public void testGetSuccessForDomain() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Urn domainUrn = Urn.createFromString("urn:li:domain:00005397daf94708a8822b8106cfd451"); + Domain domainEntity = new Domain(); + domainEntity.setUrn(domainUrn.toString()); + domainEntity.setType(EntityType.DOMAIN); + Mockito.when(mockEnv.getSource()).thenReturn(domainEntity); + + final DomainProperties parentDomain1 = new DomainProperties().setParentDomain(Urn.createFromString( + "urn:li:domain:11115397daf94708a8822b8106cfd451") + ).setName("test def"); + final DomainProperties parentDomain2 = new DomainProperties().setParentDomain(Urn.createFromString( + "urn:li:domain:22225397daf94708a8822b8106cfd451") + ).setName("test def 2"); + + Map domainAspects = new HashMap<>(); + domainAspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(parentDomain1.data()))); + + Map parentDomain1Aspects = new HashMap<>(); + parentDomain1Aspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( + new DomainProperties().setName("domain parent 1").setParentDomain(parentDomain2.getParentDomain()).data() + ))); + parentDomain1Aspects.put(DOMAIN_KEY_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( + new DomainKey().setId("domain-parent-1-key").data() + ))); + + Map parentDomain2Aspects = new HashMap<>(); + parentDomain2Aspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( + new DomainProperties().setName("domain parent 2").data() + ))); + parentDomain2Aspects.put(DOMAIN_KEY_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( + new DomainKey().setId("domain-parent-2-key").data() + ))); + + Mockito.when(mockClient.getV2( + Mockito.eq(domainUrn.getEntityType()), + Mockito.eq(domainUrn), + Mockito.eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(domainAspects))); + + Mockito.when(mockClient.getV2( + Mockito.eq(parentDomain1.getParentDomain().getEntityType()), + Mockito.eq(parentDomain1.getParentDomain()), + Mockito.eq(Collections.singleton(DOMAIN_KEY_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse() + .setEntityName(DOMAIN_ENTITY_NAME) + .setUrn(parentDomain1.getParentDomain()) + .setAspects(new EnvelopedAspectMap(parentDomain1Aspects))); + + Mockito.when(mockClient.getV2( + Mockito.eq(parentDomain1.getParentDomain().getEntityType()), + Mockito.eq(parentDomain1.getParentDomain()), + Mockito.eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentDomain1Aspects))); + + Mockito.when(mockClient.getV2( + Mockito.eq(parentDomain2.getParentDomain().getEntityType()), + Mockito.eq(parentDomain2.getParentDomain()), + Mockito.eq(Collections.singleton(DOMAIN_KEY_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse() + .setEntityName(DOMAIN_ENTITY_NAME) + .setUrn(parentDomain2.getParentDomain()) + .setAspects(new EnvelopedAspectMap(parentDomain2Aspects))); + + Mockito.when(mockClient.getV2( + Mockito.eq(parentDomain2.getParentDomain().getEntityType()), + Mockito.eq(parentDomain2.getParentDomain()), + Mockito.eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentDomain2Aspects))); + + ParentDomainsResolver resolver = new ParentDomainsResolver(mockClient); + ParentDomainsResult result = resolver.get(mockEnv).get(); + + Mockito.verify(mockClient, Mockito.times(5)).getV2( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any() + ); + assertEquals(result.getCount(), 2); + assertEquals(result.getDomains().get(0).getUrn(), parentDomain1.getParentDomain().toString()); + assertEquals(result.getDomains().get(1).getUrn(), parentDomain2.getParentDomain().toString()); + } +} \ No newline at end of file diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index de46fb8eee52b..d3f74b5e31c5f 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -44,6 +44,13 @@ query listDomains($input: ListDomainsInput!) { entities(input: { start: 0, count: 1 }) { total } + parentDomains { + count + domains { + urn + type + } + } children: relationships( input: { types: ["IsPartOf"] From 814227e1996567fcd393cd957274205541d7f3df Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Mon, 28 Aug 2023 14:24:32 -0700 Subject: [PATCH 28/91] back out parentDomains from list resolver for now --- datahub-web-react/src/graphql/domain.graphql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index d3f74b5e31c5f..de46fb8eee52b 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -44,13 +44,6 @@ query listDomains($input: ListDomainsInput!) { entities(input: { start: 0, count: 1 }) { total } - parentDomains { - count - domains { - urn - type - } - } children: relationships( input: { types: ["IsPartOf"] From eacccb03ef8017bbd5fb30a7bcf974800cbf76b2 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Mon, 28 Aug 2023 14:36:52 -0700 Subject: [PATCH 29/91] s/thenThrow/assertThrow --- .../graphql/resolvers/domain/CreateDomainResolverTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 01b9a8b6e1878..22fbe11d7af81 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -182,7 +182,7 @@ public void testGetInvalidParent() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); - Mockito.when(resolver.get(mockEnv)).thenThrow(new IllegalArgumentException("Parent Domain does not exist!")); + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); } @Test @@ -238,9 +238,7 @@ public void testGetNameConflict() throws Exception { Mockito.any(Authentication.class) )).thenReturn(entityResponseMap); - Mockito.when(resolver.get(mockEnv)).thenThrow( - new IllegalArgumentException("Domain with this name already exists at this level of the Domain!") - ); + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); } @Test From d98cf774c019cac4115a9fd5fd4c57818794091e Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Mon, 28 Aug 2023 16:50:42 -0700 Subject: [PATCH 30/91] more generic graphql resolver --- .../datahub/graphql/GmsGraphQLEngine.java | 8 +++++++ .../domain/ParentDomainsResolver.java | 14 +++++------ .../resolvers/mutate/util/DomainUtils.java | 23 +++++++++---------- .../src/main/resources/entity.graphql | 2 +- .../domain/ParentDomainsResolverTest.java | 11 ++------- .../app/permissions/policy/ManagePolicies.tsx | 3 ++- datahub-web-react/src/graphql/domain.graphql | 13 ++++------- 7 files changed, 35 insertions(+), 39 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index fa1ede2ce40d5..4c4f7b7a76e56 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -80,6 +80,7 @@ import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.Owner; import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; +import com.linkedin.datahub.graphql.generated.ParentDomainsResult; import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue; import com.linkedin.datahub.graphql.generated.QueryEntity; import com.linkedin.datahub.graphql.generated.QuerySubject; @@ -1025,6 +1026,13 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder .dataFetcher("entities", new EntityTypeBatchResolver(entityTypes, (env) -> ((BrowseResults) env.getSource()).getEntities())) ) + .type("ParentDomainsResult", typeWiring -> typeWiring + .dataFetcher("domains", new EntityTypeBatchResolver(entityTypes, + (env) -> { + final ParentDomainsResult result = env.getSource(); + return result != null ? result.getDomains() : null; + })) + ) .type("EntityRelationshipLegacy", typeWiring -> typeWiring .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((EntityRelationshipLegacy) env.getSource()).getEntity())) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java index dcbb1eae09725..1e60674705edf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java @@ -3,7 +3,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.ParentDomainsResult; import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; @@ -29,8 +28,7 @@ public ParentDomainsResolver(final EntityClient entityClient) { public CompletableFuture get(DataFetchingEnvironment environment) { final QueryContext context = environment.getContext(); final Urn urn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); - final List domains = new ArrayList<>(); - + final List parentDomains = new ArrayList<>(); if (!DOMAIN_ENTITY_NAME.equals(urn.getEntityType())) { throw new IllegalArgumentException(String.format("Failed to resolve parents for entity type %s", urn)); @@ -38,16 +36,16 @@ public CompletableFuture get(DataFetchingEnvironment enviro return CompletableFuture.supplyAsync(() -> { try { - Domain parentDomain = DomainUtils.getParentDomain(urn.toString(), context, _entityClient); + Entity parentDomain = DomainUtils.getParentDomain(urn, context, _entityClient); while (parentDomain != null) { - domains.add(parentDomain); - parentDomain = DomainUtils.getParentDomain(parentDomain.getUrn(), context, _entityClient); + parentDomains.add(parentDomain); + parentDomain = DomainUtils.getParentDomain(Urn.createFromString(parentDomain.getUrn()), context, _entityClient); } final ParentDomainsResult result = new ParentDomainsResult(); - result.setCount(domains.size()); - result.setDomains(domains); + result.setCount(parentDomains.size()); + result.setDomains(parentDomains); return result; } catch (Exception e) { throw new RuntimeException(String.format("Failed to load parent domains for entity %s", urn), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index 1f77769191b9e..44d2d0c37d0bc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -10,9 +10,9 @@ import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; -import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.ResourceRefInput; -import com.linkedin.datahub.graphql.types.domain.DomainMapper; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.domain.DomainProperties; import com.linkedin.domain.Domains; import com.linkedin.entity.EntityResponse; @@ -182,14 +182,13 @@ public static boolean hasNameConflict( } @Nullable - public static Domain getParentDomain( - @Nonnull final String urnStr, + public static Entity getParentDomain( + @Nonnull final Urn urn, @Nonnull final QueryContext context, @Nonnull final EntityClient entityClient ) { try { - Urn urn = Urn.createFromString(urnStr); - EntityResponse entityResponse = entityClient.getV2( + final EntityResponse entityResponse = entityClient.getV2( urn.getEntityType(), urn, Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), @@ -197,18 +196,18 @@ public static Domain getParentDomain( ); if (entityResponse != null && entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { - DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); + final DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); if (properties.hasParentDomain()) { - Urn parentDomainUrn = properties.getParentDomain(); + final Urn parentDomainUrn = properties.getParentDomain(); if (parentDomainUrn != null) { - EntityResponse parentResponse = entityClient.getV2( + final EntityResponse parentResponse = entityClient.getV2( parentDomainUrn.getEntityType(), parentDomainUrn, - Collections.singleton(DOMAIN_KEY_ASPECT_NAME), + null, context.getAuthentication() ); if (parentResponse != null) { - return DomainMapper.map(parentResponse); + return UrnToEntityMapper.map(parentResponse.getUrn()); } } } @@ -216,7 +215,7 @@ public static Domain getParentDomain( return null; } catch (Exception e) { - throw new RuntimeException(String.format("Failed to retrieve parent domain for entity %s", urnStr), e); + throw new RuntimeException(String.format("Failed to retrieve parent domain for entity %s", urn), e); } } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index c1b64c0bc8057..b007e2506a09f 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -9603,7 +9603,7 @@ type ParentDomainsResult { """ A list of parent domains in order from direct parent, to parent's parent etc. If there are no parents, return an empty list """ - domains: [Domain!]! + domains: [Entity!]! } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java index f8aae698b7010..544614cf81121 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java @@ -12,7 +12,6 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.key.DomainKey; import graphql.schema.DataFetchingEnvironment; import lombok.extern.slf4j.Slf4j; import org.mockito.Mockito; @@ -55,17 +54,11 @@ public void testGetSuccessForDomain() throws Exception { parentDomain1Aspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( new DomainProperties().setName("domain parent 1").setParentDomain(parentDomain2.getParentDomain()).data() ))); - parentDomain1Aspects.put(DOMAIN_KEY_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( - new DomainKey().setId("domain-parent-1-key").data() - ))); Map parentDomain2Aspects = new HashMap<>(); parentDomain2Aspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( new DomainProperties().setName("domain parent 2").data() ))); - parentDomain2Aspects.put(DOMAIN_KEY_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( - new DomainKey().setId("domain-parent-2-key").data() - ))); Mockito.when(mockClient.getV2( Mockito.eq(domainUrn.getEntityType()), @@ -77,7 +70,7 @@ public void testGetSuccessForDomain() throws Exception { Mockito.when(mockClient.getV2( Mockito.eq(parentDomain1.getParentDomain().getEntityType()), Mockito.eq(parentDomain1.getParentDomain()), - Mockito.eq(Collections.singleton(DOMAIN_KEY_ASPECT_NAME)), + Mockito.eq(null), Mockito.any(Authentication.class) )).thenReturn(new EntityResponse() .setEntityName(DOMAIN_ENTITY_NAME) @@ -94,7 +87,7 @@ public void testGetSuccessForDomain() throws Exception { Mockito.when(mockClient.getV2( Mockito.eq(parentDomain2.getParentDomain().getEntityType()), Mockito.eq(parentDomain2.getParentDomain()), - Mockito.eq(Collections.singleton(DOMAIN_KEY_ASPECT_NAME)), + Mockito.eq(null), Mockito.any(Authentication.class) )).thenReturn(new EntityResponse() .setEntityName(DOMAIN_ENTITY_NAME) diff --git a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx index 08327d40a7165..ecf11eaaa8af5 100644 --- a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx +++ b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx @@ -170,11 +170,12 @@ export const ManagePolicies = () => { variables: { input: { start, - count: pageSize, + count: pageSize + 10, query, }, }, }); + console.log({ policiesData }); // Any time a policy is removed, edited, or created, refetch the list. const [createPolicy, { error: createPolicyError }] = useCreatePolicyMutation(); diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index de46fb8eee52b..2d11726cc04d5 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -34,6 +34,10 @@ query listDomains($input: ListDomainsInput!) { domains { urn type + parentDomains { + urn + type + } properties { name description @@ -44,14 +48,7 @@ query listDomains($input: ListDomainsInput!) { entities(input: { start: 0, count: 1 }) { total } - children: relationships( - input: { - types: ["IsPartOf"] - direction: INCOMING - start: 0 - count: 0 - } - ) { + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { total } } From b54da1d90266d7418f2bd150cf3e3e5de1cfcd0d Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Mon, 28 Aug 2023 17:54:11 -0700 Subject: [PATCH 31/91] first pass --- .../src/app/domain/DomainsList.tsx | 6 +- .../src/app/entity/domain/preview/Preview.tsx | 4 +- .../renderer/component/DomainSearchList.tsx | 4 +- .../src/app/shared/admin/HeaderLinks.tsx | 6 +- .../src/app/shared/tags/DomainLink.tsx | 5 +- snowflake_golden_test.json | 6065 +++++++++++++++++ test_ingest.yaml | 10 + 7 files changed, 6088 insertions(+), 12 deletions(-) create mode 100644 snowflake_golden_test.json create mode 100644 test_ingest.yaml diff --git a/datahub-web-react/src/app/domain/DomainsList.tsx b/datahub-web-react/src/app/domain/DomainsList.tsx index f5fea36e32bda..49ad5299a505d 100644 --- a/datahub-web-react/src/app/domain/DomainsList.tsx +++ b/datahub-web-react/src/app/domain/DomainsList.tsx @@ -18,8 +18,9 @@ import { OnboardingTour } from '../onboarding/OnboardingTour'; import { DOMAINS_INTRO_ID, DOMAINS_CREATE_DOMAIN_ID } from '../onboarding/config/DomainsOnboardingConfig'; import { getElasticCappedTotalValueText } from '../entity/shared/constants'; import { StyledTable } from '../entity/shared/components/styled/StyledTable'; -import { IconStyleType } from '../entity/Entity'; +// import { IconStyleType } from '../entity/Entity'; import { DomainOwnersColumn, DomainListMenuColumn, DomainNameColumn } from './DomainListColumns'; +import DomainIcon from './DomainIcon'; const DomainsContainer = styled.div``; @@ -82,7 +83,6 @@ export const DomainsList = () => { }, 2000); }; - const logoIcon = entityRegistry.getIcon(EntityType.Domain, 12, IconStyleType.ACCENT); const allColumns = [ { title: 'Name', @@ -91,7 +91,7 @@ export const DomainsList = () => { sorter: (sourceA, sourceB) => { return sourceA.name.localeCompare(sourceB.name); }, - render: DomainNameColumn(logoIcon), + render: DomainNameColumn(), }, { title: 'Owners', diff --git a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx index 18cb2bb75df03..2eae7aee67671 100644 --- a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { EntityType, Owner, SearchInsight } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; -import { IconStyleType } from '../../Entity'; +import DomainIcon from '../../../domain/DomainIcon'; export const Preview = ({ urn, @@ -29,7 +29,7 @@ export const Preview = ({ urn={urn} description={description || ''} type="Domain" - typeIcon={entityRegistry.getIcon(EntityType.Domain, 14, IconStyleType.ACCENT)} + typeIcon={} owners={owners} insights={insights} logoComponent={logoComponent} diff --git a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx index d3cc35ef6a932..bde19e36bd573 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx @@ -2,9 +2,9 @@ import React from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { Domain, EntityType, RecommendationContent } from '../../../../types.generated'; -import { IconStyleType } from '../../../entity/Entity'; import { LogoCountCard } from '../../../shared/LogoCountCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import DomainIcon from '../../../domain/DomainIcon'; const DomainListContainer = styled.div` display: flex; @@ -38,7 +38,7 @@ export const DomainSearchList = ({ content, onClick }: Props) => { > } count={domain.count} /> diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 39035d5bff562..81f7591ef0f3b 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -5,7 +5,7 @@ import { BarChartOutlined, BookOutlined, SettingOutlined, - FolderOutlined, + // FolderOutlined, SolutionOutlined, DownOutlined, } from '@ant-design/icons'; @@ -16,6 +16,7 @@ import { ANTD_GRAY } from '../../entity/shared/constants'; import { HOME_PAGE_INGESTION_ID } from '../../onboarding/config/HomePageOnboardingConfig'; import { useUpdateEducationStepIdsAllowlist } from '../../onboarding/useUpdateEducationStepIdsAllowlist'; import { useUserContext } from '../../context/useUserContext'; +import DomainIcon from '../../domain/DomainIcon'; const LinkWrapper = styled.span` margin-right: 0px; @@ -124,7 +125,8 @@ export function HeaderLinks(props: Props) { - + {/* */} + Domains Manage related groups of data assets diff --git a/datahub-web-react/src/app/shared/tags/DomainLink.tsx b/datahub-web-react/src/app/shared/tags/DomainLink.tsx index 1c14b71369ed6..aaa5fdbf211aa 100644 --- a/datahub-web-react/src/app/shared/tags/DomainLink.tsx +++ b/datahub-web-react/src/app/shared/tags/DomainLink.tsx @@ -3,10 +3,9 @@ import React from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { Domain, EntityType } from '../../../types.generated'; -import { IconStyleType } from '../../entity/Entity'; import { HoverEntityTooltip } from '../../recommendations/renderer/component/HoverEntityTooltip'; import { useEntityRegistry } from '../../useEntityRegistry'; -import { ANTD_GRAY } from '../../entity/shared/constants'; +import DomainIcon from '../../domain/DomainIcon'; const DomainLinkContainer = styled(Link)` display: inline-block; @@ -39,7 +38,7 @@ function DomainContent({ domain, name, closable, onClose, tagStyle, fontSize }: return ( - {entityRegistry.getIcon(EntityType.Domain, fontSize || 10, IconStyleType.ACCENT, ANTD_GRAY[9])} + {displayName} diff --git a/snowflake_golden_test.json b/snowflake_golden_test.json new file mode 100644 index 0000000000000..a424b258e68ff --- /dev/null +++ b/snowflake_golden_test.json @@ -0,0 +1,6065 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "snowflake", + "env": "PROD", + "database": "test_db" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/", + "name": "TEST_DB", + "description": "Comment for TEST_DB", + "created": { + "time": 1623110400000 + }, + "lastModified": { + "time": 1623110400000 + } + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:other_db.other_schema.my_other_tag:other" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "my_other_tag: other", + "description": "Represents the Snowflake tag `OTHER_DB.OTHER_SCHEMA.my_other_tag` with value `other`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "snowflake", + "env": "PROD", + "database": "test_db", + "schema": "test_schema" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/", + "name": "TEST_SCHEMA", + "description": "comment for TEST_DB.TEST_SCHEMA", + "created": { + "time": 1623110400000 + }, + "lastModified": { + "time": 1623110400000 + } + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:other_db.other_schema.my_other_tag:other" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/", + "name": "TABLE_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_2", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/", + "name": "TABLE_2", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_3", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/", + "name": "TABLE_3", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_4", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/", + "name": "TABLE_4", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_5", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/", + "name": "TABLE_5", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_6", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/", + "name": "TABLE_6", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_7", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/", + "name": "TABLE_7", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_8", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/", + "name": "TABLE_8", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_9", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/", + "name": "TABLE_9", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.table_10", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Age" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:Email_Address" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CloudRegion" + } + ], + "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:datahub" + } + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/", + "name": "TABLE_10", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", + "description": "Comment for Table", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "security: pii", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.security` with value `pii`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.view_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:test_db.test_schema.security:pii" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/", + "name": "VIEW_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1", + "description": "Comment for View", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "my_tag_0: my_value_0", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_0` with value `my_value_0`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "my_tag_1: my_value_1", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_1` with value `my_value_1`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", + "changeType": "UPSERT", + "aspectName": "tagProperties", + "aspect": { + "json": { + "name": "my_tag_2: my_value_2", + "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_2` with value `my_value_2`." + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.view_2", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/", + "name": "VIEW_2", + "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2", + "description": "Comment for View", + "created": { + "time": 1623090600000 + }, + "lastModified": { + "time": 1623090600000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0" + }, + { + "tag": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1" + }, + { + "tag": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1654621200000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +} +] \ No newline at end of file diff --git a/test_ingest.yaml b/test_ingest.yaml new file mode 100644 index 0000000000000..f9709d14e25cd --- /dev/null +++ b/test_ingest.yaml @@ -0,0 +1,10 @@ +source: + type: file + config: + filename: snowflake_golden_test.json + +sink: + type: datahub-rest + config: + server: http://localhost:9002/gms + token: "test" \ No newline at end of file From 1e28dc4d9bf09571c44d163a5f2c6338189324a8 Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Mon, 28 Aug 2023 17:58:11 -0700 Subject: [PATCH 32/91] clean up a bit --- .../src/app/domain/DomainsList.tsx | 1 - .../src/app/shared/admin/HeaderLinks.tsx | 2 - snowflake_golden_test.json | 6065 ----------------- test_ingest.yaml | 10 - 4 files changed, 6078 deletions(-) delete mode 100644 snowflake_golden_test.json delete mode 100644 test_ingest.yaml diff --git a/datahub-web-react/src/app/domain/DomainsList.tsx b/datahub-web-react/src/app/domain/DomainsList.tsx index 49ad5299a505d..177144e6f6bd4 100644 --- a/datahub-web-react/src/app/domain/DomainsList.tsx +++ b/datahub-web-react/src/app/domain/DomainsList.tsx @@ -18,7 +18,6 @@ import { OnboardingTour } from '../onboarding/OnboardingTour'; import { DOMAINS_INTRO_ID, DOMAINS_CREATE_DOMAIN_ID } from '../onboarding/config/DomainsOnboardingConfig'; import { getElasticCappedTotalValueText } from '../entity/shared/constants'; import { StyledTable } from '../entity/shared/components/styled/StyledTable'; -// import { IconStyleType } from '../entity/Entity'; import { DomainOwnersColumn, DomainListMenuColumn, DomainNameColumn } from './DomainListColumns'; import DomainIcon from './DomainIcon'; diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 81f7591ef0f3b..87fe331886ee6 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -5,7 +5,6 @@ import { BarChartOutlined, BookOutlined, SettingOutlined, - // FolderOutlined, SolutionOutlined, DownOutlined, } from '@ant-design/icons'; @@ -125,7 +124,6 @@ export function HeaderLinks(props: Props) { - {/* */} Domains diff --git a/snowflake_golden_test.json b/snowflake_golden_test.json deleted file mode 100644 index a424b258e68ff..0000000000000 --- a/snowflake_golden_test.json +++ /dev/null @@ -1,6065 +0,0 @@ -[ -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "snowflake", - "env": "PROD", - "database": "test_db" - }, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/", - "name": "TEST_DB", - "description": "Comment for TEST_DB", - "created": { - "time": 1623110400000 - }, - "lastModified": { - "time": 1623110400000 - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:other_db.other_schema.my_other_tag:other" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", - "changeType": "UPSERT", - "aspectName": "tagProperties", - "aspect": { - "json": { - "name": "my_other_tag: other", - "description": "Represents the Snowflake tag `OTHER_DB.OTHER_SCHEMA.my_other_tag` with value `other`." - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "snowflake", - "env": "PROD", - "database": "test_db", - "schema": "test_schema" - }, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/", - "name": "TEST_SCHEMA", - "description": "comment for TEST_DB.TEST_SCHEMA", - "created": { - "time": 1623110400000 - }, - "lastModified": { - "time": 1623110400000 - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Schema" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:other_db.other_schema.my_other_tag:other" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_1", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/", - "name": "TABLE_1", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_2", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/", - "name": "TABLE_2", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_3", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/", - "name": "TABLE_3", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_4", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/", - "name": "TABLE_4", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_5", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/", - "name": "TABLE_5", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_6", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/", - "name": "TABLE_6", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_7", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/", - "name": "TABLE_7", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_8", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/", - "name": "TABLE_8", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_9", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/", - "name": "TABLE_9", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.table_10", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Age" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Email_Address" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CloudRegion" - } - ], - "auditStamp": { - "time": 1654621200000, - "actor": "urn:li:corpuser:datahub" - } - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/", - "name": "TABLE_10", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", - "description": "Comment for Table", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", - "changeType": "UPSERT", - "aspectName": "tagProperties", - "aspect": { - "json": { - "name": "security: pii", - "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.security` with value `pii`." - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.view_1", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:test_db.test_schema.security:pii" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/", - "name": "VIEW_1", - "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1", - "description": "Comment for View", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", - "changeType": "UPSERT", - "aspectName": "tagProperties", - "aspect": { - "json": { - "name": "my_tag_0: my_value_0", - "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_0` with value `my_value_0`." - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", - "changeType": "UPSERT", - "aspectName": "tagProperties", - "aspect": { - "json": { - "name": "my_tag_1: my_value_1", - "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_1` with value `my_value_1`." - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", - "changeType": "UPSERT", - "aspectName": "tagProperties", - "aspect": { - "json": { - "name": "my_tag_2: my_value_2", - "description": "Represents the Snowflake tag `TEST_DB.TEST_SCHEMA.my_tag_2` with value `my_value_2`." - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "schemaMetadata", - "aspect": { - "json": { - "schemaName": "test_db.test_schema.view_2", - "platform": "urn:li:dataPlatform:snowflake", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "col_1", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "NUMBER(38,0)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_2", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_3", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_4", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_5", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_6", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_7", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_8", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_9", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "col_10", - "nullable": false, - "description": "Comment for column", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(255)", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/", - "name": "VIEW_2", - "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2", - "description": "Comment for View", - "created": { - "time": 1623090600000 - }, - "lastModified": { - "time": 1623090600000 - }, - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2023_08_04-09_52_28" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0" - }, - { - "tag": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1" - }, - { - "tag": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" - } -} -] \ No newline at end of file diff --git a/test_ingest.yaml b/test_ingest.yaml deleted file mode 100644 index f9709d14e25cd..0000000000000 --- a/test_ingest.yaml +++ /dev/null @@ -1,10 +0,0 @@ -source: - type: file - config: - filename: snowflake_golden_test.json - -sink: - type: datahub-rest - config: - server: http://localhost:9002/gms - token: "test" \ No newline at end of file From 0cba2c4f53f61a20c02b45c00952f8c629dedbb6 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 29 Aug 2023 09:40:12 -0700 Subject: [PATCH 33/91] remove console.log --- datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx index ecf11eaaa8af5..1947662b0be7c 100644 --- a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx +++ b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx @@ -175,7 +175,6 @@ export const ManagePolicies = () => { }, }, }); - console.log({ policiesData }); // Any time a policy is removed, edited, or created, refetch the list. const [createPolicy, { error: createPolicyError }] = useCreatePolicyMutation(); From 9fe4e831096a3c91319085ce5b0d66352abf26ef Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 29 Aug 2023 12:59:22 -0400 Subject: [PATCH 34/91] Update Domain search result card to include more info --- .../domain/DomainEntitiesResolver.java | 12 +++-- .../src/app/entity/domain/DomainEntity.tsx | 4 +- .../domain/preview/DomainEntitiesSnippet.tsx | 45 +++++++++++++++++++ .../src/app/entity/domain/preview/Preview.tsx | 9 ++-- datahub-web-react/src/graphql/domain.graphql | 14 +++--- datahub-web-react/src/graphql/search.graphql | 11 +++++ 6 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 06bfa36fc3c14..2a96a521eff58 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -1,6 +1,5 @@ package com.linkedin.datahub.graphql.resolvers.domain; -import com.google.common.collect.ImmutableList; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.DomainEntitiesInput; @@ -67,17 +66,22 @@ public CompletableFuture get(final DataFetchingEnvironment enviro try { + final CriterionArray criteria = new CriterionArray(); final Criterion filterCriterion = new Criterion() .setField(DOMAINS_FIELD_NAME + ".keyword") .setCondition(Condition.EQUAL) .setValue(urn); + criteria.add(filterCriterion); + if (input.getFilters() != null) { + input.getFilters().forEach(filter -> { + criteria.add(new Criterion().setField(filter.getField()).setValue(filter.getValue())); + }); + } return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities( SEARCHABLE_ENTITY_TYPES.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()), query, - new Filter().setOr(new ConjunctiveCriterionArray( - new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of(filterCriterion))) - )), + new Filter().setOr(new ConjunctiveCriterionArray( new ConjunctiveCriterion().setAnd(criteria))), start, count, null, diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index 3b3045abe2a7c..f54a25466a7a0 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -102,11 +102,11 @@ export class DomainEntity implements Entity { renderPreview = (_: PreviewType, data: Domain) => { return ( ); @@ -116,11 +116,11 @@ export class DomainEntity implements Entity { const data = result.entity as Domain; return ( ); diff --git a/datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx b/datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx new file mode 100644 index 0000000000000..6d36964004d64 --- /dev/null +++ b/datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx @@ -0,0 +1,45 @@ +import { DatabaseOutlined, FileDoneOutlined } from '@ant-design/icons'; +import { VerticalDivider } from '@remirror/react'; +import React from 'react'; +import styled from 'styled-components'; +import { SearchResultFields_Domain_Fragment } from '../../../../graphql/search.generated'; +import { ANTD_GRAY_V2 } from '../../shared/constants'; +import DomainIcon from '../../../domain/DomainIcon'; +import { pluralize } from '../../../shared/textUtil'; + +const Wrapper = styled.div` + color: ${ANTD_GRAY_V2[8]}; + font-size: 12px; + display: flex; + align-items: center; + + svg { + margin-right: 4px; + } +`; + +const StyledDivider = styled(VerticalDivider)` + &&& { + margin: 0 8px; + } +`; + +interface Props { + domain: SearchResultFields_Domain_Fragment; +} + +export default function DomainEntitiesSnippet({ domain }: Props) { + const entityCount = domain.entities?.total || 0; + const subDomainCount = domain.children?.total || 0; + const dataProductCount = domain.dataProducts?.total || 0; + + return ( + + {entityCount} {entityCount === 1 ? 'entity' : 'entities'} + + {subDomainCount} {pluralize(subDomainCount, 'sub-domain')} + + {dataProductCount} {pluralize(dataProductCount, 'data product')} + + ); +} diff --git a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx index 18cb2bb75df03..999f51e2a55ee 100644 --- a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx @@ -1,23 +1,24 @@ import React from 'react'; -import { EntityType, Owner, SearchInsight } from '../../../../types.generated'; +import { Domain, EntityType, Owner, SearchInsight } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { IconStyleType } from '../../Entity'; +import DomainEntitiesSnippet from './DomainEntitiesSnippet'; export const Preview = ({ + domain, urn, name, description, owners, - count, insights, logoComponent, }: { + domain: Domain; urn: string; name: string; description?: string | null; owners?: Array | null; - count?: number | null; insights?: Array | null; logoComponent?: JSX.Element; }): JSX.Element => { @@ -33,7 +34,7 @@ export const Preview = ({ owners={owners} insights={insights} logoComponent={logoComponent} - entityCount={count || undefined} + snippet={} /> ); }; diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 081c2104012cf..8c9bed8ab0bea 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -42,19 +42,17 @@ query listDomains($input: ListDomainsInput!) { ownership { ...ownershipFields } - entities(input: { start: 0, count: 1 }) { + entities(input: { start: 0, count: 0 }) { total } - children: relationships( - input: { - types: ["IsPartOf"] - direction: INCOMING - start: 0 - count: 0 - } + dataProducts: entities( + input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } ) { total } + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } } } } diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 172a6d957e287..060f9ac9c5843 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -632,6 +632,17 @@ fragment searchResultFields on Entity { ownership { ...ownershipFields } + entities(input: { start: 0, count: 0 }) { + total + } + dataProducts: entities( + input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } + ) { + total + } + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } } ... on Container { properties { From 8d0d13f5c5aa202575f9826fec3eb88284d6eb78 Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Tue, 29 Aug 2023 10:45:41 -0700 Subject: [PATCH 35/91] get the rest --- .../src/app/domain/DomainIcon.tsx | 4 +-- .../src/app/domain/DomainsList.tsx | 9 +++++- .../src/app/entity/domain/DomainEntity.tsx | 28 +++++++++++-------- .../src/app/entity/domain/preview/Preview.tsx | 9 +++++- .../renderer/component/DomainSearchList.tsx | 9 +++++- .../src/app/shared/admin/HeaderLinks.tsx | 8 +++++- .../src/app/shared/tags/DomainLink.tsx | 8 +++++- 7 files changed, 57 insertions(+), 18 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainIcon.tsx b/datahub-web-react/src/app/domain/DomainIcon.tsx index 4d39a6b17d3aa..ebe3a3e797f3f 100644 --- a/datahub-web-react/src/app/domain/DomainIcon.tsx +++ b/datahub-web-react/src/app/domain/DomainIcon.tsx @@ -2,6 +2,6 @@ import Icon from '@ant-design/icons/lib/components/Icon'; import React from 'react'; import { ReactComponent as DomainsIcon } from '../../images/domain.svg'; -export default function DomainIcon() { - return ; +export default function DomainIcon(style?: any) { + return ; } diff --git a/datahub-web-react/src/app/domain/DomainsList.tsx b/datahub-web-react/src/app/domain/DomainsList.tsx index 177144e6f6bd4..b1095726808fe 100644 --- a/datahub-web-react/src/app/domain/DomainsList.tsx +++ b/datahub-web-react/src/app/domain/DomainsList.tsx @@ -90,7 +90,14 @@ export const DomainsList = () => { sorter: (sourceA, sourceB) => { return sourceA.name.localeCompare(sourceB.name); }, - render: DomainNameColumn(), + render: DomainNameColumn( + , + ), }, { title: 'Owners', diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index 3b3045abe2a7c..ae78d846f3b08 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -1,5 +1,4 @@ import * as React from 'react'; -import { FolderOutlined } from '@ant-design/icons'; import { Domain, EntityType, SearchResult } from '../../../types.generated'; import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { Preview } from './preview/Preview'; @@ -14,7 +13,7 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; import { EntityActionItem } from '../shared/entity/EntityActions'; import DataProductsTab from './DataProductsTab/DataProductsTab'; import { EntityProfileTab } from '../shared/constants'; -// import { EntityActionItem } from '../shared/entity/EntityActions'; +import DomainIcon from '../../domain/DomainIcon'; /** * Definition of the DataHub Domain entity. @@ -22,27 +21,34 @@ import { EntityProfileTab } from '../shared/constants'; export class DomainEntity implements Entity { type: EntityType = EntityType.Domain; - icon = (fontSize: number, styleType: IconStyleType, color?: string) => { + // icon = (fontSize: number, styleType: IconStyleType, color?: string) => { + icon = (fontSize: number, styleType: IconStyleType) => { if (styleType === IconStyleType.TAB_VIEW) { - return ; + return ; } if (styleType === IconStyleType.HIGHLIGHT) { - return ; + return ; + // return ; } if (styleType === IconStyleType.SVG) { return ( - + ); } return ( - ); }; diff --git a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx index 2eae7aee67671..d6877139458b3 100644 --- a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx @@ -29,7 +29,14 @@ export const Preview = ({ urn={urn} description={description || ''} type="Domain" - typeIcon={} + typeIcon={ + + } owners={owners} insights={insights} logoComponent={logoComponent} diff --git a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx index bde19e36bd573..fa38118440d21 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx @@ -38,7 +38,14 @@ export const DomainSearchList = ({ content, onClick }: Props) => { > } + logoComponent={ + + } count={domain.count} /> diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 87fe331886ee6..487be95136fb6 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -124,7 +124,13 @@ export function HeaderLinks(props: Props) { - + Domains Manage related groups of data assets diff --git a/datahub-web-react/src/app/shared/tags/DomainLink.tsx b/datahub-web-react/src/app/shared/tags/DomainLink.tsx index aaa5fdbf211aa..a14114ce43e43 100644 --- a/datahub-web-react/src/app/shared/tags/DomainLink.tsx +++ b/datahub-web-react/src/app/shared/tags/DomainLink.tsx @@ -5,6 +5,7 @@ import styled from 'styled-components'; import { Domain, EntityType } from '../../../types.generated'; import { HoverEntityTooltip } from '../../recommendations/renderer/component/HoverEntityTooltip'; import { useEntityRegistry } from '../../useEntityRegistry'; +import { ANTD_GRAY } from '../../entity/shared/constants'; import DomainIcon from '../../domain/DomainIcon'; const DomainLinkContainer = styled(Link)` @@ -38,7 +39,12 @@ function DomainContent({ domain, name, closable, onClose, tagStyle, fontSize }: return ( - + {displayName} From ca726d225fb6cfb827692cb0159ff6038f2025ea Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Tue, 29 Aug 2023 10:51:36 -0700 Subject: [PATCH 36/91] clean up --- .../src/app/entity/domain/DomainEntity.tsx | 14 ++++++-------- .../renderer/component/DomainSearchList.tsx | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index ae78d846f3b08..0dcff1510d938 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -21,15 +21,13 @@ import DomainIcon from '../../domain/DomainIcon'; export class DomainEntity implements Entity { type: EntityType = EntityType.Domain; - // icon = (fontSize: number, styleType: IconStyleType, color?: string) => { - icon = (fontSize: number, styleType: IconStyleType) => { + icon = (fontSize: number, styleType: IconStyleType, color?: string) => { if (styleType === IconStyleType.TAB_VIEW) { return ; } if (styleType === IconStyleType.HIGHLIGHT) { - return ; - // return ; + return ; } if (styleType === IconStyleType.SVG) { @@ -45,10 +43,10 @@ export class DomainEntity implements Entity { return ( ); }; diff --git a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx index fa38118440d21..37f26f9c4bdb7 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx @@ -41,7 +41,7 @@ export const DomainSearchList = ({ content, onClick }: Props) => { logoComponent={ From 87eb38d53c80c4aa729f06d548fc7114338f3fd5 Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Tue, 29 Aug 2023 13:27:09 -0700 Subject: [PATCH 37/91] PR comment --- datahub-web-react/src/app/domain/DomainIcon.tsx | 6 +++++- datahub-web-react/src/app/shared/admin/HeaderLinks.tsx | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainIcon.tsx b/datahub-web-react/src/app/domain/DomainIcon.tsx index ebe3a3e797f3f..0fe9892f0c281 100644 --- a/datahub-web-react/src/app/domain/DomainIcon.tsx +++ b/datahub-web-react/src/app/domain/DomainIcon.tsx @@ -2,6 +2,10 @@ import Icon from '@ant-design/icons/lib/components/Icon'; import React from 'react'; import { ReactComponent as DomainsIcon } from '../../images/domain.svg'; -export default function DomainIcon(style?: any) { +type Props = { + style?: React.CSSProperties; +}; + +export default function DomainIcon({ style }: Props) { return ; } diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 487be95136fb6..ced7d8642576b 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -128,7 +128,6 @@ export function HeaderLinks(props: Props) { style={{ fontSize: 14, fontWeight: 'bold', - color: '#BFBFBF', }} /> Domains From a0804b690c2bacaea35f63de7c8b753e6d119ada Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Tue, 29 Aug 2023 13:45:46 -0700 Subject: [PATCH 38/91] before testing subdomains --- .../entity/shared/EntityDropdown/EntityDropdown.tsx | 10 ++++++---- .../src/app/entity/shared/EntityDropdown/utils.ts | 5 +++++ datahub-web-react/src/graphql/domain.graphql | 3 +++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index 3442c57ba2d61..48c91930bc171 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -131,6 +131,7 @@ function EntityDropdown(props: Props) { const pageUrl = window.location.href; const isGlossaryEntity = entityType === EntityType.GlossaryNode || entityType === EntityType.GlossaryTerm; + const isDomainEntity = entityType === EntityType.Domain; const entityHasChildren = !!entityData?.children?.total; const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; const canCreateGlossaryEntity = !!entityData?.privileges?.canManageChildren; @@ -210,11 +211,12 @@ function EntityDropdown(props: Props) { onClick={onDeleteEntity} > Date: Tue, 29 Aug 2023 13:47:24 -0700 Subject: [PATCH 39/91] tweak --- .../src/app/entity/shared/EntityDropdown/EntityDropdown.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index 48c91930bc171..4e78e38677b0a 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -212,7 +212,7 @@ function EntityDropdown(props: Props) { > Date: Tue, 29 Aug 2023 18:50:26 -0400 Subject: [PATCH 40/91] Auto open domain navigator to current domain --- .../nestedDomains/ManageDomainsPageV2.tsx | 9 +++++- .../domainNavigator/DomainNode.tsx | 13 +++++++-- .../src/app/entity/shared/types.ts | 3 ++ datahub-web-react/src/app/shared/useToggle.ts | 24 ++++++++++++---- datahub-web-react/src/graphql/domain.graphql | 28 ++++++++++++++++--- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index 2994ea37f1082..eb84c4a1f6942 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -1,12 +1,13 @@ import { Button } from 'antd'; import { PlusOutlined } from '@ant-design/icons'; -import React from 'react'; +import React, { useEffect } from 'react'; import styled from 'styled-components/macro'; import DomainsTitle from './DomainsTitle'; import RootDomains from './RootDomains'; import { DOMAINS_CREATE_DOMAIN_ID, DOMAINS_INTRO_ID } from '../../onboarding/config/DomainsOnboardingConfig'; import { OnboardingTour } from '../../onboarding/OnboardingTour'; import { ANTD_GRAY_V2 } from '../../entity/shared/constants'; +import { useDomainsContext } from '../DomainsContext'; const PageWrapper = styled.div` background-color: ${ANTD_GRAY_V2[1]}; @@ -24,6 +25,12 @@ const Header = styled.div` `; export default function ManageDomainsPageV2() { + const { setEntityData } = useDomainsContext(); + + useEffect(() => { + setEntityData(null); + }, [setEntityData]); + return ( diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index 5188d871ae521..040cf29756a58 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -1,5 +1,5 @@ import { Typography } from 'antd'; -import React from 'react'; +import React, { useEffect, useMemo } from 'react'; import { useHistory } from 'react-router'; import styled from 'styled-components'; import { Domain } from '../../../../types.generated'; @@ -68,13 +68,22 @@ export default function DomainNode({ domain, numDomainChildren }: Props) { const history = useHistory(); const entityRegistry = useEntityRegistry(); const { entityData } = useDomainsContext(); - const { isOpen, isClosing, toggle } = useToggle({ + const { isOpen, isClosing, toggle, toggleOpen } = useToggle({ initialValue: false, closeDelay: 250, }); const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); const isOnEntityPage = entityData && entityData.urn === domain.urn; const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); + // TODO - only auto-open if in select mode - this var is made on a different branch when we intro selectDomainOverride + const shouldAutoOpen = useMemo( + () => entityData?.parentDomains?.domains.some((parent) => parent.urn === domain.urn), + [entityData, domain.urn], + ); + + useEffect(() => { + if (shouldAutoOpen) toggleOpen(); + }, [shouldAutoOpen, toggleOpen]); function handleSelectDomain() { history.push(entityRegistry.getEntityUrl(domain.type, domain.urn)); diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index e36f5050a24b7..6596711d4e82a 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -37,6 +37,7 @@ import { FabricType, BrowsePathV2, DataJobInputOutput, + ParentDomainsResult, } from '../../../types.generated'; import { FetchedEntity } from '../../lineage/types'; @@ -65,6 +66,7 @@ export type EntitySubHeaderSection = { export type GenericEntityProperties = { urn?: string; + type?: EntityType; name?: Maybe; properties?: Maybe<{ description?: Maybe; @@ -98,6 +100,7 @@ export type GenericEntityProperties = { status?: Maybe; deprecation?: Maybe; parentContainers?: Maybe; + parentDomains?: Maybe; children?: Maybe; parentNodes?: Maybe; isAChildren?: Maybe; diff --git a/datahub-web-react/src/app/shared/useToggle.ts b/datahub-web-react/src/app/shared/useToggle.ts index b020bf030f079..a73c702c4351b 100644 --- a/datahub-web-react/src/app/shared/useToggle.ts +++ b/datahub-web-react/src/app/shared/useToggle.ts @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useMemo, useState } from 'react'; const NOOP = (_: boolean) => {}; @@ -9,25 +9,39 @@ const useToggle = ({ initialValue = false, closeDelay = 0, openDelay = 0, onTogg const isClosing = transition === 'closing'; const isTransitioning = transition !== null; - const toggle = () => { - if (isOpen) { + const toggleClose = useMemo( + () => () => { setTransition('closing'); window.setTimeout(() => { setIsOpen(false); setTransition(null); onToggle(false); }, closeDelay); - } else { + }, + [closeDelay, onToggle], + ); + + const toggleOpen = useMemo( + () => () => { setTransition('opening'); window.setTimeout(() => { setIsOpen(true); setTransition(null); onToggle(true); }, openDelay); + }, + [openDelay, onToggle], + ); + + const toggle = () => { + if (isOpen) { + toggleClose(); + } else { + toggleOpen(); } }; - return { isOpen, isClosing, isOpening, isTransitioning, toggle } as const; + return { isOpen, isClosing, isOpening, isTransitioning, toggle, toggleOpen, toggleClose } as const; }; export default useToggle; diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 88a26032c2855..4c7868dca9bbf 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -2,10 +2,24 @@ query getDomain($urn: String!) { domain(urn: $urn) { urn id + type properties { name description } + parentDomains { + count + domains { + urn + type + ... on Domain { + properties { + name + description + } + } + } + } ownership { ...ownershipFields } @@ -35,10 +49,6 @@ query listDomains($input: ListDomainsInput!) { urn id type - parentDomains { - urn - type - } properties { name description @@ -56,6 +66,16 @@ query listDomains($input: ListDomainsInput!) { } } +query getDomainChildrenCount($urn: String!) { + domain(urn: $urn) { + urn + type + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } + } +} + mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) } From 9d1b78efa0ad984136ad86262c805ea597b33e9c Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 29 Aug 2023 17:20:39 -0700 Subject: [PATCH 41/91] fix(nested-domains): better error handling on name collisions --- .../exception/DataHubGraphQLErrorCode.java | 1 + .../domain/CreateDomainResolver.java | 9 +++++- .../resolvers/mutate/UpdateNameResolver.java | 12 +++++++- .../containers/profile/header/EntityName.tsx | 28 +++++++++++++++---- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java index db3e1dd03e419..44695c334855f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java @@ -4,6 +4,7 @@ public enum DataHubGraphQLErrorCode { BAD_REQUEST(400), UNAUTHORIZED(403), NOT_FOUND(404), + CONFLICT(409), SERVER_ERROR(500); private final int _code; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index fe2906e6d5184..13bebdc456139 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -7,6 +7,8 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.CreateDomainInput; import com.linkedin.datahub.graphql.generated.OwnerEntityType; import com.linkedin.datahub.graphql.generated.OwnershipType; @@ -75,7 +77,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws } if (DomainUtils.hasNameConflict(input.getName(), parentDomain, context, _entityClient)) { - throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain!"); + throw new DataHubGraphQLException( + String.format("\"%s\" already exists in this domain. Please pick a unique name.", input.getName()), + DataHubGraphQLErrorCode.CONFLICT + ); } // Create the MCP @@ -91,6 +96,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws } OwnerUtils.addCreatorAsOwner(context, domainUrn, OwnerEntityType.CORP_USER, ownershipType, _entityService); return domainUrn; + } catch (DataHubGraphQLException e) { + throw e; } catch (Exception e) { log.error("Failed to create Domain with id: {}, name: {}: {}", input.getId(), input.getName(), e.getMessage()); throw new RuntimeException(String.format("Failed to create Domain with id: %s, name: %s", input.getId(), input.getName()), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java index 1246821712ca9..00a2c3a80a3fb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java @@ -6,6 +6,8 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.UpdateNameInput; import com.linkedin.datahub.graphql.resolvers.dataproduct.DataProductAuthorizationUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; @@ -125,17 +127,25 @@ private Boolean updateDomainName( try { DomainProperties domainProperties = (DomainProperties) EntityUtils.getAspectFromEntity( targetUrn.toString(), Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, null); + if (domainProperties == null) { throw new IllegalArgumentException("Domain does not exist"); } + if (DomainUtils.hasNameConflict(input.getName(), domainProperties.getParentDomain(), context, _entityClient)) { - throw new IllegalArgumentException("Domain with this name already exists at this level of the Domain!"); + throw new DataHubGraphQLException( + String.format("\"%s\" already exists in this domain. Please pick a unique name.", input.getName()), + DataHubGraphQLErrorCode.CONFLICT + ); } + domainProperties.setName(input.getName()); Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); persistAspect(targetUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, domainProperties, actor, _entityService); return true; + } catch (DataHubGraphQLException e) { + throw e; } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform update against input %s", input), e); } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx index d6df1cf8818df..762bd5f9111a0 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx @@ -33,17 +33,27 @@ function EntityName(props: Props) { const { urn, entityType, entityData } = useEntityData(); const entityName = entityData ? entityRegistry.getDisplayName(entityType, entityData) : ''; const [updatedName, setUpdatedName] = useState(entityName); + const [isEditing, setIsEditing] = useState(false); useEffect(() => { setUpdatedName(entityName); }, [entityName]); - const [updateName] = useUpdateNameMutation(); + const [updateName, { loading: isMutatingName }] = useUpdateNameMutation(); - const handleSaveName = (name: string) => { + const handleStartEditing = () => { + setIsEditing(true); + }; + + const handleChangeName = (name: string) => { + if (name === entityName) { + setIsEditing(false); + return; + } setUpdatedName(name); updateName({ variables: { input: { name, urn } } }) .then(() => { + setIsEditing(false); message.success({ content: 'Name Updated', duration: 2 }); refetch(); if (isInGlossaryContext) { @@ -62,13 +72,19 @@ function EntityName(props: Props) { return ( <> {isNameEditable ? ( - + {updatedName} ) : ( - - {entityData && entityRegistry.getDisplayName(entityType, entityData)} - + {entityName} )} ); From 291c77e0bd66e33405e126b17802d3a1ef151d95 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 13:56:03 -0400 Subject: [PATCH 42/91] add parent node field in modal and set things up --- .../src/app/domain/CreateDomainModal.tsx | 10 +++ .../nestedDomains/ManageDomainsPageV2.tsx | 24 +++++- .../src/app/domain/useListDomains.tsx | 1 + datahub-web-react/src/app/domain/utils.ts | 26 +++++++ .../EntityDropdown/DomainParentSelect.tsx | 64 +++++++++++++++ .../EntityDropdown/NodeParentSelect.tsx | 78 ++++++------------- .../EntityDropdown/useParentSelector.ts | 76 ++++++++++++++++++ 7 files changed, 222 insertions(+), 57 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx create mode 100644 datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index 9fd24b551c0af..128f83591bc75 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -5,6 +5,8 @@ import { useCreateDomainMutation } from '../../graphql/domain.generated'; import { useEnterKeyListener } from '../shared/useEnterKeyListener'; import { validateCustomUrnId } from '../shared/textUtil'; import analytics, { EventType } from '../analytics'; +import { useEntityData } from '../entity/shared/EntityContext'; +import DomainParentSelect from '../entity/shared/EntityDropdown/DomainParentSelect'; const SuggestedNamesGroup = styled.div` margin-top: 12px; @@ -29,6 +31,8 @@ const DESCRIPTION_FIELD_NAME = 'description'; export default function CreateDomainModal({ onClose, onCreate }: Props) { const [createDomainMutation] = useCreateDomainMutation(); + const entityData = useEntityData(); + const [selectedParentUrn, setSelectedParentUrn] = useState(entityData.urn); const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); @@ -101,6 +105,12 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0)); }} > + Parent (optional)}> + + Name}> Give your new Domain a name. { + setEntityData(null); + }, [setEntityData]); + return (
{/* TODO: Add functionality to this button */} -
+ {isCreatingDomain && ( + setIsCreatingDomain(false)} + onCreate={(urn, id, name, description) => + updateListDomainsCache(client, urn, id, name, description) + } + /> + )} ); } diff --git a/datahub-web-react/src/app/domain/useListDomains.tsx b/datahub-web-react/src/app/domain/useListDomains.tsx index 2daea2b5bfa93..ec7aa3eb79552 100644 --- a/datahub-web-react/src/app/domain/useListDomains.tsx +++ b/datahub-web-react/src/app/domain/useListDomains.tsx @@ -15,6 +15,7 @@ export default function useListDomains({ parentDomain, skip }: Props) { parentDomain, }, }, + fetchPolicy: 'cache-first', }); return { data, error, loading, refetch }; diff --git a/datahub-web-react/src/app/domain/utils.ts b/datahub-web-react/src/app/domain/utils.ts index 2bf22db5cca71..3a30a44a95f66 100644 --- a/datahub-web-react/src/app/domain/utils.ts +++ b/datahub-web-react/src/app/domain/utils.ts @@ -1,3 +1,4 @@ +import { ApolloClient } from '@apollo/client'; import { useEffect } from 'react'; import { isEqual } from 'lodash'; import { ListDomainsDocument, ListDomainsQuery } from '../../graphql/domain.generated'; @@ -44,6 +45,31 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { }); }; +export const updateListDomainsCache = ( + client: ApolloClient, + urn: string, + id: string | undefined, + name: string, + description: string | undefined, +) => { + addToListDomainsCache( + client, + { + urn, + id: id || null, + type: EntityType.Domain, + properties: { + name, + description: description || null, + }, + ownership: null, + entities: null, + children: null, + }, + 1000, + ); +}; + /** * Remove an entry from the list domains cache. */ diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx new file mode 100644 index 0000000000000..b8aa1365378c4 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -0,0 +1,64 @@ +import React from 'react'; +import { Select } from 'antd'; +import { EntityType } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import ClickOutside from '../../../shared/ClickOutside'; +import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; +import useParentSelector from './useParentSelector'; +import DomainNavigator from '../../../domain/nestedDomains/domainNavigator/DomainNavigator'; + +interface Props { + selectedParentUrn: string; + setSelectedParentUrn: (parent: string) => void; +} + +export default function DomainParentSelect(props: Props) { + const { selectedParentUrn, setSelectedParentUrn } = props; + const entityRegistry = useEntityRegistry(); + + const { + searchResults, + searchQuery, + isFocusedOnInput, + selectedParentName, + selectParentFromBrowser, + onSelectParent, + handleSearch, + clearSelectedParent, + setIsFocusedOnInput, + } = useParentSelector({ + entityType: EntityType.Domain, + selectedParentUrn, + setSelectedParentUrn, + }); + + // TODO - select from modal + console.log(selectParentFromBrowser); + + const isShowingDomainNavigator = !searchQuery && isFocusedOnInput; + + return ( + setIsFocusedOnInput(false)}> + + + + + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx index 86c2b84a67c3d..46d77cca63f24 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx @@ -1,12 +1,12 @@ -import React, { useState, useEffect } from 'react'; +import React from 'react'; import { Select } from 'antd'; -import { useGetSearchResultsLazyQuery } from '../../../../graphql/search.generated'; -import { EntityType, GlossaryNode } from '../../../../types.generated'; +import { EntityType, GlossaryNode, SearchResult } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { useEntityData } from '../EntityContext'; import ClickOutside from '../../../shared/ClickOutside'; import GlossaryBrowser from '../../../glossary/GlossaryBrowser/GlossaryBrowser'; import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; +import useParentSelector from './useParentSelector'; // filter out entity itself and its children export function filterResultsForMove(entity: GlossaryNode, entityUrn: string) { @@ -25,60 +25,28 @@ interface Props { function NodeParentSelect(props: Props) { const { selectedParentUrn, setSelectedParentUrn, isMoving } = props; - const [selectedParentName, setSelectedParentName] = useState(''); - const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); - const [searchQuery, setSearchQuery] = useState(''); const entityRegistry = useEntityRegistry(); const { entityData, urn: entityDataUrn, entityType } = useEntityData(); - const [nodeSearch, { data: nodeData }] = useGetSearchResultsLazyQuery(); - let nodeSearchResults = nodeData?.search?.searchResults || []; - if (isMoving) { - nodeSearchResults = nodeSearchResults.filter((r) => - filterResultsForMove(r.entity as GlossaryNode, entityDataUrn), - ); - } - - useEffect(() => { - if (entityData && selectedParentUrn === entityDataUrn) { - const displayName = entityRegistry.getDisplayName(EntityType.GlossaryNode, entityData); - setSelectedParentName(displayName); - } - }, [entityData, entityRegistry, selectedParentUrn, entityDataUrn]); - - function handleSearch(text: string) { - setSearchQuery(text); - nodeSearch({ - variables: { - input: { - type: EntityType.GlossaryNode, - query: text, - start: 0, - count: 5, - }, - }, - }); - } + const { + searchResults, + searchQuery, + isFocusedOnInput, + selectedParentName, + selectParentFromBrowser, + onSelectParent, + handleSearch, + clearSelectedParent, + setIsFocusedOnInput, + } = useParentSelector({ + entityType: EntityType.GlossaryNode, + selectedParentUrn, + setSelectedParentUrn, + }); - function onSelectParentNode(parentNodeUrn: string) { - const selectedNode = nodeSearchResults.find((result) => result.entity.urn === parentNodeUrn); - if (selectedNode) { - setSelectedParentUrn(parentNodeUrn); - const displayName = entityRegistry.getDisplayName(selectedNode.entity.type, selectedNode.entity); - setSelectedParentName(displayName); - } - } - - function clearSelectedParent() { - setSelectedParentUrn(''); - setSelectedParentName(''); - setSearchQuery(''); - } - - function selectNodeFromBrowser(urn: string, displayName: string) { - setIsFocusedOnInput(false); - setSelectedParentUrn(urn); - setSelectedParentName(displayName); + let nodeSearchResults: SearchResult[] = []; + if (isMoving) { + nodeSearchResults = searchResults.filter((r) => filterResultsForMove(r.entity as GlossaryNode, entityDataUrn)); } const isShowingGlossaryBrowser = !searchQuery && isFocusedOnInput; @@ -91,7 +59,7 @@ function NodeParentSelect(props: Props) { allowClear filterOption={false} value={selectedParentName} - onSelect={onSelectParentNode} + onSelect={onSelectParent} onSearch={handleSearch} onClear={clearSelectedParent} onFocus={() => setIsFocusedOnInput(true)} @@ -107,7 +75,7 @@ function NodeParentSelect(props: Props) { diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts new file mode 100644 index 0000000000000..be3145d4923a0 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts @@ -0,0 +1,76 @@ +import { useEffect, useState } from 'react'; +import { useEntityData } from '../EntityContext'; +import { useGetSearchResultsLazyQuery } from '../../../../graphql/search.generated'; +import { EntityType } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; + +interface Props { + entityType: EntityType; + selectedParentUrn: string; + setSelectedParentUrn: (parent: string) => void; +} + +export default function useParentSelector({ entityType, selectedParentUrn, setSelectedParentUrn }: Props) { + const [selectedParentName, setSelectedParentName] = useState(''); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); + const [searchQuery, setSearchQuery] = useState(''); + const entityRegistry = useEntityRegistry(); + const { entityData, urn: entityDataUrn } = useEntityData(); + + const [search, { data }] = useGetSearchResultsLazyQuery(); + const searchResults = data?.search?.searchResults || []; + + useEffect(() => { + if (entityData && selectedParentUrn === entityDataUrn) { + const displayName = entityRegistry.getDisplayName(entityType, entityData); + setSelectedParentName(displayName); + } + }, [entityData, entityRegistry, selectedParentUrn, entityDataUrn, entityType]); + + function handleSearch(text: string) { + setSearchQuery(text); + search({ + variables: { + input: { + type: entityType, + query: text, + start: 0, + count: 5, + }, + }, + }); + } + + function onSelectParent(parentUrn: string) { + const selectedParent = searchResults.find((result) => result.entity.urn === parentUrn); + if (selectedParent) { + setSelectedParentUrn(parentUrn); + const displayName = entityRegistry.getDisplayName(selectedParent.entity.type, selectedParent.entity); + setSelectedParentName(displayName); + } + } + + function clearSelectedParent() { + setSelectedParentUrn(''); + setSelectedParentName(''); + setSearchQuery(''); + } + + function selectParentFromBrowser(urn: string, displayName: string) { + setIsFocusedOnInput(false); + setSelectedParentUrn(urn); + setSelectedParentName(displayName); + } + + return { + searchQuery, + searchResults, + isFocusedOnInput, + selectedParentName, + onSelectParent, + handleSearch, + setIsFocusedOnInput, + selectParentFromBrowser, + clearSelectedParent, + }; +} From 1a46978443c700c08bf1cabfeed20689b90da5be Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 16:25:54 -0400 Subject: [PATCH 43/91] allow selecting of parents in modals and update parent cache --- .../src/app/domain/CreateDomainModal.tsx | 12 ++++++++++-- .../domain/nestedDomains/ManageDomainsPageV2.tsx | 5 ++--- .../domainNavigator/DomainNavigator.tsx | 7 ++++++- .../nestedDomains/domainNavigator/DomainNode.tsx | 15 +++++++++++---- .../src/app/domain/useListDomains.tsx | 3 ++- datahub-web-react/src/app/domain/utils.ts | 6 +++++- .../shared/EntityDropdown/DomainParentSelect.tsx | 5 +---- 7 files changed, 37 insertions(+), 16 deletions(-) diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index 128f83591bc75..f928de3316579 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -20,7 +20,13 @@ const ClickableTag = styled(Tag)` type Props = { onClose: () => void; - onCreate: (urn: string, id: string | undefined, name: string, description: string | undefined) => void; + onCreate: ( + urn: string, + id: string | undefined, + name: string, + description: string | undefined, + parentDomain?: string, + ) => void; }; const SUGGESTED_DOMAIN_NAMES = ['Engineering', 'Marketing', 'Sales', 'Product']; @@ -32,7 +38,7 @@ const DESCRIPTION_FIELD_NAME = 'description'; export default function CreateDomainModal({ onClose, onCreate }: Props) { const [createDomainMutation] = useCreateDomainMutation(); const entityData = useEntityData(); - const [selectedParentUrn, setSelectedParentUrn] = useState(entityData.urn); + const [selectedParentUrn, setSelectedParentUrn] = useState(entityData.urn); const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); @@ -43,6 +49,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { id: form.getFieldValue(ID_FIELD_NAME), name: form.getFieldValue(NAME_FIELD_NAME), description: form.getFieldValue(DESCRIPTION_FIELD_NAME), + parentDomain: selectedParentUrn || undefined, }, }, }) @@ -60,6 +67,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { form.getFieldValue(ID_FIELD_NAME), form.getFieldValue(NAME_FIELD_NAME), form.getFieldValue(DESCRIPTION_FIELD_NAME), + selectedParentUrn || undefined, ); form.resetFields(); } diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index 3c080e15a4c25..a9e505d9dc381 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -41,7 +41,6 @@ export default function ManageDomainsPageV2() {
- {/* TODO: Add functionality to this button */} @@ -50,8 +49,8 @@ export default function ManageDomainsPageV2() { {isCreatingDomain && ( setIsCreatingDomain(false)} - onCreate={(urn, id, name, description) => - updateListDomainsCache(client, urn, id, name, description) + onCreate={(urn, id, name, description, parentDomain) => + updateListDomainsCache(client, urn, id, name, description, parentDomain) } /> )} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx index 03c91f166a731..eb6ab11cf8c68 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -12,7 +12,11 @@ const NavigatorWrapper = styled.div` overflow: auto; `; -export default function DomainNavigator() { +interface Props { + selectDomainOverride?: (urn: string, displayName: string) => void; +} + +export default function DomainNavigator({ selectDomainOverride }: Props) { const { data, error } = useListDomains({}); return ( @@ -23,6 +27,7 @@ export default function DomainNavigator() { key={domain.urn} domain={domain as Domain} numDomainChildren={domain.children?.total || 0} + selectDomainOverride={selectDomainOverride} /> ))} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index 5188d871ae521..3c9e789095f0c 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -62,9 +62,10 @@ const StyledExpander = styled(BodyGridExpander)` interface Props { domain: Domain; numDomainChildren: number; + selectDomainOverride?: (urn: string, displayName: string) => void; } -export default function DomainNode({ domain, numDomainChildren }: Props) { +export default function DomainNode({ domain, numDomainChildren, selectDomainOverride }: Props) { const history = useHistory(); const entityRegistry = useEntityRegistry(); const { entityData } = useDomainsContext(); @@ -75,9 +76,14 @@ export default function DomainNode({ domain, numDomainChildren }: Props) { const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); const isOnEntityPage = entityData && entityData.urn === domain.urn; const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); + const isInSelectMode = !!selectDomainOverride; function handleSelectDomain() { - history.push(entityRegistry.getEntityUrl(domain.type, domain.urn)); + if (selectDomainOverride) { + selectDomainOverride(domain.urn, displayName); + } else { + history.push(entityRegistry.getEntityUrl(domain.type, domain.urn)); + } } return ( @@ -91,10 +97,10 @@ export default function DomainNode({ domain, numDomainChildren }: Props) { - + {!isInSelectMode && } {displayName} @@ -105,6 +111,7 @@ export default function DomainNode({ domain, numDomainChildren }: Props) { key={domain.urn} domain={childDomain as Domain} numDomainChildren={childDomain.children?.total || 0} + selectDomainOverride={selectDomainOverride} /> ))} diff --git a/datahub-web-react/src/app/domain/useListDomains.tsx b/datahub-web-react/src/app/domain/useListDomains.tsx index ec7aa3eb79552..932b3080817de 100644 --- a/datahub-web-react/src/app/domain/useListDomains.tsx +++ b/datahub-web-react/src/app/domain/useListDomains.tsx @@ -15,7 +15,8 @@ export default function useListDomains({ parentDomain, skip }: Props) { parentDomain, }, }, - fetchPolicy: 'cache-first', + fetchPolicy: 'network-only', // always use network request first to populate cache + nextFetchPolicy: 'cache-first', // then use cache after that so we can manipulate it }); return { data, error, loading, refetch }; diff --git a/datahub-web-react/src/app/domain/utils.ts b/datahub-web-react/src/app/domain/utils.ts index 3a30a44a95f66..19097ab77bbcc 100644 --- a/datahub-web-react/src/app/domain/utils.ts +++ b/datahub-web-react/src/app/domain/utils.ts @@ -10,7 +10,7 @@ import { useDomainsContext } from './DomainsContext'; /** * Add an entry to the list domains cache. */ -export const addToListDomainsCache = (client, newDomain, pageSize) => { +export const addToListDomainsCache = (client, newDomain, pageSize, parentDomain?: string) => { // Read the data from our cache for this query. const currData: ListDomainsQuery | null = client.readQuery({ query: ListDomainsDocument, @@ -18,6 +18,7 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { input: { start: 0, count: pageSize, + parentDomain, }, }, }); @@ -32,6 +33,7 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { input: { start: 0, count: pageSize, + parentDomain, }, }, data: { @@ -51,6 +53,7 @@ export const updateListDomainsCache = ( id: string | undefined, name: string, description: string | undefined, + parentDomain?: string, ) => { addToListDomainsCache( client, @@ -67,6 +70,7 @@ export const updateListDomainsCache = ( children: null, }, 1000, + parentDomain, ); }; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index b8aa1365378c4..ab66d22380b22 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -32,9 +32,6 @@ export default function DomainParentSelect(props: Props) { setSelectedParentUrn, }); - // TODO - select from modal - console.log(selectParentFromBrowser); - const isShowingDomainNavigator = !searchQuery && isFocusedOnInput; return ( @@ -57,7 +54,7 @@ export default function DomainParentSelect(props: Props) { ))} - + ); From 4d7962215cd234e87e35672f62767376684295bb Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 16:51:26 -0400 Subject: [PATCH 44/91] implement parentDomainsToUpdate after creating a domain --- .../src/app/domain/DomainRoutes.tsx | 3 ++- .../src/app/domain/DomainsContext.tsx | 8 +++++-- .../nestedDomains/ManageDomainsPageV2.tsx | 9 ++++---- .../domainNavigator/DomainNode.tsx | 21 +++++++++++++++---- datahub-web-react/src/graphql/domain.graphql | 10 +++++++++ 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainRoutes.tsx b/datahub-web-react/src/app/domain/DomainRoutes.tsx index d4e048789dbdd..ed912cacf8fdd 100644 --- a/datahub-web-react/src/app/domain/DomainRoutes.tsx +++ b/datahub-web-react/src/app/domain/DomainRoutes.tsx @@ -19,9 +19,10 @@ const ContentWrapper = styled.div` export default function DomainRoutes() { const entityRegistry = useEntityRegistry(); const [entityData, setEntityData] = useState(null); + const [parentDomainsToUpate, setParentDomainsToUpdate] = useState([]); return ( - + diff --git a/datahub-web-react/src/app/domain/DomainsContext.tsx b/datahub-web-react/src/app/domain/DomainsContext.tsx index 3a6523e31a001..44a81d31a4d2c 100644 --- a/datahub-web-react/src/app/domain/DomainsContext.tsx +++ b/datahub-web-react/src/app/domain/DomainsContext.tsx @@ -4,14 +4,18 @@ import { GenericEntityProperties } from '../entity/shared/types'; export interface DomainsContextType { entityData: GenericEntityProperties | null; setEntityData: (entityData: GenericEntityProperties | null) => void; + parentDomainsToUpate: string[]; + setParentDomainsToUpdate: (values: string[]) => void; } export const DomainsContext = React.createContext({ entityData: null, setEntityData: () => {}, + parentDomainsToUpate: [], // used to tell domains to refetch their children count after updates (create, move, delete) + setParentDomainsToUpdate: () => {}, }); export const useDomainsContext = () => { - const { entityData, setEntityData } = useContext(DomainsContext); - return { entityData, setEntityData }; + const { entityData, setEntityData, parentDomainsToUpate, setParentDomainsToUpdate } = useContext(DomainsContext); + return { entityData, setEntityData, parentDomainsToUpate, setParentDomainsToUpdate }; }; diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx index a9e505d9dc381..486169c3559d3 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -28,7 +28,7 @@ const Header = styled.div` `; export default function ManageDomainsPageV2() { - const { setEntityData } = useDomainsContext(); + const { setEntityData, setParentDomainsToUpdate } = useDomainsContext(); const [isCreatingDomain, setIsCreatingDomain] = useState(false); const client = useApolloClient(); @@ -49,9 +49,10 @@ export default function ManageDomainsPageV2() { {isCreatingDomain && ( setIsCreatingDomain(false)} - onCreate={(urn, id, name, description, parentDomain) => - updateListDomainsCache(client, urn, id, name, description, parentDomain) - } + onCreate={(urn, id, name, description, parentDomain) => { + updateListDomainsCache(client, urn, id, name, description, parentDomain); + if (parentDomain) setParentDomainsToUpdate([parentDomain]); + }} /> )} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index 3c9e789095f0c..78c7ae52d0720 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -1,5 +1,5 @@ import { Typography } from 'antd'; -import React from 'react'; +import React, { useEffect } from 'react'; import { useHistory } from 'react-router'; import styled from 'styled-components'; import { Domain } from '../../../../types.generated'; @@ -12,6 +12,7 @@ import { BodyContainer, BodyGridExpander } from '../../../shared/components'; import { ANTD_GRAY_V2 } from '../../../entity/shared/constants'; import { useDomainsContext } from '../../DomainsContext'; import { applyOpacity } from '../../../shared/styleUtils'; +import { useGetDomainChildrenCountLazyQuery } from '../../../../graphql/domain.generated'; const RowWrapper = styled.div` align-items: center; @@ -68,15 +69,17 @@ interface Props { export default function DomainNode({ domain, numDomainChildren, selectDomainOverride }: Props) { const history = useHistory(); const entityRegistry = useEntityRegistry(); - const { entityData } = useDomainsContext(); + const { entityData, parentDomainsToUpate, setParentDomainsToUpdate } = useDomainsContext(); const { isOpen, isClosing, toggle } = useToggle({ initialValue: false, closeDelay: 250, }); const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); + const [getDomainChildrenCount, { data: childrenData }] = useGetDomainChildrenCountLazyQuery(); const isOnEntityPage = entityData && entityData.urn === domain.urn; const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); const isInSelectMode = !!selectDomainOverride; + const hasDomainChildren = childrenData ? !!childrenData.domain?.children?.total : !!numDomainChildren; function handleSelectDomain() { if (selectDomainOverride) { @@ -86,10 +89,20 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver } } + useEffect(() => { + // fetch updated children count to determine if we show triangle toggle + if (parentDomainsToUpate.includes(domain.urn)) { + setTimeout(() => { + getDomainChildrenCount({ variables: { urn: domain.urn } }); + setParentDomainsToUpdate(parentDomainsToUpate.filter((urn) => urn !== domain.urn)); + }, 2000); + } + }); + return ( <> - {!!numDomainChildren && ( + {hasDomainChildren && ( @@ -98,7 +111,7 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver ellipsis={{ tooltip: displayName }} onClick={handleSelectDomain} isSelected={!!isOnEntityPage && !isInSelectMode} - addLeftPadding={!numDomainChildren} + addLeftPadding={!hasDomainChildren} > {!isInSelectMode && } {displayName} diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 88a26032c2855..5bd92704bcb4f 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -56,6 +56,16 @@ query listDomains($input: ListDomainsInput!) { } } +query getDomainChildrenCount($urn: String!) { + domain(urn: $urn) { + urn + type + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } + } +} + mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) } From 465a875665bc88479d9cc82c8ef5f102739b7233 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 28 Aug 2023 19:28:09 -0400 Subject: [PATCH 45/91] final pieces for create domain --- .../src/app/domain/CreateDomainModal.tsx | 6 ++--- .../nestedDomains/DomainsSidebarHeader.tsx | 22 +++++++++++++--- .../domainNavigator/DomainNode.tsx | 19 +++----------- .../domainNavigator/useHasDomainChildren.ts | 25 +++++++++++++++++++ .../EntityDropdown/DomainParentSelect.tsx | 3 +++ .../EntityDropdown/NodeParentSelect.tsx | 1 + .../EntityDropdown/useParentSelector.ts | 10 ++++---- 7 files changed, 60 insertions(+), 26 deletions(-) create mode 100644 datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index f928de3316579..d17ba21bb8d3f 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -5,8 +5,8 @@ import { useCreateDomainMutation } from '../../graphql/domain.generated'; import { useEnterKeyListener } from '../shared/useEnterKeyListener'; import { validateCustomUrnId } from '../shared/textUtil'; import analytics, { EventType } from '../analytics'; -import { useEntityData } from '../entity/shared/EntityContext'; import DomainParentSelect from '../entity/shared/EntityDropdown/DomainParentSelect'; +import { useDomainsContext } from './DomainsContext'; const SuggestedNamesGroup = styled.div` margin-top: 12px; @@ -37,8 +37,8 @@ const DESCRIPTION_FIELD_NAME = 'description'; export default function CreateDomainModal({ onClose, onCreate }: Props) { const [createDomainMutation] = useCreateDomainMutation(); - const entityData = useEntityData(); - const [selectedParentUrn, setSelectedParentUrn] = useState(entityData.urn); + const { entityData } = useDomainsContext(); + const [selectedParentUrn, setSelectedParentUrn] = useState(entityData?.urn || ''); const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); diff --git a/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx index f70b7e1d28d33..d9ff18514d8cf 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx @@ -1,11 +1,15 @@ +import { useApolloClient } from '@apollo/client'; import { PlusOutlined } from '@ant-design/icons'; import { Button } from 'antd'; -import React from 'react'; +import React, { useState } from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { ANTD_GRAY, ANTD_GRAY_V2 } from '../../entity/shared/constants'; import DomainsTitle from './DomainsTitle'; import { PageRoutes } from '../../../conf/Global'; +import CreateDomainModal from '../CreateDomainModal'; +import { updateListDomainsCache } from '../utils'; +import { useDomainsContext } from '../DomainsContext'; const HeaderWrapper = styled.div` border-bottom: 1px solid ${ANTD_GRAY[4]}; @@ -30,13 +34,25 @@ const StyledLink = styled(Link)` `; export default function DomainsSidebarHeader() { + const { setParentDomainsToUpdate } = useDomainsContext(); + const [isCreatingDomain, setIsCreatingDomain] = useState(false); + const client = useApolloClient(); + return ( - {/* TODO - give functionality to this button */} - } /> + } onClick={() => setIsCreatingDomain(true)} /> + {isCreatingDomain && ( + setIsCreatingDomain(false)} + onCreate={(urn, id, name, description, parentDomain) => { + updateListDomainsCache(client, urn, id, name, description, parentDomain); + if (parentDomain) setParentDomainsToUpdate([parentDomain]); + }} + /> + )} ); } diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index 78c7ae52d0720..22aa8edc3b169 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -1,5 +1,5 @@ import { Typography } from 'antd'; -import React, { useEffect } from 'react'; +import React from 'react'; import { useHistory } from 'react-router'; import styled from 'styled-components'; import { Domain } from '../../../../types.generated'; @@ -12,7 +12,7 @@ import { BodyContainer, BodyGridExpander } from '../../../shared/components'; import { ANTD_GRAY_V2 } from '../../../entity/shared/constants'; import { useDomainsContext } from '../../DomainsContext'; import { applyOpacity } from '../../../shared/styleUtils'; -import { useGetDomainChildrenCountLazyQuery } from '../../../../graphql/domain.generated'; +import useHasDomainChildren from './useHasDomainChildren'; const RowWrapper = styled.div` align-items: center; @@ -69,17 +69,16 @@ interface Props { export default function DomainNode({ domain, numDomainChildren, selectDomainOverride }: Props) { const history = useHistory(); const entityRegistry = useEntityRegistry(); - const { entityData, parentDomainsToUpate, setParentDomainsToUpdate } = useDomainsContext(); + const { entityData } = useDomainsContext(); const { isOpen, isClosing, toggle } = useToggle({ initialValue: false, closeDelay: 250, }); const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); - const [getDomainChildrenCount, { data: childrenData }] = useGetDomainChildrenCountLazyQuery(); const isOnEntityPage = entityData && entityData.urn === domain.urn; const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); const isInSelectMode = !!selectDomainOverride; - const hasDomainChildren = childrenData ? !!childrenData.domain?.children?.total : !!numDomainChildren; + const hasDomainChildren = useHasDomainChildren({ domainUrn: domain.urn, numDomainChildren }); function handleSelectDomain() { if (selectDomainOverride) { @@ -89,16 +88,6 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver } } - useEffect(() => { - // fetch updated children count to determine if we show triangle toggle - if (parentDomainsToUpate.includes(domain.urn)) { - setTimeout(() => { - getDomainChildrenCount({ variables: { urn: domain.urn } }); - setParentDomainsToUpdate(parentDomainsToUpate.filter((urn) => urn !== domain.urn)); - }, 2000); - } - }); - return ( <> diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts new file mode 100644 index 0000000000000..58aa6629d541f --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts @@ -0,0 +1,25 @@ +import { useEffect } from 'react'; +import { useGetDomainChildrenCountLazyQuery } from '../../../../graphql/domain.generated'; +import { useDomainsContext } from '../../DomainsContext'; + +interface Props { + domainUrn: string; + numDomainChildren: number; // number that comes from parent query to render this domain +} + +export default function useHasDomainChildren({ domainUrn, numDomainChildren }: Props) { + const { parentDomainsToUpate, setParentDomainsToUpdate } = useDomainsContext(); + const [getDomainChildrenCount, { data: childrenData }] = useGetDomainChildrenCountLazyQuery(); + + useEffect(() => { + // fetch updated children count to determine if we show triangle toggle + if (parentDomainsToUpate.includes(domainUrn)) { + setTimeout(() => { + getDomainChildrenCount({ variables: { urn: domainUrn } }); + setParentDomainsToUpdate(parentDomainsToUpate.filter((urn) => urn !== domainUrn)); + }, 2000); + } + }); + + return childrenData ? !!childrenData.domain?.children?.total : !!numDomainChildren; +} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index ab66d22380b22..bcd93ddae6532 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -6,6 +6,7 @@ import ClickOutside from '../../../shared/ClickOutside'; import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; import useParentSelector from './useParentSelector'; import DomainNavigator from '../../../domain/nestedDomains/domainNavigator/DomainNavigator'; +import { useDomainsContext } from '../../../domain/DomainsContext'; interface Props { selectedParentUrn: string; @@ -15,6 +16,7 @@ interface Props { export default function DomainParentSelect(props: Props) { const { selectedParentUrn, setSelectedParentUrn } = props; const entityRegistry = useEntityRegistry(); + const { entityData } = useDomainsContext(); const { searchResults, @@ -28,6 +30,7 @@ export default function DomainParentSelect(props: Props) { setIsFocusedOnInput, } = useParentSelector({ entityType: EntityType.Domain, + entityData, selectedParentUrn, setSelectedParentUrn, }); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx index 46d77cca63f24..c3bfac35c2ca6 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx @@ -40,6 +40,7 @@ function NodeParentSelect(props: Props) { setIsFocusedOnInput, } = useParentSelector({ entityType: EntityType.GlossaryNode, + entityData, selectedParentUrn, setSelectedParentUrn, }); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts index be3145d4923a0..a9d4ffa6ee747 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts @@ -1,31 +1,31 @@ import { useEffect, useState } from 'react'; -import { useEntityData } from '../EntityContext'; import { useGetSearchResultsLazyQuery } from '../../../../graphql/search.generated'; import { EntityType } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import { GenericEntityProperties } from '../types'; interface Props { entityType: EntityType; + entityData: GenericEntityProperties | null; selectedParentUrn: string; setSelectedParentUrn: (parent: string) => void; } -export default function useParentSelector({ entityType, selectedParentUrn, setSelectedParentUrn }: Props) { +export default function useParentSelector({ entityType, entityData, selectedParentUrn, setSelectedParentUrn }: Props) { const [selectedParentName, setSelectedParentName] = useState(''); const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); const [searchQuery, setSearchQuery] = useState(''); const entityRegistry = useEntityRegistry(); - const { entityData, urn: entityDataUrn } = useEntityData(); const [search, { data }] = useGetSearchResultsLazyQuery(); const searchResults = data?.search?.searchResults || []; useEffect(() => { - if (entityData && selectedParentUrn === entityDataUrn) { + if (entityData && selectedParentUrn === entityData.urn) { const displayName = entityRegistry.getDisplayName(entityType, entityData); setSelectedParentName(displayName); } - }, [entityData, entityRegistry, selectedParentUrn, entityDataUrn, entityType]); + }, [entityData, entityRegistry, selectedParentUrn, entityData?.urn, entityType]); function handleSearch(text: string) { setSearchQuery(text); From 0581fde40be94ab822e0d55270a535977481efa5 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 30 Aug 2023 13:22:28 -0400 Subject: [PATCH 46/91] update useEffect --- .../nestedDomains/domainNavigator/useHasDomainChildren.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts index 58aa6629d541f..1d6b83863b0ed 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts @@ -12,14 +12,18 @@ export default function useHasDomainChildren({ domainUrn, numDomainChildren }: P const [getDomainChildrenCount, { data: childrenData }] = useGetDomainChildrenCountLazyQuery(); useEffect(() => { + let timer; // fetch updated children count to determine if we show triangle toggle if (parentDomainsToUpate.includes(domainUrn)) { - setTimeout(() => { + timer = setTimeout(() => { getDomainChildrenCount({ variables: { urn: domainUrn } }); setParentDomainsToUpdate(parentDomainsToUpate.filter((urn) => urn !== domainUrn)); }, 2000); } - }); + return () => { + if (timer) window.clearTimeout(timer); + }; + }, [domainUrn, getDomainChildrenCount, parentDomainsToUpate, setParentDomainsToUpdate]); return childrenData ? !!childrenData.domain?.children?.total : !!numDomainChildren; } From e1ca30289e6fb0135102e1177084850bd79b8abc Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 29 Aug 2023 16:45:55 -0400 Subject: [PATCH 47/91] Handle deleting a domain and update sidebar --- datahub-web-react/src/app/domain/utils.ts | 4 ++- .../shared/EntityDropdown/useDeleteEntity.tsx | 7 +++++ .../EntityDropdown/useHandleDeleteDomain.ts | 27 +++++++++++++++++++ .../src/app/entity/shared/types.ts | 3 +++ .../src/app/shared/deleteUtils.ts | 4 ++- datahub-web-react/src/graphql/domain.graphql | 18 ++++++++++--- 6 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts diff --git a/datahub-web-react/src/app/domain/utils.ts b/datahub-web-react/src/app/domain/utils.ts index 2bf22db5cca71..68e0cccfaed59 100644 --- a/datahub-web-react/src/app/domain/utils.ts +++ b/datahub-web-react/src/app/domain/utils.ts @@ -47,7 +47,7 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { /** * Remove an entry from the list domains cache. */ -export const removeFromListDomainsCache = (client, urn, page, pageSize) => { +export const removeFromListDomainsCache = (client, urn, page, pageSize, parentDomain?: string) => { // Read the data from our cache for this query. const currData: ListDomainsQuery | null = client.readQuery({ query: ListDomainsDocument, @@ -55,6 +55,7 @@ export const removeFromListDomainsCache = (client, urn, page, pageSize) => { input: { start: (page - 1) * pageSize, count: pageSize, + parentDomain, }, }, }); @@ -69,6 +70,7 @@ export const removeFromListDomainsCache = (client, urn, page, pageSize) => { input: { start: (page - 1) * pageSize, count: pageSize, + parentDomain, }, }, data: { diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx index c4647b995337b..1e4737135ed74 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx @@ -6,6 +6,7 @@ import { getDeleteEntityMutation } from '../../../shared/deleteUtils'; import analytics, { EventType } from '../../../analytics'; import { useGlossaryEntityData } from '../GlossaryEntityContext'; import { getParentNodeToUpdate, updateGlossarySidebar } from '../../../glossary/utils'; +import { useHandleDeleteDomain } from './useHandleDeleteDomain'; /** * Performs the flow for deleting an entity of a given type. @@ -25,6 +26,7 @@ function useDeleteEntity( const [hasBeenDeleted, setHasBeenDeleted] = useState(false); const entityRegistry = useEntityRegistry(); const { isInGlossaryContext, urnsToUpdate, setUrnsToUpdate } = useGlossaryEntityData(); + const { handleDeleteDomain } = useHandleDeleteDomain({ entityData, urn }); const maybeDeleteEntity = getDeleteEntityMutation(type)(); const deleteEntity = (maybeDeleteEntity && maybeDeleteEntity[0]) || undefined; @@ -47,6 +49,11 @@ function useDeleteEntity( duration: 2, }); } + + if (entityData.type === EntityType.Domain) { + handleDeleteDomain(); + } + setTimeout( () => { setHasBeenDeleted(true); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts new file mode 100644 index 0000000000000..0479ea0c5f5d1 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts @@ -0,0 +1,27 @@ +import { useApolloClient } from '@apollo/client'; +import { GenericEntityProperties } from '../types'; +import { removeFromListDomainsCache } from '../../../domain/utils'; +import { useDomainsContext } from '../../../domain/DomainsContext'; + +interface DeleteDomainProps { + entityData: GenericEntityProperties; + urn: string; +} + +export function useHandleDeleteDomain({ entityData, urn }: DeleteDomainProps) { + const client = useApolloClient(); + const { parentDomainsToUpate, setParentDomainsToUpdate } = useDomainsContext(); + + const handleDeleteDomain = () => { + if (entityData.parentDomains && entityData.parentDomains.domains.length > 0) { + const parentDomainUrn = entityData.parentDomains.domains[0].urn; + + removeFromListDomainsCache(client, urn, 1, 1000, parentDomainUrn); + setParentDomainsToUpdate([...parentDomainsToUpate, parentDomainUrn]); + } else { + removeFromListDomainsCache(client, urn, 1, 1000); + } + }; + + return { handleDeleteDomain }; +} diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index e36f5050a24b7..6596711d4e82a 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -37,6 +37,7 @@ import { FabricType, BrowsePathV2, DataJobInputOutput, + ParentDomainsResult, } from '../../../types.generated'; import { FetchedEntity } from '../../lineage/types'; @@ -65,6 +66,7 @@ export type EntitySubHeaderSection = { export type GenericEntityProperties = { urn?: string; + type?: EntityType; name?: Maybe; properties?: Maybe<{ description?: Maybe; @@ -98,6 +100,7 @@ export type GenericEntityProperties = { status?: Maybe; deprecation?: Maybe; parentContainers?: Maybe; + parentDomains?: Maybe; children?: Maybe; parentNodes?: Maybe; isAChildren?: Maybe; diff --git a/datahub-web-react/src/app/shared/deleteUtils.ts b/datahub-web-react/src/app/shared/deleteUtils.ts index c1bfeac37372b..37a3758712ad6 100644 --- a/datahub-web-react/src/app/shared/deleteUtils.ts +++ b/datahub-web-react/src/app/shared/deleteUtils.ts @@ -1,3 +1,4 @@ +import { PageRoutes } from '../../conf/Global'; import { useDeleteAssertionMutation } from '../../graphql/assertion.generated'; import { useDeleteDataProductMutation } from '../../graphql/dataProduct.generated'; import { useDeleteDomainMutation } from '../../graphql/domain.generated'; @@ -18,10 +19,11 @@ export const getEntityProfileDeleteRedirectPath = (type: EntityType, entityData: switch (type) { case EntityType.CorpGroup: case EntityType.CorpUser: - case EntityType.Domain: case EntityType.Tag: // Return Home. return '/'; + case EntityType.Domain: + return `${PageRoutes.DOMAINS}`; case EntityType.GlossaryNode: case EntityType.GlossaryTerm: // Return to glossary page. diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 88a26032c2855..5967033988c73 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -2,10 +2,24 @@ query getDomain($urn: String!) { domain(urn: $urn) { urn id + type properties { name description } + parentDomains { + count + domains { + urn + type + ... on Domain { + properties { + name + description + } + } + } + } ownership { ...ownershipFields } @@ -35,10 +49,6 @@ query listDomains($input: ListDomainsInput!) { urn id type - parentDomains { - urn - type - } properties { name description From 8be73d6a58a2a7dd6cb6767b2b290933dc0a05f0 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 30 Aug 2023 10:51:39 -0700 Subject: [PATCH 48/91] fix(nested-domains): unbreak gql file --- datahub-web-react/src/graphql/domain.graphql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 25222fb6d6e0b..64d0ef9e9bf90 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -62,6 +62,8 @@ query listDomains($input: ListDomainsInput!) { dataProducts: entities( input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } ) { + total + } children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { total } From e087183cd660ccc42d1498c92d97566e4a6c185a Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Tue, 29 Aug 2023 09:30:26 -0700 Subject: [PATCH 49/91] add search to sidebar --- .../src/app/domain/DomainSearch.tsx | 120 ++++++++++++++++++ .../nestedDomains/ManageDomainsSidebar.tsx | 2 + .../app/entity/shared/DomainEntityContext.tsx | 27 ++++ 3 files changed, 149 insertions(+) create mode 100644 datahub-web-react/src/app/domain/DomainSearch.tsx create mode 100644 datahub-web-react/src/app/entity/shared/DomainEntityContext.tsx diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx new file mode 100644 index 0000000000000..8536c518ed7ac --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -0,0 +1,120 @@ +import React, { useState } from 'react'; +import { Link } from 'react-router-dom'; +import styled from 'styled-components/macro'; +import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated'; +import { EntityType } from '../../types.generated'; +import { IconStyleType } from '../entity/Entity'; +import { ANTD_GRAY } from '../entity/shared/constants'; +import { SearchBar } from '../search/SearchBar'; +import ClickOutside from '../shared/ClickOutside'; +import { useEntityRegistry } from '../useEntityRegistry'; +import DomainIcon from './DomainIcon'; + +const DomainSearchWrapper = styled.div` + position: relative; +`; + +const ResultsWrapper = styled.div` + background-color: white; + border-radius: 5px; + box-shadow: 0 3px 6px -4px rgb(0 0 0 / 12%), 0 6px 16px 0 rgb(0 0 0 / 8%), 0 9px 28px 8px rgb(0 0 0 / 5%); + max-height: 380px; + overflow: auto; + padding: 8px; + position: absolute; + max-height: 210px; + overflow: auto; + width: calc(100% - 24px); + left: 12px; + top: 45px; +`; + +const SearchResult = styled(Link)` + color: #262626; + display: inline-block; + height: 100%; + padding: 6px 8px; + width: 100%; + &:hover { + background-color: ${ANTD_GRAY[3]}; + color: #262626; + } +`; + +const IconWrapper = styled.span` + margin-right: 8px; +`; + +function DomainSearch() { + const [query, setQuery] = useState(''); + const [isSearchBarFocused, setIsSearchBarFocused] = useState(false); + const entityRegistry = useEntityRegistry(); + + const { data } = useGetSearchResultsForMultipleQuery({ + variables: { + input: { + types: [EntityType.Domain], // TODO: add sub-domain here? + query, + start: 0, + count: 50, + }, + }, + skip: !query, + }); + + const searchResults = data?.searchAcrossEntities?.searchResults; + + return ( + + setIsSearchBarFocused(false)}> + null} + onQueryChange={(q) => setQuery(q)} + entityRegistry={entityRegistry} + onFocus={() => setIsSearchBarFocused(true)} + /> + {isSearchBarFocused && searchResults && !!searchResults.length && ( + + {searchResults.map((result) => { + return ( + setIsSearchBarFocused(false)} + > + + {result.entity.type === EntityType.Domain ? ( + + ) : ( + entityRegistry.getIcon(result.entity.type, 12, IconStyleType.ACCENT) + )} + + {entityRegistry.getDisplayName(result.entity.type, result.entity)} + + ); + })} + + )} + + + ); +} + +export default DomainSearch; diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx index 4db2f516023e5..827031138dcdb 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx @@ -4,6 +4,7 @@ import { ProfileSidebarResizer } from '../../entity/shared/containers/profile/si import DomainsSidebarHeader from './DomainsSidebarHeader'; import { SidebarWrapper } from '../../shared/sidebar/components'; import DomainNavigator from './domainNavigator/DomainNavigator'; +import DomainSearch from '../DomainSearch'; export default function ManageDomainsSidebar() { const [browserWidth, setBrowserWith] = useState(window.innerWidth * 0.2); @@ -12,6 +13,7 @@ export default function ManageDomainsSidebar() { <> + void; + // Since we have domain data in the profile and in the sidebar browser, we need to communicate to the + // sidebar when to refetch for a given domain or sub-domain. This will happen when you edit a name, + // move a domain/sub-domain, create a new domain/sub-domain, and delete a domain/sub-domain + urnsToUpdate: string[]; + setUrnsToUpdate: (updatdUrns: string[]) => void; +} + +export const DomainEntityContext = React.createContext({ + isInDomainContext: false, + entityData: null, + setEntityData: () => {}, + urnsToUpdate: [], + setUrnsToUpdate: () => {}, +}); + +export const useDomainEntityData = () => { + const { isInDomainContext, entityData, setEntityData, urnsToUpdate, setUrnsToUpdate } = + useContext(DomainEntityContext); + return { isInDomainContext, entityData, setEntityData, urnsToUpdate, setUrnsToUpdate }; +}; From 7f71bb212774e69764ec842e836e66c5f4c074ed Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Wed, 30 Aug 2023 13:04:53 -0700 Subject: [PATCH 50/91] add search bar --- datahub-web-react/src/app/domain/DomainSearch.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx index 8536c518ed7ac..26a814807a718 100644 --- a/datahub-web-react/src/app/domain/DomainSearch.tsx +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -53,7 +53,7 @@ function DomainSearch() { const { data } = useGetSearchResultsForMultipleQuery({ variables: { input: { - types: [EntityType.Domain], // TODO: add sub-domain here? + types: [EntityType.Domain], query, start: 0, count: 50, @@ -69,7 +69,7 @@ function DomainSearch() { setIsSearchBarFocused(false)}> Date: Wed, 30 Aug 2023 13:17:20 -0700 Subject: [PATCH 51/91] clean up --- .../src/app/domain/DomainSearch.tsx | 9 +++---- .../app/entity/shared/DomainEntityContext.tsx | 27 ------------------- 2 files changed, 4 insertions(+), 32 deletions(-) delete mode 100644 datahub-web-react/src/app/entity/shared/DomainEntityContext.tsx diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx index 26a814807a718..4e3aa72c64db4 100644 --- a/datahub-web-react/src/app/domain/DomainSearch.tsx +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -96,11 +96,10 @@ function DomainSearch() { {result.entity.type === EntityType.Domain ? ( ) : ( entityRegistry.getIcon(result.entity.type, 12, IconStyleType.ACCENT) diff --git a/datahub-web-react/src/app/entity/shared/DomainEntityContext.tsx b/datahub-web-react/src/app/entity/shared/DomainEntityContext.tsx deleted file mode 100644 index b623517708dbc..0000000000000 --- a/datahub-web-react/src/app/entity/shared/DomainEntityContext.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import React, { useContext } from 'react'; -import { GenericEntityProperties } from './types'; - -export interface DomainEntityContextType { - isInDomainContext: boolean; - entityData: GenericEntityProperties | null; - setEntityData: (entityData: GenericEntityProperties | null) => void; - // Since we have domain data in the profile and in the sidebar browser, we need to communicate to the - // sidebar when to refetch for a given domain or sub-domain. This will happen when you edit a name, - // move a domain/sub-domain, create a new domain/sub-domain, and delete a domain/sub-domain - urnsToUpdate: string[]; - setUrnsToUpdate: (updatdUrns: string[]) => void; -} - -export const DomainEntityContext = React.createContext({ - isInDomainContext: false, - entityData: null, - setEntityData: () => {}, - urnsToUpdate: [], - setUrnsToUpdate: () => {}, -}); - -export const useDomainEntityData = () => { - const { isInDomainContext, entityData, setEntityData, urnsToUpdate, setUrnsToUpdate } = - useContext(DomainEntityContext); - return { isInDomainContext, entityData, setEntityData, urnsToUpdate, setUrnsToUpdate }; -}; From 1d932631e22468710e02391e02ead61784a0e4db Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 30 Aug 2023 17:18:38 -0700 Subject: [PATCH 52/91] feat(nested-domains): authorization by parent domain --- .../resolvers/mutate/util/DomainUtils.java | 20 ++----- .../authorization/ResolvedResourceSpec.java | 31 ---------- .../DomainFieldResolverProvider.java | 57 +++++++++++++++++-- 3 files changed, 57 insertions(+), 51 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index dc628ca70bc63..a07b942d1b65d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -39,7 +39,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.management.Query; import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; @@ -216,7 +215,7 @@ public static Entity getParentDomain( ) { try { final EntityResponse entityResponse = entityClient.getV2( - urn.getEntityType(), + DOMAIN_ENTITY_NAME, urn, Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), context.getAuthentication() @@ -225,24 +224,13 @@ public static Entity getParentDomain( if (entityResponse != null && entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { final DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); if (properties.hasParentDomain()) { - final Urn parentDomainUrn = properties.getParentDomain(); - if (parentDomainUrn != null) { - final EntityResponse parentResponse = entityClient.getV2( - parentDomainUrn.getEntityType(), - parentDomainUrn, - null, - context.getAuthentication() - ); - if (parentResponse != null) { - return UrnToEntityMapper.map(parentResponse.getUrn()); - } - } + return properties.getParentDomain() != null ? UrnToEntityMapper.map(properties.getParentDomain()) : null; } } - - return null; } catch (Exception e) { throw new RuntimeException(String.format("Failed to retrieve parent domain for entity %s", urn), e); } + + return null; } } \ No newline at end of file diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java index 0dae1bd386ccd..4c3a6b461553c 100644 --- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java +++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java @@ -26,21 +26,6 @@ public Set getFieldValues(ResourceFieldType resourceFieldType) { return fieldResolvers.get(resourceFieldType).getFieldValuesFuture().join().getValues(); } - /** - * Fetch the entity-registry type for a resource. ('dataset', 'dashboard', 'chart'). - * @return the entity type. - */ - public String getType() { - if (!fieldResolvers.containsKey(ResourceFieldType.RESOURCE_TYPE)) { - throw new UnsupportedOperationException( - "Failed to resolve resource type! No field resolver for RESOURCE_TYPE provided."); - } - Set resourceTypes = - fieldResolvers.get(ResourceFieldType.RESOURCE_TYPE).getFieldValuesFuture().join().getValues(); - assert resourceTypes.size() == 1; // There should always be a single resource type. - return resourceTypes.stream().findFirst().get(); - } - /** * Fetch the owners for a resource. * @return a set of owner urns, or empty set if none exist. @@ -51,20 +36,4 @@ public Set getOwners() { } return fieldResolvers.get(ResourceFieldType.OWNER).getFieldValuesFuture().join().getValues(); } - - /** - * Fetch the domain for a Resolved Resource Spec - * @return a Domain or null if one does not exist. - */ - @Nullable - public String getDomain() { - if (!fieldResolvers.containsKey(ResourceFieldType.DOMAIN)) { - return null; - } - Set domains = fieldResolvers.get(ResourceFieldType.DOMAIN).getFieldValuesFuture().join().getValues(); - if (domains.size() > 0) { - return domains.stream().findFirst().get(); - } - return null; - } } diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java index ae87812f3b79c..aa3e20801b549 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -6,15 +6,23 @@ import com.datahub.authorization.ResourceSpec; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.domain.DomainProperties; import com.linkedin.domain.Domains; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; + import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import javax.annotation.Nonnull; + import static com.linkedin.metadata.Constants.*; @@ -38,6 +46,38 @@ public FieldResolver getFieldResolver(ResourceSpec resourceSpec) { return FieldResolver.getResolverFromFunction(resourceSpec, this::getDomains); } + private Set getBatchedParentDomains(@Nonnull final Set urns) { + Set parentUrns = new HashSet<>(); + + try { + final Map batchResponse = _entityClient.batchGetV2( + DOMAIN_ENTITY_NAME, + urns, + Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), + _systemAuthentication + ); + + batchResponse.forEach((urn, entityResponse) -> { + if (entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { + final DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); + if (properties.hasParentDomain()) { + parentUrns.add(properties.getParentDomain()); + } + } + }); + + } catch (Exception e) { + log.error( + "Error while retrieving parent domains for {} urns including \"{}\"", + urns.size(), + urns.stream().findFirst().map(Urn::toString).orElse(""), + e + ); + } + + return parentUrns; + } + private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); // In the case that the entity is a domain, the associated domain is the domain itself @@ -59,9 +99,18 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { log.error("Error while retrieving domains aspect for urn {}", entityUrn, e); return FieldResolver.emptyFieldValue(); } - Domains domains = new Domains(domainsAspect.getValue().data()); - return FieldResolver.FieldValue.builder() - .values(domains.getDomains().stream().map(Object::toString).collect(Collectors.toSet())) - .build(); + + Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); + Set batchedParentUrns = getBatchedParentDomains(domainUrns); + + while (!batchedParentUrns.isEmpty()) { + domainUrns.addAll(batchedParentUrns); + batchedParentUrns = getBatchedParentDomains(batchedParentUrns); + } + + return FieldResolver.FieldValue.builder().values(domainUrns + .stream() + .map(Object::toString) + .collect(Collectors.toSet())).build(); } } From f9007f715987f56971a5acf61809f34abeb6721d Mon Sep 17 00:00:00 2001 From: Ellie O'Neil Date: Thu, 31 Aug 2023 12:03:03 -0700 Subject: [PATCH 53/91] Add debounce --- datahub-web-react/src/app/domain/DomainSearch.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx index 4e3aa72c64db4..a8222de76ba60 100644 --- a/datahub-web-react/src/app/domain/DomainSearch.tsx +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useRef, useState } from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components/macro'; import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated'; @@ -63,6 +63,13 @@ function DomainSearch() { }); const searchResults = data?.searchAcrossEntities?.searchResults; + const timerRef = useRef(-1); + const handleQueryChange = (q: string) => { + window.clearTimeout(timerRef.current); + timerRef.current = window.setTimeout(() => { + setQuery(q); + }, 250); + }; return ( @@ -81,7 +88,7 @@ function DomainSearch() { fontSize: 12, }} onSearch={() => null} - onQueryChange={(q) => setQuery(q)} + onQueryChange={(q) => handleQueryChange(q)} entityRegistry={entityRegistry} onFocus={() => setIsSearchBarFocused(true)} /> From 3c0742b9a97b46a9d1d42219c957247a071d8d04 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 31 Aug 2023 15:03:26 -0400 Subject: [PATCH 54/91] Display parent domain path in various places for domains --- .../src/app/entity/domain/preview/Preview.tsx | 1 + .../entity/glossaryNode/preview/Preview.tsx | 2 +- .../entity/glossaryTerm/preview/Preview.tsx | 2 +- .../PlatformContentContainer.tsx | 1 + .../PlatformContent/PlatformContentView.tsx | 13 ++--- .../src/app/preview/DefaultPreviewCard.tsx | 8 +-- .../autoComplete/AutoCompleteEntity.tsx | 7 ++- .../src/app/search/filters/FilterOption.tsx | 21 +++++--- .../{ParentNodes.tsx => ParentEntities.tsx} | 51 ++++++++++--------- .../src/app/search/filters/utils.tsx | 15 ++++++ datahub-web-react/src/graphql/domain.graphql | 27 ++-------- .../src/graphql/fragments.graphql | 32 ++++++++++++ datahub-web-react/src/graphql/search.graphql | 19 ++++--- 13 files changed, 124 insertions(+), 75 deletions(-) rename datahub-web-react/src/app/search/filters/{ParentNodes.tsx => ParentEntities.tsx} (57%) diff --git a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx index d1f768266f061..83198f6eba2d8 100644 --- a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx @@ -41,6 +41,7 @@ export const Preview = ({ owners={owners} insights={insights} logoComponent={logoComponent} + parentEntities={domain.parentDomains?.domains} snippet={} /> ); diff --git a/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx b/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx index 6c6ea163c6786..3938049059e4d 100644 --- a/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx @@ -27,7 +27,7 @@ export const Preview = ({ owners={owners} logoComponent={} type={entityRegistry.getEntityName(EntityType.GlossaryNode)} - parentNodes={parentNodes} + parentEntities={parentNodes?.nodes} /> ); }; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx b/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx index 26d3cf456ab7a..77f7476336dc7 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx @@ -37,7 +37,7 @@ export const Preview = ({ type="Glossary Term" typeIcon={entityRegistry.getIcon(EntityType.GlossaryTerm, 14, IconStyleType.ACCENT)} deprecation={deprecation} - parentNodes={parentNodes} + parentEntities={parentNodes?.nodes} domain={domain} /> ); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx index 5e87f093c3778..0eb223c04d439 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx @@ -50,6 +50,7 @@ function PlatformContentContainer() { parentContainers={entityData?.parentContainers?.containers} parentContainersRef={contentRef} areContainersTruncated={isContentTruncated} + parentEntities={entityData?.parentDomains?.domains} /> ); } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx index 5605bacc1d4e4..472069a3a6a16 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx @@ -2,15 +2,16 @@ import React from 'react'; import styled from 'styled-components'; import { Typography, Image } from 'antd'; import { Maybe } from 'graphql/jsutils/Maybe'; -import { Container, GlossaryNode } from '../../../../../../../types.generated'; +import { Container, Entity } from '../../../../../../../types.generated'; import { ANTD_GRAY } from '../../../../constants'; import ContainerLink from './ContainerLink'; -import ParentNodesView, { +import { StyledRightOutlined, ParentNodesWrapper as ParentContainersWrapper, Ellipsis, StyledTooltip, } from './ParentNodesView'; +import ParentEntities from '../../../../../../search/filters/ParentEntities'; const LogoIcon = styled.span` display: flex; @@ -74,14 +75,14 @@ interface Props { typeIcon?: JSX.Element; entityType?: string; parentContainers?: Maybe[] | null; - parentNodes?: GlossaryNode[] | null; + parentEntities?: Entity[] | null; parentContainersRef: React.RefObject; areContainersTruncated: boolean; } function PlatformContentView(props: Props) { const { - parentNodes, + parentEntities, platformName, platformLogoUrl, platformNames, @@ -102,7 +103,7 @@ function PlatformContentView(props: Props) { {typeIcon && {typeIcon}} {entityType} - {(!!platformName || !!instanceId || !!parentContainers?.length || !!parentNodes?.length) && ( + {(!!platformName || !!instanceId || !!parentContainers?.length || !!parentEntities?.length) && ( )} {platformName && ( @@ -145,7 +146,7 @@ function PlatformContentView(props: Props) { {directParentContainer && } - + ); } diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index 36713cfb7ffcf..4a5cbb3ed54f6 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -14,10 +14,10 @@ import { CorpUser, Deprecation, Domain, - ParentNodesResult, EntityPath, DataProduct, Health, + Entity, } from '../../types.generated'; import TagTermGroup from '../shared/tags/TagTermGroup'; import { ANTD_GRAY } from '../entity/shared/constants'; @@ -189,7 +189,7 @@ interface Props { // how the listed node is connected to the source node degree?: number; parentContainers?: ParentContainersResult | null; - parentNodes?: ParentNodesResult | null; + parentEntities?: Entity[] | null; previewType?: Maybe; paths?: EntityPath[]; health?: Health[]; @@ -228,7 +228,7 @@ export default function DefaultPreviewCard({ onClick, degree, parentContainers, - parentNodes, + parentEntities, platforms, logoUrls, previewType, @@ -277,7 +277,7 @@ export default function DefaultPreviewCard({ typeIcon={typeIcon} entityType={type} parentContainers={parentContainers?.containers} - parentNodes={parentNodes?.nodes} + parentEntities={parentEntities} parentContainersRef={contentRef} areContainersTruncated={isContentTruncated} /> diff --git a/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx b/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx index 60bb21713ba58..8292238732a80 100644 --- a/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx +++ b/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx @@ -10,6 +10,8 @@ import AutoCompleteEntityIcon from './AutoCompleteEntityIcon'; import { SuggestionText } from './styledComponents'; import AutoCompletePlatformNames from './AutoCompletePlatformNames'; import { getPlatformName } from '../../entity/shared/utils'; +import { getParentEntities } from '../filters/utils'; +import ParentEntities from '../filters/ParentEntities'; const AutoCompleteEntityWrapper = styled.div` display: flex; @@ -20,6 +22,7 @@ const AutoCompleteEntityWrapper = styled.div` const IconsContainer = styled.div` display: flex; + margin-bottom: 3px; `; const ContentWrapper = styled.div` @@ -75,11 +78,12 @@ export default function AutoCompleteEntity({ query, entity, siblings, hasParentT // Need to reverse parentContainers since it returns direct parent first. const orderedParentContainers = [...parentContainers].reverse(); const subtype = genericEntityProps?.subTypes?.typeNames?.[0]; + const parentEntities = getParentEntities(entity) || []; const showPlatforms = !!platforms.length; const showPlatformDivider = !!platforms.length && !!parentContainers.length; const showParentContainers = !!parentContainers.length; - const showHeader = showPlatforms || showParentContainers; + const showHeader = showPlatforms || showParentContainers || parentEntities.length > 0; return ( @@ -95,6 +99,7 @@ export default function AutoCompleteEntity({ query, entity, siblings, hasParentT {showPlatforms && } {showPlatformDivider && } {showParentContainers && } + )} ` @@ -102,6 +102,10 @@ const ArrowButton = styled(Button)<{ isOpen: boolean }>` `} `; +const ParentWrapper = styled.div` + max-width: 220px; +`; + interface Props { filterOption: FilterOptionType; selectedFilterOptions: FilterOptionType[]; @@ -124,8 +128,7 @@ export default function FilterOption({ const shouldShowIcon = field === PLATFORM_FILTER_NAME && icon !== null; const shouldShowTagColor = field === TAGS_FILTER_NAME && entity?.type === EntityType.Tag; const isSubTypeFilter = field === TYPE_NAMES_FILTER_NAME; - const isGlossaryTerm = entity?.type === EntityType.GlossaryTerm; - const parentNodes: GlossaryNode[] = isGlossaryTerm ? (entity as GlossaryTerm).parentNodes?.nodes || [] : []; + const parentEntities: Entity[] = getParentEntities(entity as Entity) || []; // only entity type filters return 10,000 max aggs const countText = count === MAX_COUNT_VAL && field === ENTITY_SUB_TYPE_FILTER_NAME ? '10k+' : formatNumber(count); @@ -143,7 +146,7 @@ export default function FilterOption({ return ( <> - 0} addPadding={addPadding}> + 0} addPadding={addPadding}> - {isGlossaryTerm && } + {parentEntities.length > 0 && ( + + + + )} {shouldShowIcon && <>{icon}} {shouldShowTagColor && ( diff --git a/datahub-web-react/src/app/search/filters/ParentNodes.tsx b/datahub-web-react/src/app/search/filters/ParentEntities.tsx similarity index 57% rename from datahub-web-react/src/app/search/filters/ParentNodes.tsx rename to datahub-web-react/src/app/search/filters/ParentEntities.tsx index 7012f07c16e64..f2b805749e0e9 100644 --- a/datahub-web-react/src/app/search/filters/ParentNodes.tsx +++ b/datahub-web-react/src/app/search/filters/ParentEntities.tsx @@ -2,19 +2,16 @@ import { FolderOpenOutlined } from '@ant-design/icons'; import { Tooltip, Typography } from 'antd'; import React from 'react'; import styled from 'styled-components'; -import { EntityType, GlossaryNode, GlossaryTerm } from '../../../types.generated'; +import { Entity } from '../../../types.generated'; import { ANTD_GRAY } from '../../entity/shared/constants'; import { useEntityRegistry } from '../../useEntityRegistry'; -const NUM_VISIBLE_NODES = 2; - const ParentNodesWrapper = styled.div` font-size: 12px; color: ${ANTD_GRAY[7]}; display: flex; align-items: center; margin-bottom: 3px; - max-width: 220px; overflow: hidden; `; @@ -27,54 +24,62 @@ export const ArrowWrapper = styled.span` margin: 0 3px; `; +const StyledTooltip = styled(Tooltip)` + display: flex; + white-space: nowrap; + overflow: hidden; +`; + +const DEFAULT_NUM_VISIBLE = 2; + interface Props { - glossaryTerm: GlossaryTerm; + parentEntities: Entity[]; + numVisible?: number; } -export default function ParentNodes({ glossaryTerm }: Props) { +export default function ParentEntities({ parentEntities, numVisible = DEFAULT_NUM_VISIBLE }: Props) { const entityRegistry = useEntityRegistry(); - const parentNodes: GlossaryNode[] = glossaryTerm.parentNodes?.nodes || []; - // parent nodes are returned with direct parent first - const orderedParentNodes = [...parentNodes].reverse(); - const visibleNodes = orderedParentNodes.slice(orderedParentNodes.length - NUM_VISIBLE_NODES); - const numHiddenNodes = orderedParentNodes.length - NUM_VISIBLE_NODES; - const includeNodePathTooltip = parentNodes.length > NUM_VISIBLE_NODES; + // parent nodes/domains are returned with direct parent first + const orderedParentEntities = [...parentEntities].reverse(); + const visibleNodes = orderedParentEntities.slice(orderedParentEntities.length - numVisible); + const numHiddenNodes = orderedParentEntities.length - numVisible; + const hasHiddenNodes = numHiddenNodes > 0; - if (!parentNodes.length) return null; + if (!parentEntities.length) return null; return ( - - {orderedParentNodes.map((glossaryNode, index) => ( + {orderedParentEntities.map((parentEntity, index) => ( <> - {entityRegistry.getDisplayName(EntityType.GlossaryNode, glossaryNode)} + {entityRegistry.getDisplayName(parentEntity.type, parentEntity)} - {index !== orderedParentNodes.length - 1 && {'>'}} + {index !== orderedParentEntities.length - 1 && {'>'}} ))} } > - {numHiddenNodes > 0 && + {hasHiddenNodes && [...Array(numHiddenNodes)].map(() => ( <> {'>'} ))} - {visibleNodes.map((glossaryNode, index) => { - const displayName = entityRegistry.getDisplayName(EntityType.GlossaryNode, glossaryNode); + {visibleNodes.map((parentEntity, index) => { + const displayName = entityRegistry.getDisplayName(parentEntity.type, parentEntity); return ( <> - + {displayName} {index !== visibleNodes.length - 1 && {'>'}} @@ -82,6 +87,6 @@ export default function ParentNodes({ glossaryTerm }: Props) { ); })} - + ); } diff --git a/datahub-web-react/src/app/search/filters/utils.tsx b/datahub-web-react/src/app/search/filters/utils.tsx index fbde71d6a2e9a..6ea9d0e8baa4f 100644 --- a/datahub-web-react/src/app/search/filters/utils.tsx +++ b/datahub-web-react/src/app/search/filters/utils.tsx @@ -14,10 +14,12 @@ import { AggregationMetadata, DataPlatform, DataPlatformInstance, + Domain, Entity, EntityType, FacetFilterInput, FacetMetadata, + GlossaryTerm, } from '../../../types.generated'; import { IconStyleType } from '../../entity/Entity'; import { @@ -331,3 +333,16 @@ export function canCreateViewFromFilters(activeFilters: FacetFilterInput[]) { } return true; } + +export function getParentEntities(entity: Entity): Entity[] | null { + if (!entity) { + return null; + } + if (entity.type === EntityType.GlossaryTerm) { + return (entity as GlossaryTerm).parentNodes?.nodes || []; + } + if (entity.type === EntityType.Domain) { + return (entity as Domain).parentDomains?.domains || []; + } + return null; +} diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 64d0ef9e9bf90..8a3b8ecde31ff 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -8,17 +8,7 @@ query getDomain($urn: String!) { description } parentDomains { - count - domains { - urn - type - ... on Domain { - properties { - name - description - } - } - } + ...parentDomainsFields } ownership { ...ownershipFields @@ -53,20 +43,13 @@ query listDomains($input: ListDomainsInput!) { name description } + parentDomains { + ...parentDomainsFields + } ownership { ...ownershipFields } - entities(input: { start: 0, count: 0 }) { - total - } - dataProducts: entities( - input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } - ) { - total - } - children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { - total - } + ...domainEntitiesFields } } } diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index c3ac2139e687b..72474911b9310 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -82,6 +82,20 @@ fragment parentNodesFields on ParentNodesResult { } } +fragment parentDomainsFields on ParentDomainsResult { + count + domains { + urn + type + ... on Domain { + properties { + name + description + } + } + } +} + fragment ownershipFields on Ownership { owners { owner { @@ -931,6 +945,20 @@ fragment parentContainerFields on Container { } } +fragment domainEntitiesFields on Domain { + entities(input: { start: 0, count: 0 }) { + total + } + dataProducts: entities( + input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } + ) { + total + } + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } +} + fragment entityDomain on DomainAssociation { domain { urn @@ -939,6 +967,10 @@ fragment entityDomain on DomainAssociation { name description } + parentDomains { + ...parentDomainsFields + } + ...domainEntitiesFields } associatedUrn } diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 060f9ac9c5843..9731c477f932b 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -145,6 +145,9 @@ fragment autoCompleteFields on Entity { properties { name } + parentDomains { + ...parentDomainsFields + } } ... on DataProduct { properties { @@ -632,17 +635,10 @@ fragment searchResultFields on Entity { ownership { ...ownershipFields } - entities(input: { start: 0, count: 0 }) { - total - } - dataProducts: entities( - input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } - ) { - total - } - children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { - total + parentDomains { + ...parentDomainsFields } + ...domainEntitiesFields } ... on Container { properties { @@ -797,6 +793,9 @@ fragment facetFields on FacetMetadata { properties { name } + parentDomains { + ...parentDomainsFields + } } ... on Container { platform { From e21a94c4c9242017d038e97f9a387ba3e40737c3 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 31 Aug 2023 12:20:10 -0700 Subject: [PATCH 55/91] chore(nested-domains): test for parent domain auth --- .../domain/DomainEntitiesResolver.java | 2 +- .../authorization/ResolvedResourceSpec.java | 1 - .../DomainFieldResolverProvider.java | 6 + .../authorization/DataHubAuthorizerTest.java | 108 +++++++++++++++--- 4 files changed, 99 insertions(+), 18 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 2a96a521eff58..0bf551c4683e6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -81,7 +81,7 @@ public CompletableFuture get(final DataFetchingEnvironment enviro return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities( SEARCHABLE_ENTITY_TYPES.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()), query, - new Filter().setOr(new ConjunctiveCriterionArray( new ConjunctiveCriterion().setAnd(criteria))), + new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(criteria))), start, count, null, diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java index 4c3a6b461553c..53dd0be44f963 100644 --- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java +++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java @@ -3,7 +3,6 @@ import java.util.Collections; import java.util.Map; import java.util.Set; -import javax.annotation.Nullable; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java index aa3e20801b549..06cd57706df26 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -79,6 +79,7 @@ private Set getBatchedParentDomains(@Nonnull final Set urns) { } private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { + System.out.println("DOMAIN LOG: getDomains"); Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); // In the case that the entity is a domain, the associated domain is the domain itself if (entityUrn.getEntityType().equals(DOMAIN_ENTITY_NAME)) { @@ -91,7 +92,9 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { try { EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn, Collections.singleton(DOMAINS_ASPECT_NAME), _systemAuthentication); + System.out.println("DOMAIN LOG: response"); if (response == null || !response.getAspects().containsKey(DOMAINS_ASPECT_NAME)) { + System.out.println("DOMAIN LOG: bail"); return FieldResolver.emptyFieldValue(); } domainsAspect = response.getAspects().get(DOMAINS_ASPECT_NAME); @@ -101,9 +104,12 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { } Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); + System.out.println("DOMAIN LOG: domainUrns=" + domainUrns.size()); Set batchedParentUrns = getBatchedParentDomains(domainUrns); + System.out.println("DOMAIN LOG: batched=" + batchedParentUrns.size()); while (!batchedParentUrns.isEmpty()) { + System.out.println("DOMAIN LOG: add stuff"); domainUrns.addAll(batchedParentUrns); batchedParentUrns = getBatchedParentDomains(batchedParentUrns); } diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java index 1ed794be15490..37567602083b5 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java @@ -13,6 +13,8 @@ import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; +import com.linkedin.domain.DomainProperties; +import com.linkedin.domain.Domains; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; @@ -25,16 +27,19 @@ import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; import com.linkedin.policy.DataHubResourceFilter; + import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.metadata.Constants.DATAHUB_POLICY_INFO_ASPECT_NAME; -import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; -import static com.linkedin.metadata.Constants.POLICY_ENTITY_NAME; +import javax.annotation.Nullable; + +import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.authorization.PoliciesConfig.ACTIVE_POLICY_STATE; import static com.linkedin.metadata.authorization.PoliciesConfig.INACTIVE_POLICY_STATE; import static com.linkedin.metadata.authorization.PoliciesConfig.METADATA_POLICY_TYPE; @@ -61,39 +66,79 @@ public void setupTest() throws Exception { // Init mocks. final Urn activePolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:0"); - final DataHubPolicyInfo activePolicy = createDataHubPolicyInfo(true, ImmutableList.of("EDIT_ENTITY_TAGS")); + final DataHubPolicyInfo activePolicy = createDataHubPolicyInfo(true, ImmutableList.of("EDIT_ENTITY_TAGS"), null); final EnvelopedAspectMap activeAspectMap = new EnvelopedAspectMap(); activeAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(activePolicy.data()))); final Urn inactivePolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:1"); - final DataHubPolicyInfo inactivePolicy = createDataHubPolicyInfo(false, ImmutableList.of("EDIT_ENTITY_OWNERS")); + final DataHubPolicyInfo inactivePolicy = createDataHubPolicyInfo(false, ImmutableList.of("EDIT_ENTITY_OWNERS"), null); final EnvelopedAspectMap inactiveAspectMap = new EnvelopedAspectMap(); inactiveAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(inactivePolicy.data()))); + final Urn domainPolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:2"); + final Urn parentDomainUrn = Urn.createFromString("urn:li:domain:parent"); + final DataHubPolicyInfo domainPolicy = createDataHubPolicyInfo(true, ImmutableList.of("EDIT_ENTITY_DOCS"), parentDomainUrn); + final EnvelopedAspectMap domainPolicyAspectMap = new EnvelopedAspectMap(); + domainPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(domainPolicy.data()))); + final SearchResult policySearchResult = new SearchResult(); - policySearchResult.setNumEntities(2); - policySearchResult.setEntities(new SearchEntityArray(ImmutableList.of(new SearchEntity().setEntity(activePolicyUrn), - new SearchEntity().setEntity(inactivePolicyUrn)))); + policySearchResult.setNumEntities(3); + policySearchResult.setEntities( + new SearchEntityArray( + ImmutableList.of( + new SearchEntity().setEntity(activePolicyUrn), + new SearchEntity().setEntity(inactivePolicyUrn), + new SearchEntity().setEntity(domainPolicyUrn) + ) + ) + ); when(_entityClient.search(eq("dataHubPolicy"), eq(""), isNull(), any(), anyInt(), anyInt(), any(), eq(new SearchFlags().setFulltext(true)))).thenReturn(policySearchResult); when(_entityClient.batchGetV2(eq(POLICY_ENTITY_NAME), - eq(ImmutableSet.of(activePolicyUrn, inactivePolicyUrn)), eq(null), any())).thenReturn( + eq(ImmutableSet.of(activePolicyUrn, inactivePolicyUrn, domainPolicyUrn)), eq(null), any())).thenReturn( ImmutableMap.of( activePolicyUrn, new EntityResponse().setUrn(activePolicyUrn).setAspects(activeAspectMap), - inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(inactiveAspectMap) + inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(inactiveAspectMap), + domainPolicyUrn, new EntityResponse().setUrn(domainPolicyUrn).setAspects(domainPolicyAspectMap) ) ); final List userUrns = ImmutableList.of(Urn.createFromString("urn:li:corpuser:user3"), Urn.createFromString("urn:li:corpuser:user4")); final List groupUrns = ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group3"), Urn.createFromString("urn:li:corpGroup:group4")); - EntityResponse entityResponse = new EntityResponse(); - EnvelopedAspectMap envelopedAspectMap = new EnvelopedAspectMap(); - envelopedAspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect() + EntityResponse ownershipResponse = new EntityResponse(); + EnvelopedAspectMap ownershipAspectMap = new EnvelopedAspectMap(); + ownershipAspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect() .setValue(new com.linkedin.entity.Aspect(createOwnershipAspect(userUrns, groupUrns).data()))); - entityResponse.setAspects(envelopedAspectMap); + ownershipResponse.setAspects(ownershipAspectMap); when(_entityClient.getV2(any(), any(), eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), any())) - .thenReturn(entityResponse); + .thenReturn(ownershipResponse); + + // Mocks to get domains on a resource + final Urn childDomainUrn = Urn.createFromString("urn:li:domain:child"); + final List domainUrns = ImmutableList.of(childDomainUrn); + final EntityResponse domainsResponse = new EntityResponse(); + EnvelopedAspectMap domainsAspectMap = new EnvelopedAspectMap(); + domainsAspectMap.put(DOMAINS_ASPECT_NAME, new EnvelopedAspect() + .setValue(new com.linkedin.entity.Aspect(createDomainsAspect(domainUrns).data()))); + domainsResponse.setAspects(domainsAspectMap); + when(_entityClient.getV2(any(), any(), eq(Collections.singleton(DOMAINS_ASPECT_NAME)), any())) + .thenReturn(domainsResponse); + + // Mocks to get parent domains on a domain + final Map parentDomainsBatchResponse = new HashMap<>(); + final EntityResponse parentDomainsResponse = new EntityResponse(); + EnvelopedAspectMap parentDomainsAspectMap = new EnvelopedAspectMap(); + parentDomainsAspectMap.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect() + .setValue(new com.linkedin.entity.Aspect(createDomainPropertiesAspect(parentDomainUrn).data()))); + parentDomainsResponse.setAspects(parentDomainsAspectMap); + parentDomainsBatchResponse.put(parentDomainUrn, parentDomainsResponse); + when(_entityClient.batchGetV2(eq(DOMAIN_ENTITY_NAME), eq(Collections.singleton(childDomainUrn)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any())) + .thenReturn(parentDomainsBatchResponse); + + // Mocks to reach the stopping point on domain parents + when(_entityClient.batchGetV2(eq(DOMAIN_ENTITY_NAME), eq(Collections.singleton(parentDomainUrn)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any())) + .thenReturn(new HashMap<>()); final Authentication systemAuthentication = new Authentication( new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID), @@ -229,7 +274,20 @@ public void testAuthorizedActorsActivePolicy() throws Exception { )); } - private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List privileges) throws Exception { + @Test + public void testGrantedPrivilegesInheritsParentDomain() { + ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test"); + + AuthorizationRequest request = new AuthorizationRequest( + "urn:li:corpuser:test", + "EDIT_ENTITY_DOCS", + Optional.of(resourceSpec) + ); + + assertEquals(_dataHubAuthorizer.authorize(request).getType(), AuthorizationResult.Type.ALLOW); + } + + private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List privileges, @Nullable final Urn domain) throws Exception { final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); dataHubPolicyInfo.setState(active ? ACTIVE_POLICY_STATE : INACTIVE_POLICY_STATE); @@ -252,7 +310,13 @@ private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List p final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); resourceFilter.setAllResources(true); resourceFilter.setType("dataset"); + + if (domain != null) { + resourceFilter.setFilter(FilterUtils.newFilter(ImmutableMap.of(ResourceFieldType.DOMAIN, Collections.singletonList(domain.toString())))); + } + dataHubPolicyInfo.setResources(resourceFilter); + return dataHubPolicyInfo; } @@ -284,6 +348,18 @@ private Ownership createOwnershipAspect(final List userOwners, final List domainUrns) { + final Domains domains = new Domains(); + domains.setDomains(new UrnArray(domainUrns)); + return domains; + } + + private DomainProperties createDomainPropertiesAspect(final Urn domainUrn) { + final DomainProperties properties = new DomainProperties(); + properties.setParentDomain(domainUrn); + return properties; + } + private AuthorizerContext createAuthorizerContext(final Authentication systemAuthentication, final EntityClient entityClient) { return new AuthorizerContext(Collections.emptyMap(), new DefaultResourceSpecResolver(systemAuthentication, entityClient)); } From f73018b410216dc17d580a85e6d4f87bc31fedf9 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 31 Aug 2023 12:20:45 -0700 Subject: [PATCH 56/91] rm logs --- .../fieldresolverprovider/DomainFieldResolverProvider.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java index 06cd57706df26..aa3e20801b549 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -79,7 +79,6 @@ private Set getBatchedParentDomains(@Nonnull final Set urns) { } private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { - System.out.println("DOMAIN LOG: getDomains"); Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); // In the case that the entity is a domain, the associated domain is the domain itself if (entityUrn.getEntityType().equals(DOMAIN_ENTITY_NAME)) { @@ -92,9 +91,7 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { try { EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn, Collections.singleton(DOMAINS_ASPECT_NAME), _systemAuthentication); - System.out.println("DOMAIN LOG: response"); if (response == null || !response.getAspects().containsKey(DOMAINS_ASPECT_NAME)) { - System.out.println("DOMAIN LOG: bail"); return FieldResolver.emptyFieldValue(); } domainsAspect = response.getAspects().get(DOMAINS_ASPECT_NAME); @@ -104,12 +101,9 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { } Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); - System.out.println("DOMAIN LOG: domainUrns=" + domainUrns.size()); Set batchedParentUrns = getBatchedParentDomains(domainUrns); - System.out.println("DOMAIN LOG: batched=" + batchedParentUrns.size()); while (!batchedParentUrns.isEmpty()) { - System.out.println("DOMAIN LOG: add stuff"); domainUrns.addAll(batchedParentUrns); batchedParentUrns = getBatchedParentDomains(batchedParentUrns); } From 16b573da197d5010e2bc63a7d42b0559e082efc1 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 31 Aug 2023 15:21:28 -0400 Subject: [PATCH 57/91] fix bug in ParentEntities --- .../src/app/search/filters/ParentEntities.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/datahub-web-react/src/app/search/filters/ParentEntities.tsx b/datahub-web-react/src/app/search/filters/ParentEntities.tsx index f2b805749e0e9..2504d5f0ff25a 100644 --- a/datahub-web-react/src/app/search/filters/ParentEntities.tsx +++ b/datahub-web-react/src/app/search/filters/ParentEntities.tsx @@ -42,15 +42,15 @@ export default function ParentEntities({ parentEntities, numVisible = DEFAULT_NU // parent nodes/domains are returned with direct parent first const orderedParentEntities = [...parentEntities].reverse(); - const visibleNodes = orderedParentEntities.slice(orderedParentEntities.length - numVisible); - const numHiddenNodes = orderedParentEntities.length - numVisible; - const hasHiddenNodes = numHiddenNodes > 0; + const numHiddenEntities = orderedParentEntities.length - numVisible; + const hasHiddenEntities = numHiddenEntities > 0; + const visibleNodes = hasHiddenEntities ? orderedParentEntities.slice(numHiddenEntities) : orderedParentEntities; if (!parentEntities.length) return null; return ( @@ -67,8 +67,8 @@ export default function ParentEntities({ parentEntities, numVisible = DEFAULT_NU } > - {hasHiddenNodes && - [...Array(numHiddenNodes)].map(() => ( + {hasHiddenEntities && + [...Array(numHiddenEntities)].map(() => ( <> {'>'} @@ -79,7 +79,7 @@ export default function ParentEntities({ parentEntities, numVisible = DEFAULT_NU return ( <> - + {displayName} {index !== visibleNodes.length - 1 && {'>'}} From 035e03a041a1ba8c79b9eb06682afb5b674779be Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 31 Aug 2023 16:18:46 -0400 Subject: [PATCH 58/91] Fix typo in nested domains context --- datahub-web-react/src/app/domain/DomainRoutes.tsx | 4 ++-- datahub-web-react/src/app/domain/DomainsContext.tsx | 8 ++++---- .../nestedDomains/domainNavigator/useHasDomainChildren.ts | 8 ++++---- .../entity/shared/EntityDropdown/useHandleDeleteDomain.ts | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainRoutes.tsx b/datahub-web-react/src/app/domain/DomainRoutes.tsx index ed912cacf8fdd..56811ddc48c0c 100644 --- a/datahub-web-react/src/app/domain/DomainRoutes.tsx +++ b/datahub-web-react/src/app/domain/DomainRoutes.tsx @@ -19,10 +19,10 @@ const ContentWrapper = styled.div` export default function DomainRoutes() { const entityRegistry = useEntityRegistry(); const [entityData, setEntityData] = useState(null); - const [parentDomainsToUpate, setParentDomainsToUpdate] = useState([]); + const [parentDomainsToUpdate, setParentDomainsToUpdate] = useState([]); return ( - + diff --git a/datahub-web-react/src/app/domain/DomainsContext.tsx b/datahub-web-react/src/app/domain/DomainsContext.tsx index 44a81d31a4d2c..ecbdaebd03817 100644 --- a/datahub-web-react/src/app/domain/DomainsContext.tsx +++ b/datahub-web-react/src/app/domain/DomainsContext.tsx @@ -4,18 +4,18 @@ import { GenericEntityProperties } from '../entity/shared/types'; export interface DomainsContextType { entityData: GenericEntityProperties | null; setEntityData: (entityData: GenericEntityProperties | null) => void; - parentDomainsToUpate: string[]; + parentDomainsToUpdate: string[]; setParentDomainsToUpdate: (values: string[]) => void; } export const DomainsContext = React.createContext({ entityData: null, setEntityData: () => {}, - parentDomainsToUpate: [], // used to tell domains to refetch their children count after updates (create, move, delete) + parentDomainsToUpdate: [], // used to tell domains to refetch their children count after updates (create, move, delete) setParentDomainsToUpdate: () => {}, }); export const useDomainsContext = () => { - const { entityData, setEntityData, parentDomainsToUpate, setParentDomainsToUpdate } = useContext(DomainsContext); - return { entityData, setEntityData, parentDomainsToUpate, setParentDomainsToUpdate }; + const { entityData, setEntityData, parentDomainsToUpdate, setParentDomainsToUpdate } = useContext(DomainsContext); + return { entityData, setEntityData, parentDomainsToUpdate, setParentDomainsToUpdate }; }; diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts index 1d6b83863b0ed..d16d5de23fbaf 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts @@ -8,22 +8,22 @@ interface Props { } export default function useHasDomainChildren({ domainUrn, numDomainChildren }: Props) { - const { parentDomainsToUpate, setParentDomainsToUpdate } = useDomainsContext(); + const { parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); const [getDomainChildrenCount, { data: childrenData }] = useGetDomainChildrenCountLazyQuery(); useEffect(() => { let timer; // fetch updated children count to determine if we show triangle toggle - if (parentDomainsToUpate.includes(domainUrn)) { + if (parentDomainsToUpdate.includes(domainUrn)) { timer = setTimeout(() => { getDomainChildrenCount({ variables: { urn: domainUrn } }); - setParentDomainsToUpdate(parentDomainsToUpate.filter((urn) => urn !== domainUrn)); + setParentDomainsToUpdate(parentDomainsToUpdate.filter((urn) => urn !== domainUrn)); }, 2000); } return () => { if (timer) window.clearTimeout(timer); }; - }, [domainUrn, getDomainChildrenCount, parentDomainsToUpate, setParentDomainsToUpdate]); + }, [domainUrn, getDomainChildrenCount, parentDomainsToUpdate, setParentDomainsToUpdate]); return childrenData ? !!childrenData.domain?.children?.total : !!numDomainChildren; } diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts index 0479ea0c5f5d1..ebbb8f9968a6a 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts @@ -10,14 +10,14 @@ interface DeleteDomainProps { export function useHandleDeleteDomain({ entityData, urn }: DeleteDomainProps) { const client = useApolloClient(); - const { parentDomainsToUpate, setParentDomainsToUpdate } = useDomainsContext(); + const { parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); const handleDeleteDomain = () => { if (entityData.parentDomains && entityData.parentDomains.domains.length > 0) { const parentDomainUrn = entityData.parentDomains.domains[0].urn; removeFromListDomainsCache(client, urn, 1, 1000, parentDomainUrn); - setParentDomainsToUpdate([...parentDomainsToUpate, parentDomainUrn]); + setParentDomainsToUpdate([...parentDomainsToUpdate, parentDomainUrn]); } else { removeFromListDomainsCache(client, urn, 1, 1000); } From d70dae65b10df74aaa7f82e8817ed008c140c102 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 31 Aug 2023 17:33:48 -0400 Subject: [PATCH 59/91] Update domains section on homepage --- .../renderer/component/DomainSearchList.tsx | 63 +++++++++++++------ .../renderer/component/HoverEntityTooltip.tsx | 6 +- .../src/app/shared/LogoCountCard.tsx | 26 ++------ .../src/app/shared/components.tsx | 20 ++++++ datahub-web-react/src/graphql/preview.graphql | 5 ++ 5 files changed, 77 insertions(+), 43 deletions(-) diff --git a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx index 37f26f9c4bdb7..f4dadf8d3c6e2 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx @@ -1,3 +1,4 @@ +import { ArrowRightOutlined } from '@ant-design/icons'; import React from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; @@ -5,6 +6,9 @@ import { Domain, EntityType, RecommendationContent } from '../../../../types.gen import { LogoCountCard } from '../../../shared/LogoCountCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import DomainIcon from '../../../domain/DomainIcon'; +import { PageRoutes } from '../../../../conf/Global'; +import { HomePageButton } from '../../../shared/components'; +import { HoverEntityTooltip } from './HoverEntityTooltip'; const DomainListContainer = styled.div` display: flex; @@ -13,6 +17,17 @@ const DomainListContainer = styled.div` flex-wrap: wrap; `; +const AllDomainsWrapper = styled.div` + color: ${(props) => props.theme.styles['primary-color']}; + font-size: 14px; +`; + +const AllDomainsText = styled.div` + margin-bottom: 8px; +`; + +const NUM_DOMAIN_CARDS = 9; + type Props = { content: Array; onClick?: (index: number) => void; @@ -23,7 +38,8 @@ export const DomainSearchList = ({ content, onClick }: Props) => { const domainsWithCounts: Array<{ domain: Domain; count?: number }> = content .map((cnt) => ({ domain: cnt.entity, count: cnt.params?.contentParams?.count })) - .filter((domainWithCount) => domainWithCount.domain !== null && domainWithCount !== undefined) as Array<{ + .filter((domainWithCount) => domainWithCount.domain !== null && domainWithCount !== undefined) + .slice(0, NUM_DOMAIN_CARDS) as Array<{ domain: Domain; count?: number; }>; @@ -31,25 +47,34 @@ export const DomainSearchList = ({ content, onClick }: Props) => { return ( {domainsWithCounts.map((domain, index) => ( - onClick?.(index)} - > - - } - count={domain.count} - /> - + + onClick?.(index)} + > + + } + count={domain.count} + /> + + ))} + + + + View All Domains + + + + ); }; diff --git a/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx b/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx index a39a39cd52db9..9ff0a1a2f940b 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx @@ -1,3 +1,4 @@ +import { TooltipPlacement } from 'antd/es/tooltip'; import { Tooltip } from 'antd'; import React from 'react'; import { Entity } from '../../../../types.generated'; @@ -9,9 +10,10 @@ type Props = { // whether the tooltip can be opened or if it should always stay closed canOpen?: boolean; children: React.ReactNode; + placement?: TooltipPlacement; }; -export const HoverEntityTooltip = ({ entity, canOpen = true, children }: Props) => { +export const HoverEntityTooltip = ({ entity, canOpen = true, children, placement }: Props) => { const entityRegistry = useEntityRegistry(); if (!entity || !entity.type || !entity.urn) { @@ -23,7 +25,7 @@ export const HoverEntityTooltip = ({ entity, canOpen = true, children }: Props) {entityRegistry.renderPreview(entity.type, PreviewType.HOVER_CARD, entity)}} diff --git a/datahub-web-react/src/app/shared/LogoCountCard.tsx b/datahub-web-react/src/app/shared/LogoCountCard.tsx index 3e2f74ebe5166..ebf0d9cd4f54e 100644 --- a/datahub-web-react/src/app/shared/LogoCountCard.tsx +++ b/datahub-web-react/src/app/shared/LogoCountCard.tsx @@ -1,27 +1,9 @@ import React from 'react'; -import { Image, Typography, Button } from 'antd'; +import { Image, Typography } from 'antd'; import styled from 'styled-components'; import { ANTD_GRAY } from '../entity/shared/constants'; import { formatNumber } from './formatNumber'; - -const Container = styled(Button)` - margin-right: 12px; - margin-left: 12px; - margin-bottom: 12px; - width: 160px; - height: 140px; - display: flex; - justify-content: center; - border-radius: 4px; - align-items: center; - flex-direction: column; - border: 1px solid ${ANTD_GRAY[4]}; - box-shadow: ${(props) => props.theme.styles['box-shadow']}; - &&:hover { - box-shadow: ${(props) => props.theme.styles['box-shadow-hover']}; - } - white-space: unset; -`; +import { HomePageButton } from './components'; const PlatformLogo = styled(Image)` max-height: 32px; @@ -53,7 +35,7 @@ type Props = { export const LogoCountCard = ({ logoUrl, logoComponent, name, count, onClick }: Props) => { return ( - + {(logoUrl && ) || logoComponent} @@ -68,6 +50,6 @@ export const LogoCountCard = ({ logoUrl, logoComponent, name, count, onClick }: {count !== undefined && {formatNumber(count)}} - + ); }; diff --git a/datahub-web-react/src/app/shared/components.tsx b/datahub-web-react/src/app/shared/components.tsx index 425fa0577da02..68d2fb52cfdba 100644 --- a/datahub-web-react/src/app/shared/components.tsx +++ b/datahub-web-react/src/app/shared/components.tsx @@ -1,5 +1,25 @@ import { Button } from 'antd'; import styled from 'styled-components'; +import { ANTD_GRAY } from '../entity/shared/constants'; + +export const HomePageButton = styled(Button)` + margin-right: 12px; + margin-left: 12px; + margin-bottom: 12px; + width: 160px; + height: 140px; + display: flex; + justify-content: center; + border-radius: 4px; + align-items: center; + flex-direction: column; + border: 1px solid ${ANTD_GRAY[4]}; + box-shadow: ${(props) => props.theme.styles['box-shadow']}; + &&:hover { + box-shadow: ${(props) => props.theme.styles['box-shadow-hover']}; + } + white-space: unset; +`; export const BaseButton = styled(Button)` &&& { diff --git a/datahub-web-react/src/graphql/preview.graphql b/datahub-web-react/src/graphql/preview.graphql index 03635ab1b66d5..e104d62c67074 100644 --- a/datahub-web-react/src/graphql/preview.graphql +++ b/datahub-web-react/src/graphql/preview.graphql @@ -304,7 +304,12 @@ fragment entityPreview on Entity { urn properties { name + description + } + parentDomains { + ...parentDomainsFields } + ...domainEntitiesFields } ... on Container { ...entityContainer From c585d7b4b98eaae977e08810a96cfb7c86c96a0f Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 31 Aug 2023 16:36:17 -0700 Subject: [PATCH 60/91] chore(nested-domains): more auth tests --- .../DomainFieldResolverProvider.java | 8 +- .../authorization/DataHubAuthorizerTest.java | 103 ++++++++++++------ 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java index aa3e20801b549..b2617f3a5462b 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -47,7 +47,7 @@ public FieldResolver getFieldResolver(ResourceSpec resourceSpec) { } private Set getBatchedParentDomains(@Nonnull final Set urns) { - Set parentUrns = new HashSet<>(); + final Set parentUrns = new HashSet<>(); try { final Map batchResponse = _entityClient.batchGetV2( @@ -79,7 +79,7 @@ private Set getBatchedParentDomains(@Nonnull final Set urns) { } private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { - Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); + final Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); // In the case that the entity is a domain, the associated domain is the domain itself if (entityUrn.getEntityType().equals(DOMAIN_ENTITY_NAME)) { return FieldResolver.FieldValue.builder() @@ -87,7 +87,7 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { .build(); } - EnvelopedAspect domainsAspect; + final EnvelopedAspect domainsAspect; try { EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn, Collections.singleton(DOMAINS_ASPECT_NAME), _systemAuthentication); @@ -100,7 +100,7 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { return FieldResolver.emptyFieldValue(); } - Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); + final Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); Set batchedParentUrns = getBatchedParentDomains(domainUrns); while (!batchedParentUrns.isEmpty()) { diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java index 37567602083b5..2e48123fb1813 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java @@ -12,6 +12,7 @@ import com.linkedin.common.OwnershipType; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.StringArray; import com.linkedin.domain.DomainProperties; import com.linkedin.domain.Domains; @@ -57,6 +58,9 @@ public class DataHubAuthorizerTest { public static final String DATAHUB_SYSTEM_CLIENT_ID = "__datahub_system"; + private static final Urn PARENT_DOMAIN_URN = UrnUtils.getUrn("urn:li:domain:parent"); + private static final Urn CHILD_DOMAIN_URN = UrnUtils.getUrn("urn:li:domain:child"); + private EntityClient _entityClient; private DataHubAuthorizer _dataHubAuthorizer; @@ -75,11 +79,15 @@ public void setupTest() throws Exception { final EnvelopedAspectMap inactiveAspectMap = new EnvelopedAspectMap(); inactiveAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(inactivePolicy.data()))); - final Urn domainPolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:2"); - final Urn parentDomainUrn = Urn.createFromString("urn:li:domain:parent"); - final DataHubPolicyInfo domainPolicy = createDataHubPolicyInfo(true, ImmutableList.of("EDIT_ENTITY_DOCS"), parentDomainUrn); - final EnvelopedAspectMap domainPolicyAspectMap = new EnvelopedAspectMap(); - domainPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(domainPolicy.data()))); + final Urn parentDomainPolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:2"); + final DataHubPolicyInfo parentDomainPolicy = createDataHubPolicyInfo(true, ImmutableList.of("EDIT_ENTITY_DOCS"), PARENT_DOMAIN_URN); + final EnvelopedAspectMap parentDomainPolicyAspectMap = new EnvelopedAspectMap(); + parentDomainPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(parentDomainPolicy.data()))); + + final Urn childDomainPolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:3"); + final DataHubPolicyInfo childDomainPolicy = createDataHubPolicyInfo(true, ImmutableList.of("EDIT_ENTITY_STATUS"), CHILD_DOMAIN_URN); + final EnvelopedAspectMap childDomainPolicyAspectMap = new EnvelopedAspectMap(); + childDomainPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(childDomainPolicy.data()))); final SearchResult policySearchResult = new SearchResult(); policySearchResult.setNumEntities(3); @@ -88,7 +96,8 @@ public void setupTest() throws Exception { ImmutableList.of( new SearchEntity().setEntity(activePolicyUrn), new SearchEntity().setEntity(inactivePolicyUrn), - new SearchEntity().setEntity(domainPolicyUrn) + new SearchEntity().setEntity(parentDomainPolicyUrn), + new SearchEntity().setEntity(childDomainPolicyUrn) ) ) ); @@ -96,11 +105,12 @@ public void setupTest() throws Exception { when(_entityClient.search(eq("dataHubPolicy"), eq(""), isNull(), any(), anyInt(), anyInt(), any(), eq(new SearchFlags().setFulltext(true)))).thenReturn(policySearchResult); when(_entityClient.batchGetV2(eq(POLICY_ENTITY_NAME), - eq(ImmutableSet.of(activePolicyUrn, inactivePolicyUrn, domainPolicyUrn)), eq(null), any())).thenReturn( + eq(ImmutableSet.of(activePolicyUrn, inactivePolicyUrn, parentDomainPolicyUrn, childDomainPolicyUrn)), eq(null), any())).thenReturn( ImmutableMap.of( activePolicyUrn, new EntityResponse().setUrn(activePolicyUrn).setAspects(activeAspectMap), inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(inactiveAspectMap), - domainPolicyUrn, new EntityResponse().setUrn(domainPolicyUrn).setAspects(domainPolicyAspectMap) + parentDomainPolicyUrn, new EntityResponse().setUrn(parentDomainPolicyUrn).setAspects(parentDomainPolicyAspectMap), + childDomainPolicyUrn, new EntityResponse().setUrn(childDomainPolicyUrn).setAspects(childDomainPolicyAspectMap) ) ); @@ -115,30 +125,16 @@ inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(ina .thenReturn(ownershipResponse); // Mocks to get domains on a resource - final Urn childDomainUrn = Urn.createFromString("urn:li:domain:child"); - final List domainUrns = ImmutableList.of(childDomainUrn); - final EntityResponse domainsResponse = new EntityResponse(); - EnvelopedAspectMap domainsAspectMap = new EnvelopedAspectMap(); - domainsAspectMap.put(DOMAINS_ASPECT_NAME, new EnvelopedAspect() - .setValue(new com.linkedin.entity.Aspect(createDomainsAspect(domainUrns).data()))); - domainsResponse.setAspects(domainsAspectMap); when(_entityClient.getV2(any(), any(), eq(Collections.singleton(DOMAINS_ASPECT_NAME)), any())) - .thenReturn(domainsResponse); + .thenReturn(createDomainsResponse(CHILD_DOMAIN_URN)); // Mocks to get parent domains on a domain - final Map parentDomainsBatchResponse = new HashMap<>(); - final EntityResponse parentDomainsResponse = new EntityResponse(); - EnvelopedAspectMap parentDomainsAspectMap = new EnvelopedAspectMap(); - parentDomainsAspectMap.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect() - .setValue(new com.linkedin.entity.Aspect(createDomainPropertiesAspect(parentDomainUrn).data()))); - parentDomainsResponse.setAspects(parentDomainsAspectMap); - parentDomainsBatchResponse.put(parentDomainUrn, parentDomainsResponse); - when(_entityClient.batchGetV2(eq(DOMAIN_ENTITY_NAME), eq(Collections.singleton(childDomainUrn)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any())) - .thenReturn(parentDomainsBatchResponse); + when(_entityClient.batchGetV2(any(), eq(Collections.singleton(CHILD_DOMAIN_URN)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any())) + .thenReturn(createDomainPropertiesBatchResponse(PARENT_DOMAIN_URN)); // Mocks to reach the stopping point on domain parents - when(_entityClient.batchGetV2(eq(DOMAIN_ENTITY_NAME), eq(Collections.singleton(parentDomainUrn)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any())) - .thenReturn(new HashMap<>()); + when(_entityClient.batchGetV2(any(), eq(Collections.singleton(PARENT_DOMAIN_URN)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any())) + .thenReturn(createDomainPropertiesBatchResponse(null)); final Authentication systemAuthentication = new Authentication( new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID), @@ -275,7 +271,20 @@ public void testAuthorizedActorsActivePolicy() throws Exception { } @Test - public void testGrantedPrivilegesInheritsParentDomain() { + public void testAuthorizationOnDomainWithPrivilegeIsAllowed() { + ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test"); + + AuthorizationRequest request = new AuthorizationRequest( + "urn:li:corpuser:test", + "EDIT_ENTITY_STATUS", + Optional.of(resourceSpec) + ); + + assertEquals(_dataHubAuthorizer.authorize(request).getType(), AuthorizationResult.Type.ALLOW); + } + + @Test + public void testAuthorizationOnDomainWithParentPrivilegeIsAllowed() { ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test"); AuthorizationRequest request = new AuthorizationRequest( @@ -287,6 +296,19 @@ public void testGrantedPrivilegesInheritsParentDomain() { assertEquals(_dataHubAuthorizer.authorize(request).getType(), AuthorizationResult.Type.ALLOW); } + @Test + public void testAuthorizationOnDomainWithoutPrivilegeIsDenied() { + ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test"); + + AuthorizationRequest request = new AuthorizationRequest( + "urn:li:corpuser:test", + "EDIT_ENTITY_DOC_LINKS", + Optional.of(resourceSpec) + ); + + assertEquals(_dataHubAuthorizer.authorize(request).getType(), AuthorizationResult.Type.DENY); + } + private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List privileges, @Nullable final Urn domain) throws Exception { final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); @@ -348,16 +370,31 @@ private Ownership createOwnershipAspect(final List userOwners, final List domainUrns) { + private EntityResponse createDomainsResponse(final Urn domainUrn) { + final List domainUrns = ImmutableList.of(domainUrn); + final EntityResponse domainsResponse = new EntityResponse(); + EnvelopedAspectMap domainsAspectMap = new EnvelopedAspectMap(); final Domains domains = new Domains(); domains.setDomains(new UrnArray(domainUrns)); - return domains; + domainsAspectMap.put(DOMAINS_ASPECT_NAME, new EnvelopedAspect() + .setValue(new com.linkedin.entity.Aspect(domains.data()))); + domainsResponse.setAspects(domainsAspectMap); + return domainsResponse; } - private DomainProperties createDomainPropertiesAspect(final Urn domainUrn) { + private Map createDomainPropertiesBatchResponse(@Nullable final Urn parentDomainUrn) { + final Map batchResponse = new HashMap<>(); + final EntityResponse response = new EntityResponse(); + EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); final DomainProperties properties = new DomainProperties(); - properties.setParentDomain(domainUrn); - return properties; + if (parentDomainUrn != null) { + properties.setParentDomain(parentDomainUrn); + } + aspectMap.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect() + .setValue(new com.linkedin.entity.Aspect(properties.data()))); + response.setAspects(aspectMap); + batchResponse.put(parentDomainUrn, response); + return batchResponse; } private AuthorizerContext createAuthorizerContext(final Authentication systemAuthentication, final EntityClient entityClient) { From b4af40d3fbf766780c57287a43e0b4e118e51326 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 31 Aug 2023 16:51:08 -0700 Subject: [PATCH 61/91] feat(nested-domains): adjust copy on policy builder --- .../src/app/permissions/policy/PolicyPrivilegeForm.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index c57273c2ea3d9..28c5b38c7d13e 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -344,7 +344,8 @@ export default function PolicyPrivilegeForm({ {showResourceFilterInput && ( Domain}> - Search for domains the policy should apply to. If none is selected, policy is applied to{' '} + Search for domains the policy should apply to. If selected, policy applies to the domain + and all domains nested below it. If none is selected, policy is applied to{' '} all resources in all domains. Date: Fri, 1 Sep 2023 09:08:44 -0700 Subject: [PATCH 65/91] more copy --- .../src/app/permissions/policy/PolicyPrivilegeForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index d51533ce3df03..6f6365b8f9bb2 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -342,7 +342,7 @@ export default function PolicyPrivilegeForm({ )} {showResourceFilterInput && ( - Select Domains to Apply Policy}> + Select Domains}> The policy will apply to any chosen domains and all their nested subdomains. If no domains are selected, the policy will be applied globally to all entities in all domains. From 9f6af1bebddbf510ce49959c415f70cf327edb5e Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 1 Sep 2023 09:12:19 -0700 Subject: [PATCH 66/91] adjust --- .../src/app/permissions/policy/PolicyPrivilegeForm.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index 6f6365b8f9bb2..1f2b29f571214 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -344,8 +344,8 @@ export default function PolicyPrivilegeForm({ {showResourceFilterInput && ( Select Domains}> - The policy will apply to any chosen domains and all their nested subdomains. If no domains are - selected, the policy will be applied globally to all entities in all domains. + The policy will apply to any chosen domains and all their nested domains. If none are + selected, the policy is applied to all resources of in all domains. onSelectDomain(domainUrn)} - onDeselect={onDeselectDomain} - onSearch={(value: string) => { - // eslint-disable-next-line react/prop-types - handleSearch(value.trim()); - // eslint-disable-next-line react/prop-types - setInputValue(value.trim()); - }} - ref={inputEl} - value={selectValue} - tagRender={tagRender} - onBlur={handleBlur} - > - {domainSearchOptions} - + + + + + + From 90d7be21dd716399c3c3ab8f2aa6f96f3d83d3a4 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 5 Sep 2023 14:28:16 -0700 Subject: [PATCH 68/91] feat(nested-domains): move domain resolver --- .../resolvers/mutate/MoveDomainResolver.java | 83 +++++++++++++++++++ .../src/main/resources/entity.graphql | 15 ++++ datahub-web-react/src/graphql/domain.graphql | 4 + 3 files changed, 102 insertions(+) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java new file mode 100644 index 0000000000000..d0bebd05c0c5e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java @@ -0,0 +1,83 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; +import com.linkedin.datahub.graphql.generated.MoveDomainInput; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.CompletableFuture; + +@Slf4j +@RequiredArgsConstructor +public class MoveDomainResolver implements DataFetcher> { + + private final EntityService _entityService; + private final EntityClient _entityClient; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final MoveDomainInput input = ResolverUtils.bindArgument(environment.getArgument("input"), MoveDomainInput.class); + final QueryContext context = environment.getContext(); + final Urn resourceUrn = UrnUtils.getUrn(input.getResourceUrn()); + final Urn newParentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; + + return CompletableFuture.supplyAsync(() -> { + if (!AuthorizationUtils.canCreateDomains(context)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + try { + if (!resourceUrn.getEntityType().equals(Constants.DOMAIN_ENTITY_NAME)) { + throw new IllegalArgumentException("Resource is not a domain."); + } + + DomainProperties properties = (DomainProperties) EntityUtils.getAspectFromEntity(resourceUrn.toString(), Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, null); + + if (properties == null) { + throw new IllegalArgumentException("Domain properties do not exist."); + } + + if (newParentDomainUrn != null) { + if (!newParentDomainUrn.getEntityType().equals(Constants.DOMAIN_ENTITY_NAME)) { + throw new IllegalArgumentException("Parent entity is not a domain."); + } + if (!_entityService.exists(newParentDomainUrn)) { + throw new IllegalArgumentException("Parent entity does not exist."); + } + } + + if (DomainUtils.hasNameConflict(properties.getName(), newParentDomainUrn, context, _entityClient)) { + throw new DataHubGraphQLException( + String.format("\"%s\" already exists in the destination domain. Please pick a unique name.", properties.getName()), + DataHubGraphQLErrorCode.CONFLICT + ); + } + + properties.setParentDomain(newParentDomainUrn, SetMode.REMOVE_IF_NULL); + Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); + MutationUtils.persistAspect(resourceUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, properties, actor, _entityService); + return true; + } catch (Exception e) { + log.error("Failed to move domain {} to parent {} : {}", input.getResourceUrn(), input.getParentDomain(), e.getMessage()); + throw new RuntimeException(String.format("Failed to move domain %s to %s", input.getResourceUrn(), input.getParentDomain()), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index b007e2506a09f..bfcc185e18348 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -7731,6 +7731,21 @@ input UpdateParentNodeInput { resourceUrn: String! } +""" +Input for updating the parent domain of a domain. +""" +input MoveDomainInput { + """ + The new parent domain urn. If parentDomain is null, this will remove the parent from this entity + """ + parentDomain: String + + """ + The primary key of the resource to update the parent domain for + """ + resourceUrn: String! +} + """ Input for updating the name of an entity """ diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 7ffe217ca0fed..149a70f676511 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -71,6 +71,10 @@ mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) } +mutation moveDomain($input: UpdateParentDomainInput!) { + moveDomain(input: $input) +} + mutation deleteDomain($urn: String!) { deleteDomain(urn: $urn) } From a04f9a4c5941970fc28b29d0edb705981403ca4e Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 5 Sep 2023 17:34:24 -0400 Subject: [PATCH 69/91] use navigator in polict builder --- .../domainNavigator/DomainNavigator.tsx | 2 +- .../domainNavigator/DomainNode.tsx | 4 +- .../EntityDropdown/DomainParentSelect.tsx | 8 +- .../profile/sidebar/Domain/SetDomainModal.tsx | 8 +- .../policy/PolicyPrivilegeForm.tsx | 82 +++++++++++++------ .../src/app/shared/tags/AddTagsTermsModal.tsx | 6 +- 6 files changed, 72 insertions(+), 38 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx index eb6ab11cf8c68..cd4c3a1fce1dc 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -13,7 +13,7 @@ const NavigatorWrapper = styled.div` `; interface Props { - selectDomainOverride?: (urn: string, displayName: string) => void; + selectDomainOverride?: (domain: Domain) => void; } export default function DomainNavigator({ selectDomainOverride }: Props) { diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index a45713b25a101..3603b255c0f95 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -63,7 +63,7 @@ const StyledExpander = styled(BodyGridExpander)` interface Props { domain: Domain; numDomainChildren: number; - selectDomainOverride?: (urn: string, displayName: string) => void; + selectDomainOverride?: (domain: Domain) => void; } export default function DomainNode({ domain, numDomainChildren, selectDomainOverride }: Props) { @@ -91,7 +91,7 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver function handleSelectDomain() { if (selectDomainOverride) { - selectDomainOverride(domain.urn, displayName); + selectDomainOverride(domain); } else { history.push(entityRegistry.getEntityUrl(domain.type, domain.urn)); } diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index bcd93ddae6532..3963326249a46 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { Select } from 'antd'; -import { EntityType } from '../../../../types.generated'; +import { Domain, EntityType } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; import ClickOutside from '../../../shared/ClickOutside'; import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; @@ -35,6 +35,10 @@ export default function DomainParentSelect(props: Props) { setSelectedParentUrn, }); + function selectDomain(domain: Domain) { + selectParentFromBrowser(domain.urn, entityRegistry.getDisplayName(EntityType.Domain, domain)); + } + const isShowingDomainNavigator = !searchQuery && isFocusedOnInput; return ( @@ -57,7 +61,7 @@ export default function DomainParentSelect(props: Props) { ))} - + ); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx index 684312ffcff95..405442e8d7f50 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx @@ -2,7 +2,7 @@ import React, { useRef, useState } from 'react'; import { Button, Form, message, Modal, Select } from 'antd'; import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/search.generated'; -import { Entity, EntityType } from '../../../../../../../types.generated'; +import { Domain, Entity, EntityType } from '../../../../../../../types.generated'; import { useBatchSetDomainMutation } from '../../../../../../../graphql/mutations.generated'; import { useEntityRegistry } from '../../../../../../useEntityRegistry'; import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener'; @@ -98,12 +98,12 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk } }; - function selectDomainFromBrowser(urn: string, displayName: string) { + function selectDomainFromBrowser(domain: Domain) { setIsFocusedOnInput(false); setSelectedDomain({ - displayName, + displayName: entityRegistry.getDisplayName(EntityType.Domain, domain), type: EntityType.Domain, - urn, + urn: domain.urn, }); } diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index c57273c2ea3d9..222a33e0ee39d 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import { Link } from 'react-router-dom'; import { Form, Select, Tag, Tooltip, Typography } from 'antd'; import styled from 'styled-components/macro'; @@ -9,7 +9,7 @@ import { useGetSearchResultsForMultipleLazyQuery, useGetSearchResultsLazyQuery, } from '../../../graphql/search.generated'; -import { ResourceFilter, PolicyType, EntityType } from '../../../types.generated'; +import { ResourceFilter, PolicyType, EntityType, Domain } from '../../../types.generated'; import { convertLegacyResourceFilter, createCriterionValue, @@ -21,6 +21,9 @@ import { mapResourceTypeToPrivileges, setFieldValues, } from './policyUtils'; +import DomainNavigator from '../../domain/nestedDomains/domainNavigator/DomainNavigator'; +import { BrowserWrapper } from '../../shared/tags/AddTagsTermsModal'; +import ClickOutside from '../../shared/ClickOutside'; type Props = { policyType: PolicyType; @@ -55,6 +58,8 @@ export default function PolicyPrivilegeForm({ setPrivileges, }: Props) { const entityRegistry = useEntityRegistry(); + const [domainInputValue, setDomainInputValue] = useState(''); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); // Configuration used for displaying options const { @@ -98,6 +103,7 @@ export default function PolicyPrivilegeForm({ const resourceSelectValue = resourceEntities.map((criterionValue) => criterionValue.value); const domainSelectValue = getFieldValues(resources.filter, 'DOMAIN').map((criterionValue) => criterionValue.value); const privilegesSelectValue = privileges; + const isShowingDomainNavigator = !domainInputValue && isFocusedOnInput; // Construct privilege options for dropdown const platformPrivileges = policiesConfig?.platformPrivileges || []; @@ -193,13 +199,14 @@ export default function PolicyPrivilegeForm({ }; // When a domain is selected, add its urn to the list of domains - const onSelectDomain = (domain) => { + const onSelectDomain = (domainUrn, domainObj?: Domain) => { const filter = resources.filter || { criteria: [], }; + const domainEntity = domainObj || getEntityFromSearchResults(domainSearchResults, domainUrn); const updatedFilter = setFieldValues(filter, 'DOMAIN', [ ...domains, - createCriterionValueWithEntity(domain, getEntityFromSearchResults(domainSearchResults, domain) || null), + createCriterionValueWithEntity(domainUrn, domainEntity || null), ]); setResources({ ...resources, @@ -207,6 +214,11 @@ export default function PolicyPrivilegeForm({ }); }; + function selectDomainFromBrowser(domain: Domain) { + onSelectDomain(domain.urn, domain); + setIsFocusedOnInput(false); + } + // When a domain is deselected, remove its urn from the list of domains const onDeselectDomain = (domain) => { const filter = resources.filter || { @@ -243,6 +255,7 @@ export default function PolicyPrivilegeForm({ // Handle domain search, if the domain type has an associated EntityType mapping. const handleDomainSearch = (text: string) => { const trimmedText: string = text.trim(); + setDomainInputValue(trimmedText); searchDomains({ variables: { input: { @@ -276,6 +289,15 @@ export default function PolicyPrivilegeForm({ : displayStr; }; + function handleCLickOutside() { + // delay closing the domain navigator so we don't get a UI "flash" between showing search results and navigator + setTimeout(() => setIsFocusedOnInput(false), 0); + } + + function handleBlur() { + setDomainInputValue(''); + } + return ( {showResourceFilterInput && ( @@ -347,28 +369,36 @@ export default function PolicyPrivilegeForm({ Search for domains the policy should apply to. If none is selected, policy is applied to{' '} all resources in all domains. - + + + + + + )} Privileges}> diff --git a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx index 01e11ceb9a738..80d239def391c 100644 --- a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx +++ b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx @@ -50,15 +50,15 @@ const StyleTag = styled(CustomTag)` line-height: 16px; `; -export const BrowserWrapper = styled.div<{ isHidden: boolean }>` +export const BrowserWrapper = styled.div<{ isHidden: boolean; width?: string; maxHeight?: number }>` background-color: white; border-radius: 5px; box-shadow: 0 3px 6px -4px rgb(0 0 0 / 12%), 0 6px 16px 0 rgb(0 0 0 / 8%), 0 9px 28px 8px rgb(0 0 0 / 5%); - max-height: 380px; + max-height: ${(props) => (props.maxHeight ? props.maxHeight : '380')}px; overflow: auto; position: absolute; transition: opacity 0.2s; - width: 480px; + width: ${(props) => (props.width ? props.width : '480px')}; z-index: 1051; ${(props) => props.isHidden && From 6bb4e6bd1c29e7c9426bc26eb1bd1ad993ed0350 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 5 Sep 2023 15:06:25 -0700 Subject: [PATCH 70/91] test updates --- .../resolvers/mutate/MoveDomainResolver.java | 6 +- .../domain/MoveDomainResolverTest.java | 135 ++++++++++++++++++ .../domain/ParentDomainsResolverTest.java | 22 +-- 3 files changed, 141 insertions(+), 22 deletions(-) create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java index d0bebd05c0c5e..a40365cf20912 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java @@ -48,7 +48,11 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw throw new IllegalArgumentException("Resource is not a domain."); } - DomainProperties properties = (DomainProperties) EntityUtils.getAspectFromEntity(resourceUrn.toString(), Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, null); + DomainProperties properties = (DomainProperties) EntityUtils.getAspectFromEntity( + resourceUrn.toString(), + Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, + null + ); if (properties == null) { throw new IllegalArgumentException("Domain properties do not exist."); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java new file mode 100644 index 0000000000000..0af8768401504 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java @@ -0,0 +1,135 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.MoveDomainInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MoveDomainResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.util.concurrent.CompletionException; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +public class MoveDomainResolverTest { + + private static final String CONTAINER_URN = "urn:li:container:00005397daf94708a8822b8106cfd451"; + private static final String PARENT_DOMAIN_URN = "urn:li:domain:00005397daf94708a8822b8106cfd451"; + private static final String DOMAIN_URN = "urn:li:domain:11115397daf94708a8822b8106cfd451"; + private static final MoveDomainInput INPUT = new MoveDomainInput(PARENT_DOMAIN_URN, DOMAIN_URN); + private static final MoveDomainInput INVALID_INPUT = new MoveDomainInput(CONTAINER_URN, DOMAIN_URN); + private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); + + private MetadataChangeProposal setupTests(DataFetchingEnvironment mockEnv, EntityService mockService, EntityClient mockClient) throws Exception { + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + final String name = "test name"; + Mockito.when(mockService.getAspect( + Urn.createFromString(DOMAIN_URN), + Constants.DOMAIN_PROPERTIES_ASPECT_NAME, + 0)) + .thenReturn(new DomainProperties().setName(name)); + + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(name, Urn.createFromString(PARENT_DOMAIN_URN))), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + + DomainProperties properties = new DomainProperties(); + properties.setName(name); + properties.setParentDomain(Urn.createFromString(PARENT_DOMAIN_URN)); + return MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(DOMAIN_URN), + DOMAIN_PROPERTIES_ASPECT_NAME, properties); + } + + @Test + public void testGetSuccess() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + final MetadataChangeProposal proposal = setupTests(mockEnv, mockService, mockClient); + + assertTrue(resolver.get(mockEnv).get()); + verifyIngestProposal(mockService, 1, proposal); + } + + @Test + public void testGetFailureEntityDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT); + + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockService.getAspect( + Urn.createFromString(DOMAIN_URN), + DOMAIN_PROPERTIES_ASPECT_NAME, + 0)) + .thenReturn(null); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + verifyNoIngestProposal(mockService); + } + + @Test + public void testGetFailureParentDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(false); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + setupTests(mockEnv, mockService, mockClient); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + verifyNoIngestProposal(mockService); + } + + @Test + public void testGetFailureParentIsNotDomain() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INVALID_INPUT); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + setupTests(mockEnv, mockService, mockClient); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + verifyNoIngestProposal(mockService); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java index 544614cf81121..8b2083745dcd4 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java @@ -67,16 +67,6 @@ public void testGetSuccessForDomain() throws Exception { Mockito.any(Authentication.class) )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(domainAspects))); - Mockito.when(mockClient.getV2( - Mockito.eq(parentDomain1.getParentDomain().getEntityType()), - Mockito.eq(parentDomain1.getParentDomain()), - Mockito.eq(null), - Mockito.any(Authentication.class) - )).thenReturn(new EntityResponse() - .setEntityName(DOMAIN_ENTITY_NAME) - .setUrn(parentDomain1.getParentDomain()) - .setAspects(new EnvelopedAspectMap(parentDomain1Aspects))); - Mockito.when(mockClient.getV2( Mockito.eq(parentDomain1.getParentDomain().getEntityType()), Mockito.eq(parentDomain1.getParentDomain()), @@ -84,16 +74,6 @@ public void testGetSuccessForDomain() throws Exception { Mockito.any(Authentication.class) )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentDomain1Aspects))); - Mockito.when(mockClient.getV2( - Mockito.eq(parentDomain2.getParentDomain().getEntityType()), - Mockito.eq(parentDomain2.getParentDomain()), - Mockito.eq(null), - Mockito.any(Authentication.class) - )).thenReturn(new EntityResponse() - .setEntityName(DOMAIN_ENTITY_NAME) - .setUrn(parentDomain2.getParentDomain()) - .setAspects(new EnvelopedAspectMap(parentDomain2Aspects))); - Mockito.when(mockClient.getV2( Mockito.eq(parentDomain2.getParentDomain().getEntityType()), Mockito.eq(parentDomain2.getParentDomain()), @@ -104,7 +84,7 @@ public void testGetSuccessForDomain() throws Exception { ParentDomainsResolver resolver = new ParentDomainsResolver(mockClient); ParentDomainsResult result = resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(5)).getV2( + Mockito.verify(mockClient, Mockito.times(3)).getV2( Mockito.any(), Mockito.any(), Mockito.any(), From aaa745ac1032eabe1c9fe0a3a0a330afc05a6e1f Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 5 Sep 2023 15:59:21 -0700 Subject: [PATCH 71/91] fix(nested-domains): do not overlap button icon --- datahub-web-react/src/app/domain/DomainSearch.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx index a8222de76ba60..49f3ad2e862aa 100644 --- a/datahub-web-react/src/app/domain/DomainSearch.tsx +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -27,6 +27,7 @@ const ResultsWrapper = styled.div` width: calc(100% - 24px); left: 12px; top: 45px; + z-index: 1; `; const SearchResult = styled(Link)` From 3c04bfdbb63e1387fb8b8ba1e1cab222f3475842 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 5 Sep 2023 16:03:47 -0700 Subject: [PATCH 72/91] remove an unused graphql definition that wasn't compiling --- datahub-web-react/src/graphql/domain.graphql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 149a70f676511..7ffe217ca0fed 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -71,10 +71,6 @@ mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) } -mutation moveDomain($input: UpdateParentDomainInput!) { - moveDomain(input: $input) -} - mutation deleteDomain($urn: String!) { deleteDomain(urn: $urn) } From cdd1bc86021cd88e974e5e1fc40fde04141f542a Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Tue, 5 Sep 2023 16:52:27 -0700 Subject: [PATCH 73/91] feat(nested-domains): move parent --- .../resolvers/mutate/MoveDomainResolver.java | 2 +- .../src/app/entity/domain/DomainEntity.tsx | 2 +- .../shared/EntityDropdown/EntityDropdown.tsx | 29 +++++------ .../app/entity/shared/EntityDropdown/utils.ts | 50 +++++++++++++++++-- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java index a40365cf20912..d99f88e2707c3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java @@ -39,7 +39,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw final Urn newParentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; return CompletableFuture.supplyAsync(() -> { - if (!AuthorizationUtils.canCreateDomains(context)) { + if (!AuthorizationUtils.canManageDomains(context)) { throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index 56bc16f0db134..68c06935dbbe5 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -72,7 +72,7 @@ export class DomainEntity implements Entity { useEntityQuery={useGetDomainQuery} useUpdateQuery={undefined} getOverrideProperties={this.getOverridePropertiesFromEntity} - headerDropdownItems={new Set([EntityMenuItems.DELETE])} + headerDropdownItems={new Set([EntityMenuItems.MOVE, EntityMenuItems.DELETE])} headerActionItems={new Set([EntityActionItem.BATCH_ADD_DOMAIN])} isNameEditable tabs={[ diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index 4e78e38677b0a..adbd7d6d369d5 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -20,7 +20,8 @@ import { ANTD_GRAY } from '../constants'; import { useEntityRegistry } from '../../../useEntityRegistry'; import useDeleteEntity from './useDeleteEntity'; import { getEntityProfileDeleteRedirectPath } from '../../../shared/deleteUtils'; -import { isDeleteDisabled } from './utils'; +import { shouldDisplayChildDeletionWarning, isDeleteDisabled, isMoveDisabled } from './utils'; +import { useUserContext } from '../../../context/useUserContext'; export enum EntityMenuItems { COPY_URL, @@ -89,6 +90,7 @@ function EntityDropdown(props: Props) { options, } = props; + const me = useUserContext(); const entityRegistry = useEntityRegistry(); const [updateDeprecation] = useUpdateDeprecationMutation(); const { onDeleteEntity, hasBeenDeleted } = useDeleteEntity( @@ -132,8 +134,6 @@ function EntityDropdown(props: Props) { const pageUrl = window.location.href; const isGlossaryEntity = entityType === EntityType.GlossaryNode || entityType === EntityType.GlossaryTerm; const isDomainEntity = entityType === EntityType.Domain; - const entityHasChildren = !!entityData?.children?.total; - const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; const canCreateGlossaryEntity = !!entityData?.privileges?.canManageChildren; /** @@ -196,7 +196,7 @@ function EntityDropdown(props: Props) { {menuItems.has(EntityMenuItems.MOVE) && ( setIsMoveModalVisible(true)} > @@ -207,18 +207,16 @@ function EntityDropdown(props: Props) { {menuItems.has(EntityMenuItems.DELETE) && ( @@ -254,7 +252,10 @@ function EntityDropdown(props: Props) { refetch={refetchForEntity} /> )} - {isMoveModalVisible && setIsMoveModalVisible(false)} />} + {isMoveModalVisible && isGlossaryEntity && ( + setIsMoveModalVisible(false)} /> + )} + {isMoveModalVisible && isDomainEntity &&
domain move modal goes here
} {hasBeenDeleted && !onDelete && deleteRedirectPath && } ); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts index 979fa241d94a3..53286fa9ccd22 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts @@ -1,7 +1,11 @@ -import { EntityType } from '../../../../types.generated'; +import { EntityType, PlatformPrivileges } from '../../../../types.generated'; import { GenericEntityProperties } from '../types'; -export function isDeleteDisabled(entityType: EntityType, entityData: GenericEntityProperties | null) { +export function isDeleteDisabled( + entityType: EntityType, + entityData: GenericEntityProperties | null, + platformPrivileges: PlatformPrivileges | null | undefined, +) { if (entityType === EntityType.GlossaryTerm || entityType === EntityType.GlossaryNode) { const entityHasChildren = !!entityData?.children?.total; const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; @@ -13,8 +17,48 @@ export function isDeleteDisabled(entityType: EntityType, entityData: GenericEnti } if (entityType === EntityType.Domain) { const entityHasChildren = !!entityData?.children?.total; - const canDeleteDomainEntity = !entityHasChildren; + const canManageDomains = !!platformPrivileges?.manageDomains; + const canDeleteDomainEntity = !entityHasChildren && canManageDomains; return !canDeleteDomainEntity; } return false; } + +export function isMoveDisabled( + entityType: EntityType, + entityData: GenericEntityProperties | null, + platformPrivileges: PlatformPrivileges | null | undefined, +) { + if (entityType === EntityType.GlossaryTerm || entityType === EntityType.GlossaryNode) { + const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; + return !canManageGlossaryEntity; + } + + if (entityType === EntityType.Domain) { + return !!platformPrivileges?.manageDomains; + } + + return false; +} + +export function shouldDisplayChildDeletionWarning( + entityType: EntityType, + entityData: GenericEntityProperties | null, + platformPrivileges: PlatformPrivileges | null | undefined, +) { + if (entityType === EntityType.GlossaryTerm || entityType === EntityType.GlossaryNode) { + const entityHasChildren = !!entityData?.children?.total; + const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; + const hasTooltip = entityHasChildren && canManageGlossaryEntity; + return hasTooltip; + } + + if (entityType === EntityType.Domain) { + const entityHasChildren = !!entityData?.children?.total; + const canManageDomains = !!platformPrivileges?.manageDomains; + const hasTooltip = entityHasChildren && canManageDomains; + return hasTooltip; + } + + return false; +} From 41bfaae20aff0dd6cf049a36adf62eefc0d50336 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 6 Sep 2023 10:11:28 -0700 Subject: [PATCH 74/91] wip --- .../EntityDropdown/DomainParentSelect.tsx | 21 +++- .../shared/EntityDropdown/EntityDropdown.tsx | 3 +- .../shared/EntityDropdown/MoveDomainModal.tsx | 101 ++++++++++++++++++ .../app/entity/shared/EntityDropdown/utils.ts | 7 +- datahub-web-react/src/graphql/domain.graphql | 4 + 5 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index 3963326249a46..cc1c66e6b85f4 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -7,16 +7,27 @@ import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; import useParentSelector from './useParentSelector'; import DomainNavigator from '../../../domain/nestedDomains/domainNavigator/DomainNavigator'; import { useDomainsContext } from '../../../domain/DomainsContext'; +import { useEntityData } from '../EntityContext'; + +// filter out entity itself and its children +export function filterResultsForMove(entity: Domain, entityUrn: string) { + return ( + entity.urn !== entityUrn && + entity.__typename === 'Domain' && + !entity.parentDomains?.domains.some((node) => node.urn === entityUrn) + ); +} interface Props { selectedParentUrn: string; setSelectedParentUrn: (parent: string) => void; + isMoving?: boolean; } -export default function DomainParentSelect(props: Props) { - const { selectedParentUrn, setSelectedParentUrn } = props; +export default function DomainParentSelect({ selectedParentUrn, setSelectedParentUrn, isMoving }: Props) { const entityRegistry = useEntityRegistry(); const { entityData } = useDomainsContext(); + const { urn: entityDataUrn } = useEntityData(); const { searchResults, @@ -35,6 +46,10 @@ export default function DomainParentSelect(props: Props) { setSelectedParentUrn, }); + const domainSearchResultsFiltered = isMoving + ? searchResults.filter((r) => filterResultsForMove(r.entity as Domain, entityDataUrn)) + : []; + function selectDomain(domain: Domain) { selectParentFromBrowser(domain.urn, entityRegistry.getDisplayName(EntityType.Domain, domain)); } @@ -54,7 +69,7 @@ export default function DomainParentSelect(props: Props) { onFocus={() => setIsFocusedOnInput(true)} dropdownStyle={isShowingDomainNavigator || !searchQuery ? { display: 'none' } : {}} > - {searchResults?.map((result) => ( + {domainSearchResultsFiltered.map((result) => ( {entityRegistry.getDisplayName(result.entity.type, result.entity)} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index adbd7d6d369d5..3bb88fc309f1e 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -22,6 +22,7 @@ import useDeleteEntity from './useDeleteEntity'; import { getEntityProfileDeleteRedirectPath } from '../../../shared/deleteUtils'; import { shouldDisplayChildDeletionWarning, isDeleteDisabled, isMoveDisabled } from './utils'; import { useUserContext } from '../../../context/useUserContext'; +import MoveDomainModal from './MoveDomainModal'; export enum EntityMenuItems { COPY_URL, @@ -255,7 +256,7 @@ function EntityDropdown(props: Props) { {isMoveModalVisible && isGlossaryEntity && ( setIsMoveModalVisible(false)} /> )} - {isMoveModalVisible && isDomainEntity &&
domain move modal goes here
} + {isMoveModalVisible && isDomainEntity && setIsMoveModalVisible(false)} />} {hasBeenDeleted && !onDelete && deleteRedirectPath && } ); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx new file mode 100644 index 0000000000000..d2e3b7f820ae6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx @@ -0,0 +1,101 @@ +import React, { useState } from 'react'; +import styled from 'styled-components/macro'; +import { message, Button, Modal, Typography, Form } from 'antd'; +import { useEntityData, useRefetch } from '../EntityContext'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { useUpdateParentNodeMutation } from '../../../../graphql/glossary.generated'; +import NodeParentSelect from './NodeParentSelect'; +import { getGlossaryRootToUpdate, getParentNodeToUpdate } from '../../../glossary/utils'; +import { useDomainsContext } from '../../../domain/DomainsContext'; + +const StyledItem = styled(Form.Item)` + margin-bottom: 0; +`; + +const OptionalWrapper = styled.span` + font-weight: normal; +`; + +interface Props { + onClose: () => void; +} + +function MoveDomainModal(props: Props) { + const { onClose } = props; + const { urn: entityDataUrn, entityData, entityType } = useEntityData(); + const { parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); + const [form] = Form.useForm(); + const entityRegistry = useEntityRegistry(); + const [selectedParentUrn, setSelectedParentUrn] = useState(''); + const refetch = useRefetch(); + + const [updateParentNode] = useUpdateParentNodeMutation(); + + function moveDomain() { + updateParentNode({ + variables: { + input: { + resourceUrn: entityDataUrn, + parentNode: selectedParentUrn || null, + }, + }, + }) + .then(() => { + message.loading({ content: 'Updating...', duration: 2 }); + setTimeout(() => { + message.success({ + content: `Moved ${entityRegistry.getEntityName(entityType)}!`, + duration: 2, + }); + refetch(); + // if (isInGlossaryContext) { + if (true) { + const oldParentToUpdate = getParentNodeToUpdate(entityData, entityType); + const newParentToUpdate = selectedParentUrn || getGlossaryRootToUpdate(entityType); + setParentDomainsToUpdate([...parentDomainsToUpdate, oldParentToUpdate, newParentToUpdate]); + } + }, 2000); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to move: \n ${e.message || ''}`, duration: 3 }); + }); + onClose(); + } + + return ( + + + + + } + > +
+ + Move To (optional) + + } + > + + + + +
+ + ); +} + +export default MoveDomainModal; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts index 53286fa9ccd22..0a4c2c34441a4 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts @@ -33,11 +33,10 @@ export function isMoveDisabled( const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; return !canManageGlossaryEntity; } - if (entityType === EntityType.Domain) { - return !!platformPrivileges?.manageDomains; + const canManageDomains = !!platformPrivileges?.manageDomains; + return !canManageDomains; } - return false; } @@ -52,13 +51,11 @@ export function shouldDisplayChildDeletionWarning( const hasTooltip = entityHasChildren && canManageGlossaryEntity; return hasTooltip; } - if (entityType === EntityType.Domain) { const entityHasChildren = !!entityData?.children?.total; const canManageDomains = !!platformPrivileges?.manageDomains; const hasTooltip = entityHasChildren && canManageDomains; return hasTooltip; } - return false; } diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 7ffe217ca0fed..951b93fcba9af 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -71,6 +71,10 @@ mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) } +mutation moveDomain($input: MoveDomainInput!) { + moveDomain(input: $input) +} + mutation deleteDomain($urn: String!) { deleteDomain(urn: $urn) } From dff1fe7317961cc8274713005392f1be0c904b9b Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 6 Sep 2023 17:09:52 -0700 Subject: [PATCH 75/91] big wip --- .../datahub/graphql/GmsGraphQLEngine.java | 2 + .../resolvers/mutate/MoveDomainResolver.java | 2 + .../resolvers/mutate/UpdateNameResolver.java | 2 +- .../resolvers/mutate/util/DomainUtils.java | 110 +++++++++++++++--- .../src/main/resources/entity.graphql | 5 + .../EntityDropdown/DomainParentSelect.tsx | 1 - .../shared/EntityDropdown/MoveDomainModal.tsx | 28 ++--- .../useHandleMoveDomainComplete.ts | 31 +++++ 8 files changed, 145 insertions(+), 36 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 4c4f7b7a76e56..a54797b741735 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -187,6 +187,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.BatchSetDomainResolver; import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateDeprecationResolver; import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateSoftDeletedResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MoveDomainResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeBatchResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeResolver; import com.linkedin.datahub.graphql.resolvers.mutate.RemoveLinkResolver; @@ -945,6 +946,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("removeGroup", new RemoveGroupResolver(this.entityClient)) .dataFetcher("updateUserStatus", new UpdateUserStatusResolver(this.entityClient)) .dataFetcher("createDomain", new CreateDomainResolver(this.entityClient, this.entityService)) + .dataFetcher("moveDomain", new MoveDomainResolver(this.entityService, this.entityClient)) .dataFetcher("deleteDomain", new DeleteDomainResolver(entityClient)) .dataFetcher("setDomain", new SetDomainResolver(this.entityClient, this.entityService)) .dataFetcher("batchSetDomain", new BatchSetDomainResolver(this.entityService)) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java index d99f88e2707c3..e5e3a5a0ee42e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java @@ -78,6 +78,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); MutationUtils.persistAspect(resourceUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, properties, actor, _entityService); return true; + } catch (DataHubGraphQLException e) { + throw e; } catch (Exception e) { log.error("Failed to move domain {} to parent {} : {}", input.getResourceUrn(), input.getParentDomain(), e.getMessage()); throw new RuntimeException(String.format("Failed to move domain %s to %s", input.getResourceUrn(), input.getParentDomain()), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java index 00a2c3a80a3fb..0e316ac1296ee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java @@ -132,7 +132,7 @@ private Boolean updateDomainName( throw new IllegalArgumentException("Domain does not exist"); } - if (DomainUtils.hasNameConflict(input.getName(), domainProperties.getParentDomain(), context, _entityClient)) { + if (DomainUtils.hasNameConflict(input.getName(), DomainUtils.getParentDomainSafely(domainProperties), context, _entityClient)) { throw new DataHubGraphQLException( String.format("\"%s\" already exists in this domain. Please pick a unique name.", input.getName()), DataHubGraphQLErrorCode.CONFLICT diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index a07b942d1b65d..585fbdf53a2ba 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -22,11 +22,13 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityUtils; import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; @@ -48,8 +50,9 @@ // TODO: Move to consuming from DomainService. @Slf4j public class DomainUtils { - public static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; - public static final String NAME_INDEX_FIELD_NAME = "name"; + private static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; + private static final String HAS_PARENT_DOMAIN_INDEX_FIELD_NAME = "hasParentDomain"; + private static final String NAME_INDEX_FIELD_NAME = "name"; private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() @@ -109,26 +112,86 @@ public static void validateDomain(Urn domainUrn, EntityService entityService) { } } - private static Criterion buildParentDomainCriterion(@Nullable final Urn parentDomainUrn) { - return QueryUtils.newCriterion( - PARENT_DOMAIN_INDEX_FIELD_NAME, - parentDomainUrn == null ? "" : parentDomainUrn.toString(), - parentDomainUrn == null ? Condition.IS_NULL : Condition.EQUAL + private static List buildRootDomainCriteria() { + final List criteria = new ArrayList<>(); + + criteria.add( + new Criterion() + .setField(HAS_PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue("false") + .setCondition(Condition.EQUAL) + ); + criteria.add( + new Criterion() + .setField(HAS_PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue("") + .setCondition(Condition.IS_NULL) + ); + + return criteria; + } + + private static List buildParentDomainCriteria(@Nonnull final Urn parentDomainUrn) { + final List criteria = new ArrayList<>(); + + criteria.add( + new Criterion() + .setField(HAS_PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue("true") + .setCondition(Condition.EQUAL) + ); + criteria.add( + new Criterion() + .setField(PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue(parentDomainUrn.toString()) + .setCondition(Condition.EQUAL) ); + + return criteria; } private static Criterion buildNameCriterion(@Nonnull final String name) { - return QueryUtils.newCriterion(NAME_INDEX_FIELD_NAME, name, Condition.EQUAL); + return new Criterion() + .setField(NAME_INDEX_FIELD_NAME) + .setValue(name) + .setCondition(Condition.EQUAL); } - public static Filter buildParentDomainFilter(@Nullable final Urn parentDomainUrn) { - return QueryUtils.newFilter(buildParentDomainCriterion(parentDomainUrn)); + /** + * Builds a filter that ORs together the root parent criterion / ANDs together the parent domain criterion. + * The reason for the OR on root is elastic can have a null|false value to represent an root domain in the index. + * @param name an optional name to AND in to each condition of the filter + * @param parentDomainUrn the parent domain (null means root). + * @return the Filter + */ + public static Filter buildNameAndParentDomainFilter(@Nullable final String name, @Nullable final Urn parentDomainUrn) { + if (parentDomainUrn == null) { + return new Filter().setOr( + new ConjunctiveCriterionArray( + buildRootDomainCriteria().stream().map(parentCriterion -> { + final CriterionArray array = new CriterionArray(parentCriterion); + if (name != null) { + array.add(buildNameCriterion(name)); + } + return new ConjunctiveCriterion().setAnd(array); + }).collect(Collectors.toList()) + ) + ); + } + + final CriterionArray andArray = new CriterionArray(buildParentDomainCriteria(parentDomainUrn)); + if (name != null) { + andArray.add(buildNameCriterion(name)); + } + return new Filter().setOr( + new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd(andArray) + ) + ); } - public static Filter buildNameAndParentDomainFilter(@Nonnull final String name, @Nullable final Urn parentDomainUrn) { - final Criterion parentDomainCriterion = buildParentDomainCriterion(parentDomainUrn); - final Criterion nameCriterion = buildNameCriterion(name); - return QueryUtils.getFilterFromCriteria(List.of(parentDomainCriterion, nameCriterion)); + public static Filter buildParentDomainFilter(@Nullable final Urn parentDomainUrn) { + return buildNameAndParentDomainFilter(null, parentDomainUrn); } /** @@ -223,9 +286,8 @@ public static Entity getParentDomain( if (entityResponse != null && entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { final DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); - if (properties.hasParentDomain()) { - return properties.getParentDomain() != null ? UrnToEntityMapper.map(properties.getParentDomain()) : null; - } + final Urn parentDomainUrn = getParentDomainSafely(properties); + return parentDomainUrn != null ? UrnToEntityMapper.map(parentDomainUrn) : null; } } catch (Exception e) { throw new RuntimeException(String.format("Failed to retrieve parent domain for entity %s", urn), e); @@ -233,4 +295,16 @@ public static Entity getParentDomain( return null; } + + /** + * Get a parent domain only if hasParentDomain was set. There is strange elastic behavior where moving a domain + * to the root leaves the parentDomain field set but makes hasParentDomain false. This helper makes sure that queries + * to elastic where hasParentDomain=false and parentDomain=value only gives us the parentDomain if hasParentDomain=true. + * @param properties the domain properties aspect + * @return the parentDomain or null + */ + @Nullable + public static Urn getParentDomainSafely(@Nonnull final DomainProperties properties) { + return properties.hasParentDomain() ? properties.getParentDomain() : null; + } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index bfcc185e18348..84e504f4d70a9 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -430,6 +430,11 @@ type Mutation { """ createDomain(input: CreateDomainInput!): String + """ + Moves a domain to be parented under another domain. + """ + moveDomain(input: MoveDomainInput!): Boolean + """ Delete a Domain """ diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index cc1c66e6b85f4..c25b67bc1c1bf 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -45,7 +45,6 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen selectedParentUrn, setSelectedParentUrn, }); - const domainSearchResultsFiltered = isMoving ? searchResults.filter((r) => filterResultsForMove(r.entity as Domain, entityDataUrn)) : []; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx index d2e3b7f820ae6..2a72a01cb8b37 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx @@ -3,10 +3,9 @@ import styled from 'styled-components/macro'; import { message, Button, Modal, Typography, Form } from 'antd'; import { useEntityData, useRefetch } from '../EntityContext'; import { useEntityRegistry } from '../../../useEntityRegistry'; -import { useUpdateParentNodeMutation } from '../../../../graphql/glossary.generated'; -import NodeParentSelect from './NodeParentSelect'; -import { getGlossaryRootToUpdate, getParentNodeToUpdate } from '../../../glossary/utils'; -import { useDomainsContext } from '../../../domain/DomainsContext'; +import { useMoveDomainMutation } from '../../../../graphql/domain.generated'; +import DomainParentSelect from './DomainParentSelect'; +import { useHandleMoveDomainComplete } from './useHandleMoveDomainComplete'; const StyledItem = styled(Form.Item)` margin-bottom: 0; @@ -22,21 +21,22 @@ interface Props { function MoveDomainModal(props: Props) { const { onClose } = props; - const { urn: entityDataUrn, entityData, entityType } = useEntityData(); - const { parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); + const { urn: entityDataUrn, entityType } = useEntityData(); const [form] = Form.useForm(); const entityRegistry = useEntityRegistry(); const [selectedParentUrn, setSelectedParentUrn] = useState(''); const refetch = useRefetch(); - const [updateParentNode] = useUpdateParentNodeMutation(); + const [moveDomainMutation] = useMoveDomainMutation(); + + const { handleMoveDomainComplete } = useHandleMoveDomainComplete(); function moveDomain() { - updateParentNode({ + moveDomainMutation({ variables: { input: { resourceUrn: entityDataUrn, - parentNode: selectedParentUrn || null, + parentDomain: selectedParentUrn || null, }, }, }) @@ -48,12 +48,8 @@ function MoveDomainModal(props: Props) { duration: 2, }); refetch(); - // if (isInGlossaryContext) { - if (true) { - const oldParentToUpdate = getParentNodeToUpdate(entityData, entityType); - const newParentToUpdate = selectedParentUrn || getGlossaryRootToUpdate(entityType); - setParentDomainsToUpdate([...parentDomainsToUpdate, oldParentToUpdate, newParentToUpdate]); - } + const newParentToUpdate = selectedParentUrn || undefined; + handleMoveDomainComplete(entityDataUrn, newParentToUpdate); }, 2000); }) .catch((e) => { @@ -86,7 +82,7 @@ function MoveDomainModal(props: Props) { } > - { + if (!entityData) return; + + const domain = entityData as Domain; + const oldParentUrn = domain.parentDomains?.domains.length ? domain.parentDomains.domains[0].urn : undefined; + removeFromListDomainsCache(client, urn, 1, 1000, oldParentUrn); + updateListDomainsCache( + client, + domain.urn, + undefined, + domain.properties?.name ?? '', + domain.properties?.description ?? '', + newParentUrn, + ); + const newParentDomainsToUpdate = [...parentDomainsToUpdate]; + if (oldParentUrn) newParentDomainsToUpdate.push(oldParentUrn); + if (newParentUrn) newParentDomainsToUpdate.push(newParentUrn); + setParentDomainsToUpdate(newParentDomainsToUpdate); + }; + + return { handleMoveDomainComplete }; +} From aa2dd03eadbe1721e0dd8d016e3dd8f9d19bfde2 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 6 Sep 2023 17:29:17 -0700 Subject: [PATCH 76/91] fix(nested-domains): prevent parent cycles --- .../graphql/resolvers/domain/ParentDomainsResolver.java | 6 +++++- .../fieldresolverprovider/DomainFieldResolverProvider.java | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java index 1e60674705edf..dcaa7d61ed90c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java @@ -11,7 +11,9 @@ import graphql.schema.DataFetchingEnvironment; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import static com.linkedin.metadata.Constants.DOMAIN_ENTITY_NAME; @@ -29,6 +31,7 @@ public CompletableFuture get(DataFetchingEnvironment enviro final QueryContext context = environment.getContext(); final Urn urn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); final List parentDomains = new ArrayList<>(); + final Set visitedParentUrns = new HashSet<>(); if (!DOMAIN_ENTITY_NAME.equals(urn.getEntityType())) { throw new IllegalArgumentException(String.format("Failed to resolve parents for entity type %s", urn)); @@ -38,8 +41,9 @@ public CompletableFuture get(DataFetchingEnvironment enviro try { Entity parentDomain = DomainUtils.getParentDomain(urn, context, _entityClient); - while (parentDomain != null) { + while (parentDomain != null && !visitedParentUrns.contains(parentDomain.getUrn())) { parentDomains.add(parentDomain); + visitedParentUrns.add(parentDomain.getUrn()); parentDomain = DomainUtils.getParentDomain(Urn.createFromString(parentDomain.getUrn()), context, _entityClient); } diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java index b2617f3a5462b..090191b224bdc 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -102,10 +102,12 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { final Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); Set batchedParentUrns = getBatchedParentDomains(domainUrns); + batchedParentUrns.removeAll(domainUrns); while (!batchedParentUrns.isEmpty()) { domainUrns.addAll(batchedParentUrns); batchedParentUrns = getBatchedParentDomains(batchedParentUrns); + batchedParentUrns.removeAll(domainUrns); } return FieldResolver.FieldValue.builder().values(domainUrns From bf4a6762984117b9381d47b83040261e2ea105d3 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 6 Sep 2023 17:34:52 -0700 Subject: [PATCH 77/91] cmt --- .../fieldresolverprovider/DomainFieldResolverProvider.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java index 090191b224bdc..68c1dd4f644e5 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -100,6 +100,11 @@ private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { return FieldResolver.emptyFieldValue(); } + /* + * Build up a set of all directly referenced domains and any of the domains' parent domains. + * To avoid cycles we remove any parents we've already visited to prevent an infinite loop cycle. + */ + final Set domainUrns = new HashSet<>(new Domains(domainsAspect.getValue().data()).getDomains()); Set batchedParentUrns = getBatchedParentDomains(domainUrns); batchedParentUrns.removeAll(domainUrns); From 60cf7f4ec2696910b27b0e05d04b863bf84a12ac Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Wed, 6 Sep 2023 17:53:06 -0700 Subject: [PATCH 78/91] fix(nested-domains): Hide child when moving --- .../nestedDomains/domainNavigator/DomainNavigator.tsx | 4 +++- .../domain/nestedDomains/domainNavigator/DomainNode.tsx | 9 +++++++-- .../entity/shared/EntityDropdown/DomainParentSelect.tsx | 5 ++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx index cd4c3a1fce1dc..5683db11fac0f 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -13,10 +13,11 @@ const NavigatorWrapper = styled.div` `; interface Props { + domainUrnToHide?: string; selectDomainOverride?: (domain: Domain) => void; } -export default function DomainNavigator({ selectDomainOverride }: Props) { +export default function DomainNavigator({ domainUrnToHide, selectDomainOverride }: Props) { const { data, error } = useListDomains({}); return ( @@ -27,6 +28,7 @@ export default function DomainNavigator({ selectDomainOverride }: Props) { key={domain.urn} domain={domain as Domain} numDomainChildren={domain.children?.total || 0} + domainUrnToHide={domainUrnToHide} selectDomainOverride={selectDomainOverride} /> ))} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx index 3603b255c0f95..4541227352014 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -63,10 +63,12 @@ const StyledExpander = styled(BodyGridExpander)` interface Props { domain: Domain; numDomainChildren: number; + domainUrnToHide?: string; selectDomainOverride?: (domain: Domain) => void; } -export default function DomainNode({ domain, numDomainChildren, selectDomainOverride }: Props) { +export default function DomainNode({ domain, numDomainChildren, domainUrnToHide, selectDomainOverride }: Props) { + const shouldHideDomain = domainUrnToHide === domain.urn; const history = useHistory(); const entityRegistry = useEntityRegistry(); const { entityData } = useDomainsContext(); @@ -74,7 +76,7 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver initialValue: false, closeDelay: 250, }); - const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen }); + const { data } = useListDomains({ parentDomain: domain.urn, skip: !isOpen || shouldHideDomain }); const isOnEntityPage = entityData && entityData.urn === domain.urn; const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); const isInSelectMode = !!selectDomainOverride; @@ -97,6 +99,8 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver } } + if (shouldHideDomain) return null; + return ( <> @@ -122,6 +126,7 @@ export default function DomainNode({ domain, numDomainChildren, selectDomainOver key={domain.urn} domain={childDomain as Domain} numDomainChildren={childDomain.children?.total || 0} + domainUrnToHide={domainUrnToHide} selectDomainOverride={selectDomainOverride} /> ))} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index c25b67bc1c1bf..dfdda09ab0785 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -75,7 +75,10 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen ))} - + ); From 06ba5e802d500212a1f381efc9e874109345d0ca Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 10:19:42 -0700 Subject: [PATCH 79/91] sort domains by display name --- .../src/app/domain/nestedDomains/RootDomains.tsx | 4 ++-- .../domainNavigator/DomainNavigator.tsx | 4 ++-- .../nestedDomains/domainNavigator/DomainNode.tsx | 4 ++-- datahub-web-react/src/app/domain/useListDomains.tsx | 8 ++++++-- datahub-web-react/src/app/domain/utils.ts | 13 ++++++++++++- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx index b0bca1cea82ee..b15291d1c73b1 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -19,7 +19,7 @@ const DomainsWrapper = styled.div` export default function RootDomains() { const entityRegistry = useEntityRegistry(); - const { loading, error, data } = useListDomains({}); + const { loading, error, data, sortedDomains } = useListDomains({}); return ( <> @@ -27,7 +27,7 @@ export default function RootDomains() { {!data && loading && } {error && } - {data?.listDomains?.domains.map((domain) => ( + {sortedDomains?.map((domain) => ( {entityRegistry.renderSearchResult(EntityType.Domain, { entity: domain, matchedFields: [] })} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx index 5683db11fac0f..0fbcffb9a260c 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -18,12 +18,12 @@ interface Props { } export default function DomainNavigator({ domainUrnToHide, selectDomainOverride }: Props) { - const { data, error } = useListDomains({}); + const { sortedDomains, error } = useListDomains({}); return ( {error && } - {data?.listDomains?.domains.map((domain) => ( + {sortedDomains?.map((domain) => ( - {data?.listDomains?.domains.map((childDomain) => ( + {sortedDomains?.map((childDomain) => ( (domains?: Array, sortBy?: 'displayName') { + const entityRegistry = useEntityRegistry(); + if (!domains || !sortBy) return domains; + return [...domains].sort((a, b) => { + const nameA = entityRegistry.getDisplayName(EntityType.Domain, a) || ''; + const nameB = entityRegistry.getDisplayName(EntityType.Domain, b) || ''; + return nameA.localeCompare(nameB); + }); +} From 3d4b064d7fd9798906d103009bafcf750fe2cf9f Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 10:25:50 -0700 Subject: [PATCH 80/91] code review feedback --- .../app/entity/shared/EntityDropdown/DomainParentSelect.tsx | 2 +- .../src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index c25b67bc1c1bf..598e2eb0f5bbd 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -47,7 +47,7 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen }); const domainSearchResultsFiltered = isMoving ? searchResults.filter((r) => filterResultsForMove(r.entity as Domain, entityDataUrn)) - : []; + : searchResults; function selectDomain(domain: Domain) { selectParentFromBrowser(domain.urn, entityRegistry.getDisplayName(EntityType.Domain, domain)); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx index 2a72a01cb8b37..1531fa6c5e677 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx @@ -42,14 +42,14 @@ function MoveDomainModal(props: Props) { }) .then(() => { message.loading({ content: 'Updating...', duration: 2 }); + const newParentToUpdate = selectedParentUrn || undefined; + handleMoveDomainComplete(entityDataUrn, newParentToUpdate); setTimeout(() => { message.success({ content: `Moved ${entityRegistry.getEntityName(entityType)}!`, duration: 2, }); refetch(); - const newParentToUpdate = selectedParentUrn || undefined; - handleMoveDomainComplete(entityDataUrn, newParentToUpdate); }, 2000); }) .catch((e) => { From e2345d9eb729ea9bc6578b700a85c226e9658ce9 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 11:24:37 -0700 Subject: [PATCH 81/91] feat(nested-domains): domain autocomplete improvements --- .../src/app/domain/DomainSearch.tsx | 31 ++++++++++++++----- datahub-web-react/src/app/domain/utils.ts | 6 ++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx index 49f3ad2e862aa..a943d9baf0fd7 100644 --- a/datahub-web-react/src/app/domain/DomainSearch.tsx +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -1,6 +1,7 @@ -import React, { useRef, useState } from 'react'; +import React, { CSSProperties, useRef, useState } from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components/macro'; +import Highlight from 'react-highlighter'; import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated'; import { EntityType } from '../../types.generated'; import { IconStyleType } from '../entity/Entity'; @@ -9,6 +10,8 @@ import { SearchBar } from '../search/SearchBar'; import ClickOutside from '../shared/ClickOutside'; import { useEntityRegistry } from '../useEntityRegistry'; import DomainIcon from './DomainIcon'; +import ParentEntities from '../search/filters/ParentEntities'; +import { getParentDomains } from './utils'; const DomainSearchWrapper = styled.div` position: relative; @@ -32,7 +35,9 @@ const ResultsWrapper = styled.div` const SearchResult = styled(Link)` color: #262626; - display: inline-block; + display: flex; + align-items: center; + gap: 8px; height: 100%; padding: 6px 8px; width: 100%; @@ -42,9 +47,13 @@ const SearchResult = styled(Link)` } `; -const IconWrapper = styled.span` - margin-right: 8px; -`; +const IconWrapper = styled.span``; + +const highlightMatchStyle: CSSProperties = { + fontWeight: 'bold', + background: 'none', + padding: 0, +}; function DomainSearch() { const [query, setQuery] = useState(''); @@ -71,6 +80,7 @@ function DomainSearch() { setQuery(q); }, 250); }; + console.log({ data }); return ( @@ -98,7 +108,7 @@ function DomainSearch() { {searchResults.map((result) => { return ( setIsSearchBarFocused(false)} > @@ -113,7 +123,14 @@ function DomainSearch() { entityRegistry.getIcon(result.entity.type, 12, IconStyleType.ACCENT) )} - {entityRegistry.getDisplayName(result.entity.type, result.entity)} +
+ + + {entityRegistry.getDisplayName(result.entity.type, result.entity)} + +
); })} diff --git a/datahub-web-react/src/app/domain/utils.ts b/datahub-web-react/src/app/domain/utils.ts index e33c412afd48e..8273c33e2c41d 100644 --- a/datahub-web-react/src/app/domain/utils.ts +++ b/datahub-web-react/src/app/domain/utils.ts @@ -7,6 +7,7 @@ import { GenericEntityProperties } from '../entity/shared/types'; import usePrevious from '../shared/usePrevious'; import { useDomainsContext } from './DomainsContext'; import { useEntityRegistry } from '../useEntityRegistry'; +import EntityRegistry from '../entity/EntityRegistry'; /** * Add an entry to the list domains cache. @@ -137,3 +138,8 @@ export function useSortedDomains(domains?: Array, sortBy?: return nameA.localeCompare(nameB); }); } + +export function getParentDomains(domain: T, entityRegistry: EntityRegistry) { + const props = entityRegistry.getGenericEntityProperties(EntityType.Domain, domain); + return props?.parentDomains?.domains ?? []; +} From 7502f9413eb988cc0e3d33d95afec06d192d6a77 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 11:48:53 -0700 Subject: [PATCH 82/91] nested domain analytics --- datahub-web-react/src/app/analytics/event.ts | 9 +++++++++ .../src/app/domain/CreateDomainModal.tsx | 1 + .../shared/EntityDropdown/MoveDomainModal.tsx | 17 +++++++++++------ .../useHandleMoveDomainComplete.ts | 9 +++++++++ .../datahubusage/DataHubUsageEventType.java | 1 + 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index 84173b522fb07..28cd61ff3171a 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -55,6 +55,7 @@ export enum EventType { ShowStandardHomepageEvent, CreateGlossaryEntityEvent, CreateDomainEvent, + MoveDomainEvent, CreateIngestionSourceEvent, UpdateIngestionSourceEvent, DeleteIngestionSourceEvent, @@ -454,6 +455,13 @@ export interface CreateGlossaryEntityEvent extends BaseEvent { export interface CreateDomainEvent extends BaseEvent { type: EventType.CreateDomainEvent; + parentDomainUrn?: string; +} + +export interface MoveDomainEvent extends BaseEvent { + type: EventType.MoveDomainEvent; + oldParentDomainUrn?: string; + parentDomainUrn?: string; } // Managed Ingestion Events @@ -653,6 +661,7 @@ export type Event = | ShowStandardHomepageEvent | CreateGlossaryEntityEvent | CreateDomainEvent + | MoveDomainEvent | CreateIngestionSourceEvent | UpdateIngestionSourceEvent | DeleteIngestionSourceEvent diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index d17ba21bb8d3f..f22c305f48f16 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -57,6 +57,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { if (!errors) { analytics.event({ type: EventType.CreateDomainEvent, + parentDomainUrn: selectedParentUrn || undefined, }); message.success({ content: `Created domain!`, diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx index 1531fa6c5e677..cdbf6fdabf3c9 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx @@ -1,11 +1,13 @@ import React, { useState } from 'react'; import styled from 'styled-components/macro'; import { message, Button, Modal, Typography, Form } from 'antd'; -import { useEntityData, useRefetch } from '../EntityContext'; +import { useRefetch } from '../EntityContext'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { useMoveDomainMutation } from '../../../../graphql/domain.generated'; import DomainParentSelect from './DomainParentSelect'; import { useHandleMoveDomainComplete } from './useHandleMoveDomainComplete'; +import { useDomainsContext } from '../../../domain/DomainsContext'; +import { EntityType } from '../../../../types.generated'; const StyledItem = styled(Form.Item)` margin-bottom: 0; @@ -21,7 +23,8 @@ interface Props { function MoveDomainModal(props: Props) { const { onClose } = props; - const { urn: entityDataUrn, entityType } = useEntityData(); + const { entityData } = useDomainsContext(); + const domainUrn = entityData?.urn; const [form] = Form.useForm(); const entityRegistry = useEntityRegistry(); const [selectedParentUrn, setSelectedParentUrn] = useState(''); @@ -32,21 +35,23 @@ function MoveDomainModal(props: Props) { const { handleMoveDomainComplete } = useHandleMoveDomainComplete(); function moveDomain() { + if (!domainUrn) return; + moveDomainMutation({ variables: { input: { - resourceUrn: entityDataUrn, - parentDomain: selectedParentUrn || null, + resourceUrn: domainUrn, + parentDomain: selectedParentUrn || undefined, }, }, }) .then(() => { message.loading({ content: 'Updating...', duration: 2 }); const newParentToUpdate = selectedParentUrn || undefined; - handleMoveDomainComplete(entityDataUrn, newParentToUpdate); + handleMoveDomainComplete(domainUrn, newParentToUpdate); setTimeout(() => { message.success({ - content: `Moved ${entityRegistry.getEntityName(entityType)}!`, + content: `Moved ${entityRegistry.getEntityName(EntityType.Domain)}!`, duration: 2, }); refetch(); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts index c978bc53f8611..81f19331e18b7 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts @@ -2,6 +2,8 @@ import { useApolloClient } from '@apollo/client'; import { removeFromListDomainsCache, updateListDomainsCache } from '../../../domain/utils'; import { useDomainsContext } from '../../../domain/DomainsContext'; import { Domain } from '../../../../types.generated'; +import analytics from '../../../analytics/analytics'; +import { EventType } from '../../../analytics'; export function useHandleMoveDomainComplete() { const client = useApolloClient(); @@ -12,6 +14,13 @@ export function useHandleMoveDomainComplete() { const domain = entityData as Domain; const oldParentUrn = domain.parentDomains?.domains.length ? domain.parentDomains.domains[0].urn : undefined; + + analytics.event({ + type: EventType.MoveDomainEvent, + oldParentDomainUrn: oldParentUrn, + parentDomainUrn: newParentUrn, + }); + removeFromListDomainsCache(client, urn, 1, 1000, oldParentUrn); updateListDomainsCache( client, diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java index 14b301f93f4ef..036fb20b33f20 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java @@ -53,6 +53,7 @@ public enum DataHubUsageEventType { SHOW_STANDARD_HOME_PAGE_EVENT("ShowStandardHomepageEvent"), CREATE_GLOSSARY_ENTITY_EVENT("CreateGlossaryEntityEvent"), CREATE_DOMAIN_EVENT("CreateDomainEvent"), + MOVE_DOMAIN_EVENT("MoveDomainEvent"), CREATE_INGESTION_SOURCE_EVENT("CreateIngestionSourceEvent"), UPDATE_INGESTION_SOURCE_EVENT("UpdateIngestionSourceEvent"), DELETE_INGESTION_SOURCE_EVENT("DeleteIngestionSourceEvent"), From b8aa3e67ce6b69ccf53be12bd5563bc6f904b964 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 11:50:46 -0700 Subject: [PATCH 83/91] remove log --- datahub-web-react/src/app/domain/DomainSearch.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx index a943d9baf0fd7..e82dae9c2c9e6 100644 --- a/datahub-web-react/src/app/domain/DomainSearch.tsx +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -80,7 +80,6 @@ function DomainSearch() { setQuery(q); }, 250); }; - console.log({ data }); return ( From 27ddd91c2ed1bc97895db04d0cc1f5f7f63c6569 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 14:05:05 -0700 Subject: [PATCH 84/91] remove "your domains" header --- .../src/app/domain/nestedDomains/RootDomains.tsx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx index b15291d1c73b1..65b54451aedbe 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -6,11 +6,11 @@ import { useEntityRegistry } from '../../useEntityRegistry'; import { EntityType } from '../../../types.generated'; import useListDomains from '../useListDomains'; -const RootDomainsHeader = styled.div` - font-size: 20px; - margin-bottom: 18px; - padding: 0 28px; -`; +// const RootDomainsHeader = styled.div` +// font-size: 20px; +// margin-bottom: 18px; +// padding: 0 28px; +// `; const DomainsWrapper = styled.div` overflow: auto; @@ -23,7 +23,6 @@ export default function RootDomains() { return ( <> - Your Domains {!data && loading && } {error && } From 7439bfe58c6241e56f55dab46af386f8363c325c Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 14:47:48 -0700 Subject: [PATCH 85/91] styling feedback --- .../src/app/domain/CreateDomainModal.tsx | 74 ++++++++++++------- .../EntityDropdown/DomainParentSelect.tsx | 1 + .../EntityDropdown/useParentSelector.ts | 2 +- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index d17ba21bb8d3f..8052be05ffa30 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -9,7 +9,7 @@ import DomainParentSelect from '../entity/shared/EntityDropdown/DomainParentSele import { useDomainsContext } from './DomainsContext'; const SuggestedNamesGroup = styled.div` - margin-top: 12px; + margin-top: 8px; `; const ClickableTag = styled(Tag)` @@ -18,6 +18,29 @@ const ClickableTag = styled(Tag)` } `; +const FormItem = styled(Form.Item)` + .ant-form-item-label { + padding-bottom: 2px; + } +`; + +const FormItemWithMargin = styled(FormItem)` + margin-bottom: 16px; +`; + +const FormItemNoMargin = styled(FormItem)` + margin-bottom: 0; +`; + +const FormItemLabel = styled(Typography.Text)` + font-weight: 600; + color: #373d44; +`; + +const AdvancedLabel = styled(Typography.Text)` + color: #373d44; +`; + type Props = { onClose: () => void; onCreate: ( @@ -86,7 +109,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { return ( field.errors.length > 0)); }} > - Parent (optional)}> + Parent (optional)}> - - Name}> - Give your new Domain a name. - + Name}> + - + {SUGGESTED_DOMAIN_NAMES.map((name) => { return ( @@ -152,29 +174,29 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { ); })} - - Description}> - - An optional description for your new domain. You can change this later. - - + Description} + help="You can always change the description later." + > + - - + + - Advanced} key="1"> - Domain Id}> - - By default, a random UUID will be generated to uniquely identify this domain. If - you'd like to provide a custom id instead to more easily keep track of this domain, + Advanced Options} key="1"> + Domain Id} + help="By default, a random UUID will be generated to uniquely identify this domain. If + you'd like to provide a custom id instead to more easily keep track of this domain, you may provide it here. Be careful, you cannot easily change the domain id after - creation. - - + ({ @@ -188,8 +210,8 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { ]} > - - + + diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index ab60ae7b81731..506bdbfb900f7 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -60,6 +60,7 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen } placeholder="Select" filterOption={false} value={selectedParentName} onSelect={onSelectParent} onSearch={handleSearch} - onClear={clearSelectedParent} - onFocus={() => setIsFocusedOnInput(true)} + onFocus={handleFocus} dropdownStyle={isShowingDomainNavigator || !searchQuery ? { display: 'none' } : {}} > {domainSearchResultsFiltered.map((result) => ( diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts index af199c94e98dd..32b5d8ca790cc 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts @@ -52,7 +52,7 @@ export default function useParentSelector({ entityType, entityData, selectedPare function clearSelectedParent() { setSelectedParentUrn(''); - setSelectedParentName(''); + setSelectedParentName(undefined); setSearchQuery(''); } From f4330e217f3709a759a5f823b2627de648087c39 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Thu, 7 Sep 2023 15:29:52 -0700 Subject: [PATCH 87/91] tidy up --- .../src/app/domain/nestedDomains/RootDomains.tsx | 6 ------ .../app/entity/shared/EntityDropdown/DomainParentSelect.tsx | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx index 65b54451aedbe..757119919e336 100644 --- a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -6,12 +6,6 @@ import { useEntityRegistry } from '../../useEntityRegistry'; import { EntityType } from '../../../types.generated'; import useListDomains from '../useListDomains'; -// const RootDomainsHeader = styled.div` -// font-size: 20px; -// margin-bottom: 18px; -// padding: 0 28px; -// `; - const DomainsWrapper = styled.div` overflow: auto; padding: 0 28px 16px 28px; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index 2cea047631e0f..3a14d41d900ec 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -60,7 +60,7 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen const handleClickOutside = () => setIsFocusedOnInput(false); const handleClear = (event: MouseEvent) => { - // Prevent, otherwise the default clear button triggers the onClickOutside + // Prevent, otherwise antd will close the select menu but leaves it focused event.stopPropagation(); clearSelectedParent(); }; From af9310ec223c788274728d4ae4c13afe5fd996c9 Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 8 Sep 2023 10:13:49 -0700 Subject: [PATCH 88/91] feature flag wire up --- .../src/app/domain/CreateDomainModal.tsx | 20 ++++++++++++------- .../shared/EntityDropdown/EntityDropdown.tsx | 4 +++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index 750536fa9686c..ca1bc30596003 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -6,6 +6,7 @@ import { useEnterKeyListener } from '../shared/useEnterKeyListener'; import { validateCustomUrnId } from '../shared/textUtil'; import analytics, { EventType } from '../analytics'; import DomainParentSelect from '../entity/shared/EntityDropdown/DomainParentSelect'; +import { useIsNestedDomainsEnabled } from '../useAppConfig'; import { useDomainsContext } from './DomainsContext'; const SuggestedNamesGroup = styled.div` @@ -59,9 +60,12 @@ const NAME_FIELD_NAME = 'name'; const DESCRIPTION_FIELD_NAME = 'description'; export default function CreateDomainModal({ onClose, onCreate }: Props) { + const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); const [createDomainMutation] = useCreateDomainMutation(); const { entityData } = useDomainsContext(); - const [selectedParentUrn, setSelectedParentUrn] = useState(entityData?.urn || ''); + const [selectedParentUrn, setSelectedParentUrn] = useState( + (isNestedDomainsEnabled && entityData?.urn) || '', + ); const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); @@ -137,12 +141,14 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0)); }} > - Parent (optional)}> - - + {isNestedDomainsEnabled && ( + Parent (optional)}> + + + )} Name}> )} - {menuItems.has(EntityMenuItems.MOVE) && ( + {isNestedDomainsEnabled && menuItems.has(EntityMenuItems.MOVE) && ( Date: Fri, 8 Sep 2023 11:04:35 -0700 Subject: [PATCH 89/91] use domains context in v1 --- .../src/app/domain/ManageDomainsPage.tsx | 31 ++++++++++++------- .../EntityDropdown/DomainParentSelect.tsx | 12 +++---- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/datahub-web-react/src/app/domain/ManageDomainsPage.tsx b/datahub-web-react/src/app/domain/ManageDomainsPage.tsx index 6172ac0246f58..3e19da1875037 100644 --- a/datahub-web-react/src/app/domain/ManageDomainsPage.tsx +++ b/datahub-web-react/src/app/domain/ManageDomainsPage.tsx @@ -1,7 +1,9 @@ import { Typography } from 'antd'; -import React from 'react'; +import React, { useState } from 'react'; import styled from 'styled-components'; import { DomainsList } from './DomainsList'; +import { DomainsContext } from './DomainsContext'; +import { GenericEntityProperties } from '../entity/shared/types'; const PageContainer = styled.div` padding-top: 20px; @@ -22,17 +24,22 @@ const PageTitle = styled(Typography.Title)` const ListContainer = styled.div``; export const ManageDomainsPage = () => { + const [entityData, setEntityData] = useState(null); + const [parentDomainsToUpdate, setParentDomainsToUpdate] = useState([]); + return ( - - - Domains - - View your DataHub Domains. Take administrative actions. - - - - - - + + + + Domains + + View your DataHub Domains. Take administrative actions. + + + + + + + ); }; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index 3a14d41d900ec..00a7a030f6b6c 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -8,7 +8,6 @@ import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; import useParentSelector from './useParentSelector'; import DomainNavigator from '../../../domain/nestedDomains/domainNavigator/DomainNavigator'; import { useDomainsContext } from '../../../domain/DomainsContext'; -import { useEntityData } from '../EntityContext'; // filter out entity itself and its children export function filterResultsForMove(entity: Domain, entityUrn: string) { @@ -28,7 +27,7 @@ interface Props { export default function DomainParentSelect({ selectedParentUrn, setSelectedParentUrn, isMoving }: Props) { const entityRegistry = useEntityRegistry(); const { entityData } = useDomainsContext(); - const { urn: entityDataUrn } = useEntityData(); + const domainUrn = entityData?.urn; const { searchResults, @@ -46,9 +45,10 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen selectedParentUrn, setSelectedParentUrn, }); - const domainSearchResultsFiltered = isMoving - ? searchResults.filter((r) => filterResultsForMove(r.entity as Domain, entityDataUrn)) - : searchResults; + const domainSearchResultsFiltered = + isMoving && domainUrn + ? searchResults.filter((r) => filterResultsForMove(r.entity as Domain, domainUrn)) + : searchResults; function selectDomain(domain: Domain) { selectParentFromBrowser(domain.urn, entityRegistry.getDisplayName(EntityType.Domain, domain)); @@ -87,7 +87,7 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen From e0be65de98601dee88cd22b07db58b68ef399b9f Mon Sep 17 00:00:00 2001 From: Josh Eilers Date: Fri, 8 Sep 2023 12:30:09 -0700 Subject: [PATCH 90/91] fix move link condition --- .../src/app/entity/shared/EntityDropdown/EntityDropdown.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index cbeae68288a7b..be975249b2670 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -138,6 +138,7 @@ function EntityDropdown(props: Props) { const isGlossaryEntity = entityType === EntityType.GlossaryNode || entityType === EntityType.GlossaryTerm; const isDomainEntity = entityType === EntityType.Domain; const canCreateGlossaryEntity = !!entityData?.privileges?.canManageChildren; + const isDomainMoveHidden = !isNestedDomainsEnabled && isDomainEntity; /** * A default path to redirect to if the entity is deleted. @@ -196,7 +197,7 @@ function EntityDropdown(props: Props) { )} - {isNestedDomainsEnabled && menuItems.has(EntityMenuItems.MOVE) && ( + {!isDomainMoveHidden && menuItems.has(EntityMenuItems.MOVE) && ( Date: Wed, 13 Sep 2023 10:07:29 -0700 Subject: [PATCH 91/91] fix(nested-domains): show parent path in modals --- .../shared/EntityDropdown/DomainParentSelect.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx index 00a7a030f6b6c..d43b04ec11a16 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -1,6 +1,7 @@ import React, { MouseEvent } from 'react'; import { Select } from 'antd'; import { CloseCircleFilled } from '@ant-design/icons'; +import styled from 'styled-components'; import { Domain, EntityType } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; import ClickOutside from '../../../shared/ClickOutside'; @@ -8,6 +9,14 @@ import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; import useParentSelector from './useParentSelector'; import DomainNavigator from '../../../domain/nestedDomains/domainNavigator/DomainNavigator'; import { useDomainsContext } from '../../../domain/DomainsContext'; +import ParentEntities from '../../../search/filters/ParentEntities'; +import { getParentDomains } from '../../../domain/utils'; + +const SearchResultContainer = styled.div` + display: flex; + flex-direction: column; + justify-content: center; +`; // filter out entity itself and its children export function filterResultsForMove(entity: Domain, entityUrn: string) { @@ -81,7 +90,10 @@ export default function DomainParentSelect({ selectedParentUrn, setSelectedParen > {domainSearchResultsFiltered.map((result) => ( - {entityRegistry.getDisplayName(result.entity.type, result.entity)} + + + {entityRegistry.getDisplayName(result.entity.type, result.entity)} + ))}