diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
index 5919d2dbf5b7e..e5c01252a865b 100644
--- a/datahub-web-react/src/app/home/HomePageHeader.tsx
+++ b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -275,6 +275,7 @@ export const HomePageHeader = () => {
                         viewsEnabled={viewsEnabled}
                         combineSiblings
                         showQuickFilters
+                        showViewAllResults
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>
diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
index f4c048bcaf0d2..3092364bb8bdd 100644
--- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -190,7 +190,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                             <Input
                                 data-testid="cli-version-input"
                                 className="cli-version-input"
-                                placeholder="(e.g. 0.10.5)"
+                                placeholder="(e.g. 0.12.0)"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
                             />
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
index 68e91983babdb..d3e01df3a66e8 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
@@ -67,8 +67,8 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege
     const isMetadataPolicy = policy?.type === PolicyType.Metadata;
 
     const resources = convertLegacyResourceFilter(policy?.resources);
-    const resourceTypes = getFieldValues(resources?.filter, 'RESOURCE_TYPE') || [];
-    const resourceEntities = getFieldValues(resources?.filter, 'RESOURCE_URN') || [];
+    const resourceTypes = getFieldValues(resources?.filter, 'TYPE') || [];
+    const resourceEntities = getFieldValues(resources?.filter, 'URN') || [];
     const domains = getFieldValues(resources?.filter, 'DOMAIN') || [];
 
     const {
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
index 1520388a5033a..b8e1505fceaec 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
@@ -67,8 +67,8 @@ export default function PolicyPrivilegeForm({
     } = useAppConfig();
 
     const resources: ResourceFilter = convertLegacyResourceFilter(maybeResources) || EMPTY_POLICY.resources;
-    const resourceTypes = getFieldValues(resources.filter, 'RESOURCE_TYPE') || [];
-    const resourceEntities = getFieldValues(resources.filter, 'RESOURCE_URN') || [];
+    const resourceTypes = getFieldValues(resources.filter, 'TYPE') || [];
+    const resourceEntities = getFieldValues(resources.filter, 'URN') || [];
 
     const getDisplayName = (entity) => {
         if (!entity) {
@@ -145,10 +145,7 @@ export default function PolicyPrivilegeForm({
         };
         setResources({
             ...resources,
-            filter: setFieldValues(filter, 'RESOURCE_TYPE', [
-                ...resourceTypes,
-                createCriterionValue(selectedResourceType),
-            ]),
+            filter: setFieldValues(filter, 'TYPE', [...resourceTypes, createCriterionValue(selectedResourceType)]),
         });
     };
 
@@ -160,7 +157,7 @@ export default function PolicyPrivilegeForm({
             ...resources,
             filter: setFieldValues(
                 filter,
-                'RESOURCE_TYPE',
+                'TYPE',
                 resourceTypes?.filter((criterionValue) => criterionValue.value !== deselectedResourceType),
             ),
         });
@@ -173,7 +170,7 @@ export default function PolicyPrivilegeForm({
         };
         setResources({
             ...resources,
-            filter: setFieldValues(filter, 'RESOURCE_URN', [
+            filter: setFieldValues(filter, 'URN', [
                 ...resourceEntities,
                 createCriterionValueWithEntity(
                     resource,
@@ -192,7 +189,7 @@ export default function PolicyPrivilegeForm({
             ...resources,
             filter: setFieldValues(
                 filter,
-                'RESOURCE_URN',
+                'URN',
                 resourceEntities?.filter((criterionValue) => criterionValue.value !== resource),
             ),
         });
diff --git a/datahub-web-react/src/app/permissions/policy/policyUtils.ts b/datahub-web-react/src/app/permissions/policy/policyUtils.ts
index c7af7342f6efa..2f178fcdeb5c3 100644
--- a/datahub-web-react/src/app/permissions/policy/policyUtils.ts
+++ b/datahub-web-react/src/app/permissions/policy/policyUtils.ts
@@ -99,10 +99,10 @@ export const convertLegacyResourceFilter = (resourceFilter: Maybe<ResourceFilter
     }
     const criteria = new Array<PolicyMatchCriterion>();
     if (resourceFilter.type) {
-        criteria.push(createCriterion('RESOURCE_TYPE', [createCriterionValue(resourceFilter.type)]));
+        criteria.push(createCriterion('TYPE', [createCriterionValue(resourceFilter.type)]));
     }
     if (resourceFilter.resources && resourceFilter.resources.length > 0) {
-        criteria.push(createCriterion('RESOURCE_URN', resourceFilter.resources.map(createCriterionValue)));
+        criteria.push(createCriterion('URN', resourceFilter.resources.map(createCriterionValue)));
     }
     return {
         filter: {
diff --git a/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx b/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx
index d5722429aaf6b..2bb76714d6119 100644
--- a/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx
+++ b/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx
@@ -39,6 +39,7 @@ export default function EntityPathsModal({ paths, resultEntityUrn, hideModal }:
 
     return (
         <StyledModal
+            data-testid="entity-paths-modal"
             title={
                 <Header>
                     Column path{paths.length > 1 && 's'} from{' '}
diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
index b4699994bc460..5f797e68fe0e8 100644
--- a/datahub-web-react/src/app/search/SearchBar.tsx
+++ b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -119,6 +119,7 @@ interface Props {
     setIsSearchBarFocused?: (isSearchBarFocused: boolean) => void;
     onFocus?: () => void;
     onBlur?: () => void;
+    showViewAllResults?: boolean;
 }
 
 const defaultProps = {
@@ -146,6 +147,7 @@ export const SearchBar = ({
     setIsSearchBarFocused,
     onFocus,
     onBlur,
+    showViewAllResults = false,
 }: Props) => {
     const history = useHistory();
     const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
@@ -203,7 +205,7 @@ export const SearchBar = ({
     const { quickFilters, selectedQuickFilter, setSelectedQuickFilter } = useQuickFiltersContext();
 
     const autoCompleteQueryOptions = useMemo(() => {
-        if (effectiveQuery === '') return [];
+        if (effectiveQuery === '' || !showViewAllResults) return [];
 
         return [
             {
@@ -212,7 +214,7 @@ export const SearchBar = ({
                 type: EXACT_AUTOCOMPLETE_OPTION_TYPE,
             },
         ];
-    }, [effectiveQuery]);
+    }, [effectiveQuery, showViewAllResults]);
 
     const autoCompleteEntityOptions = useMemo(() => {
         return suggestions.map((suggestion: AutoCompleteResultForEntity) => {
diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx
index 74bc562e275d1..91f9753a3d601 100644
--- a/datahub-web-react/src/app/search/SearchHeader.tsx
+++ b/datahub-web-react/src/app/search/SearchHeader.tsx
@@ -107,6 +107,7 @@ export const SearchHeader = ({
                     combineSiblings
                     fixAutoComplete
                     showQuickFilters
+                    showViewAllResults
                 />
             </LogoSearchContainer>
             <NavGroup>
diff --git a/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts b/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
index 05419e5abed35..9a3af8fb8d56c 100644
--- a/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
+++ b/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
@@ -3,7 +3,7 @@ import { useLocation, useParams } from 'react-router';
 import { useMemo } from 'react';
 import { FacetFilterInput, EntityType } from '../../types.generated';
 import { useEntityRegistry } from '../useEntityRegistry';
-import { ENTITY_FILTER_NAME, FILTER_DELIMITER, UnionType } from './utils/constants';
+import { ENTITY_FILTER_NAME, UnionType } from './utils/constants';
 import { useUserContext } from '../context/useUserContext';
 import useFilters from './utils/useFilters';
 import { generateOrFilters } from './utils/generateOrFilters';
@@ -27,12 +27,6 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     const sortInput = useSortInput();
 
     const filters: Array<FacetFilterInput> = useFilters(params);
-    const nonNestedFilters = filters.filter(
-        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
-    const nestedFilters = filters.filter(
-        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
     const entityFilters: Array<EntityType> = useMemo(
         () =>
             filters
@@ -43,8 +37,8 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     );
 
     const orFilters = useMemo(
-        () => generateOrFilters(unionType, nonNestedFilters, nestedFilters),
-        [nonNestedFilters, nestedFilters, unionType],
+        () => generateOrFilters(unionType, filters, excludedFilterFields),
+        [filters, excludedFilterFields, unionType],
     );
 
     return { entityFilters, query, unionType, filters, orFilters, viewUrn, page, activeType, sortInput };
diff --git a/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts b/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
index 505c50efb289f..fd5a5691b454e 100644
--- a/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
+++ b/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
@@ -1,7 +1,7 @@
 import {
     DOMAINS_FILTER_NAME,
     ENTITY_SUB_TYPE_FILTER_NAME,
-    ENTITY_TYPE_FILTER_NAME,
+    ENTITY_FILTER_NAME,
     TAGS_FILTER_NAME,
     UnionType,
 } from '../constants';
@@ -10,7 +10,7 @@ import { generateOrFilters } from '../generateOrFilters';
 describe('generateOrFilters', () => {
     it('should generate orFilters with UnionType.AND', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.AND, filters);
@@ -24,7 +24,7 @@ describe('generateOrFilters', () => {
 
     it('should generate orFilters with UnionType.OR', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.OR, filters);
@@ -43,17 +43,23 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.AND, filters, nestedFilters);
+        // const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
+        const orFilters = generateOrFilters(UnionType.AND, filters);
 
         expect(orFilters).toMatchObject([
             {
-                and: [...filters, { field: '_entityType', values: ['CONTAINER'] }],
+                and: [
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+                    { field: '_entityType', values: ['CONTAINER'] },
+                ],
             },
             {
                 and: [
-                    ...filters,
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
                     { field: '_entityType', values: ['DATASET'] },
                     { field: 'typeNames', values: ['table'] },
                 ],
@@ -65,9 +71,9 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.OR, filters, nestedFilters);
+        const orFilters = generateOrFilters(UnionType.OR, filters);
 
         expect(orFilters).toMatchObject([
             {
@@ -87,4 +93,18 @@ describe('generateOrFilters', () => {
             },
         ]);
     });
+
+    it('should generate orFilters and exclude filters with a provided exclude field', () => {
+        const filters = [
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+        ];
+        const orFilters = generateOrFilters(UnionType.AND, filters, [ENTITY_FILTER_NAME]);
+
+        expect(orFilters).toMatchObject([
+            {
+                and: [{ field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] }],
+            },
+        ]);
+    });
 });
diff --git a/datahub-web-react/src/app/search/utils/generateOrFilters.ts b/datahub-web-react/src/app/search/utils/generateOrFilters.ts
index b665a2e0f0495..fa2939b3436f5 100644
--- a/datahub-web-react/src/app/search/utils/generateOrFilters.ts
+++ b/datahub-web-react/src/app/search/utils/generateOrFilters.ts
@@ -26,20 +26,26 @@ function generateInputWithNestedFilters(filters: FacetFilterInput[], nestedFilte
 export function generateOrFilters(
     unionType: UnionType,
     filters: FacetFilterInput[],
-    nestedFilters: FacetFilterInput[] = [],
+    excludedFilterFields: string[] = [],
 ): AndFilterInput[] {
-    if ((filters?.length || 0) === 0 && nestedFilters.length === 0) {
+    if ((filters?.length || 0) === 0) {
         return [];
     }
+    const nonNestedFilters = filters.filter(
+        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
+    const nestedFilters = filters.filter(
+        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
 
     if (unionType === UnionType.OR) {
         const orFiltersWithNestedFilters = generateInputWithNestedFilters([], nestedFilters);
-        const orFilters = filters.map((filter) => ({
+        const orFilters = nonNestedFilters.map((filter) => ({
             and: [filter],
         }));
         return [...orFilters, ...orFiltersWithNestedFilters];
     }
-    const andFiltersWithNestedFilters = generateInputWithNestedFilters(filters, nestedFilters);
+    const andFiltersWithNestedFilters = generateInputWithNestedFilters(nonNestedFilters, nestedFilters);
 
     if (andFiltersWithNestedFilters.length) {
         return andFiltersWithNestedFilters;
@@ -47,7 +53,7 @@ export function generateOrFilters(
 
     return [
         {
-            and: filters,
+            and: nonNestedFilters,
         },
     ];
 }
diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh
index 6f977513fd919..439ffb4d4d829 100755
--- a/docker/kafka-setup/kafka-setup.sh
+++ b/docker/kafka-setup/kafka-setup.sh
@@ -36,7 +36,9 @@ if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SSL" ]]; then
     fi
     if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION ]]; then
         echo "ssl.truststore.location=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH
-        echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH
+        if [[ $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE != "PEM" ]]; then
+          echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH
+        fi
         if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE ]]; then
             echo "ssl.truststore.type=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH
         fi
diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
index 68ea1ebffa6c9..506e263933394 100644
--- a/docs-website/docusaurus.config.js
+++ b/docs-website/docusaurus.config.js
@@ -13,6 +13,13 @@ module.exports = {
   projectName: "datahub", // Usually your repo name.
   staticDirectories: ["static", "genStatic"],
   stylesheets: ["https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap"],
+  scripts: [
+    {
+      src: "https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38",
+      async: true,
+      defer: true,
+    },
+  ],
   noIndex: isSaas,
   customFields: {
     isSaas: isSaas,
@@ -50,44 +57,41 @@ module.exports = {
           position: "right",
         },
         {
-          to: "https://demo.datahubproject.io/",
-          label: "Demo",
-          position: "right",
-        },
-        {
-          href: "https://blog.datahubproject.io/",
-          label: "Blog",
-          position: "right",
-        },
-        {
-          href: "https://feature-requests.datahubproject.io/roadmap",
-          label: "Roadmap",
+          type: "dropdown",
+          label: "Resources",
           position: "right",
+          items: [
+            {
+              href: "https://demo.datahubproject.io/",
+              label: "Demo",
+            },
+            {
+              href: "https://blog.datahubproject.io/",
+              label: "Blog",
+            },
+            {
+              href: "https://feature-requests.datahubproject.io/roadmap",
+              label: "Roadmap",
+            },
+            {
+              href: "https://slack.datahubproject.io",
+              label: "Slack",
+            },
+            {
+              href: "https://github.com/datahub-project/datahub",
+              label: "GitHub",
+            },
+            {
+              href: "https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w",
+              label: "YouTube",
+            },
+          ],
         },
         {
           type: "docsVersionDropdown",
-          position: "right",
+          position: "left",
           dropdownActiveClassDisabled: true,
         },
-        {
-          href: "https://slack.datahubproject.io",
-          "aria-label": "Slack",
-          position: "right",
-          className: "item__icon item__slack",
-        },
-        {
-          href: "https://github.com/datahub-project/datahub",
-          "aria-label": "GitHub",
-          position: "right",
-          className: "item__icon item__github",
-        },
-
-        {
-          href: "https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w",
-          "aria-label": "YouTube",
-          position: "right",
-          className: "item__icon item__youtube",
-        },
       ],
     },
     footer: {
diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index b2b3df4dfb33c..39eaea57444ed 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -442,10 +442,29 @@ module.exports = {
         },
         "docs/act-on-metadata/impact-analysis",
         {
-          Observability: [
-            "docs/managed-datahub/observe/freshness-assertions",
-            "docs/managed-datahub/observe/volume-assertions",
-            "docs/managed-datahub/observe/custom-sql-assertions",
+          label: "Observability",
+          type: "category",
+          items: [
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/freshness-assertions",
+              className: "saasOnly",
+            },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/volume-assertions",
+              className: "saasOnly",
+            },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/custom-sql-assertions",
+              className: "saasOnly",
+            },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/column-assertions",
+              className: "saasOnly",
+            },
           ],
         },
         {
@@ -605,7 +624,6 @@ module.exports = {
         {
           type: "doc",
           id: "docs/managed-datahub/chrome-extension",
-          className: "saasOnly",
         },
         {
           "Managed DataHub Release History": [
diff --git a/docs-website/src/styles/global.scss b/docs-website/src/styles/global.scss
index 55a54876b41ac..16e3893ed08b7 100644
--- a/docs-website/src/styles/global.scss
+++ b/docs-website/src/styles/global.scss
@@ -144,20 +144,29 @@ div[class^="announcementBar"] {
 
 /** Navbar */
 
-@media only screen and (max-width: 1050px) {
-  .navbar__toggle {
-    display: inherit;
-  }
-  .navbar__item {
-    display: none;
-  }
-}
-
 .navbar {
   .navbar__logo {
     height: 3rem;
   }
+
+  .navbar__link {
+    align-items: center;
+    margin: 0 1rem 0;
+    padding: 0;
+    border-bottom: 2px solid transparent;
+  }
+
+  .dropdown > .navbar__link:after {
+    top: -1px;
+    border-width: 0.3em 0.3em 0;
+    margin-left: 0.4em;
+  }
+
+  .navbar__link--active {
+    border-bottom-color: var(--ifm-navbar-link-hover-color);
+  }
   .navbar__item {
+    padding: 0.25rem 0;
     svg[class*="iconExternalLink"] {
       display: none;
     }
diff --git a/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js b/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js
index cc04ab23d3cf3..661d64392e67f 100644
--- a/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js
+++ b/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js
@@ -6,6 +6,9 @@ import { translate } from "@docusaurus/Translate";
 import { useLocation } from "@docusaurus/router";
 import DefaultNavbarItem from "@theme/NavbarItem/DefaultNavbarItem";
 import DropdownNavbarItem from "@theme/NavbarItem/DropdownNavbarItem";
+
+import styles from "./styles.module.scss";
+
 const getVersionMainDoc = (version) => version.docs.find((doc) => doc.id === version.mainDocId);
 export default function DocsVersionDropdownNavbarItem({
   mobile,
@@ -60,6 +63,7 @@ export default function DocsVersionDropdownNavbarItem({
   return (
     <DropdownNavbarItem
       {...props}
+      className={styles.versionNavItem}
       mobile={mobile}
       label={dropdownLabel}
       to={false} // This component is Swizzled to disable the link here
diff --git a/docs-website/src/theme/NavbarItem/styles.module.scss b/docs-website/src/theme/NavbarItem/styles.module.scss
new file mode 100644
index 0000000000000..d533115a9c9f0
--- /dev/null
+++ b/docs-website/src/theme/NavbarItem/styles.module.scss
@@ -0,0 +1,8 @@
+.versionNavItem {
+  margin-left: 0 !important;
+  padding: 0.2em 1em !important;
+  display: block;
+  border-radius: var(--ifm-button-border-radius) !important;
+  color: var(--ifm-menu-color-active);
+  background: var(--ifm-menu-color-background-active);
+}
diff --git a/docs-website/versions.json b/docs-website/versions.json
index a5493c26a4c65..a66607b67ddd5 100644
--- a/docs-website/versions.json
+++ b/docs-website/versions.json
@@ -1,4 +1,5 @@
 [
+  "0.12.0",
   "0.11.0",
   "0.10.5"
 ]
diff --git a/docs/api/tutorials/lineage.md b/docs/api/tutorials/lineage.md
index 4baad09099d07..13ec716b7870b 100644
--- a/docs/api/tutorials/lineage.md
+++ b/docs/api/tutorials/lineage.md
@@ -113,12 +113,10 @@ Expected Response:
 
 You can now see the lineage between `fct_users_deleted` and `logging_events`.
 
-
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/apis/tutorials/lineage-added.png"/>
 </p>
 
-
 ## Add Column-level Lineage
 
 <Tabs>
@@ -135,12 +133,10 @@ You can now see the lineage between `fct_users_deleted` and `logging_events`.
 
 You can now see the column-level lineage between datasets. Note that you have to enable `Show Columns` to be able to see the column-level lineage.
 
-
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/apis/tutorials/column-level-lineage-added.png"/>
 </p>
 
-
 ## Read Lineage
 
 <Tabs>
@@ -180,7 +176,7 @@ query searchAcrossLineage {
 }
 ```
 
-This example shows using lineage degrees as a filter, but additional search filters can be included here as well. 
+This example shows using lineage degrees as a filter, but additional search filters can be included here as well.
 
 </TabItem>
 <TabItem value="curl" label="Curl">
@@ -188,7 +184,7 @@ This example shows using lineage degrees as a filter, but additional search filt
 ```shell
 curl --location --request POST 'http://localhost:8080/api/graphql' \
 --header 'Authorization: Bearer <my-access-token>' \
---header 'Content-Type: application/json'  --data-raw '{ { "query": "mutation searchAcrossLineage { searchAcrossLineage( input: { query: \"*\" urn: \"urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.adoption.human_profiles,PROD)\" start: 0 count: 10 direction: DOWNSTREAM orFilters: [ { and: [ { condition: EQUAL negated: false field: \"degree\" values: [\"1\", \"2\", \"3+\"] } ] } ] } ) { searchResults { degree entity { urn type } } }}"
+--header 'Content-Type: application/json'  --data-raw '{ { "query": "query searchAcrossLineage { searchAcrossLineage( input: { query: \"*\" urn: \"urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.adoption.human_profiles,PROD)\" start: 0 count: 10 direction: DOWNSTREAM orFilters: [ { and: [ { condition: EQUAL negated: false field: \"degree\" values: [\"1\", \"2\", \"3+\"] } ] } ] } ) { searchResults { degree entity { urn type } } }}"
 }}'
 ```
 
diff --git a/docs/authorization/policies.md b/docs/authorization/policies.md
index e3606f2a3e48d..63aa6688d3eec 100644
--- a/docs/authorization/policies.md
+++ b/docs/authorization/policies.md
@@ -137,7 +137,7 @@ We currently support the following:
 #### Resources
 
 Resource filter defines the set of resources that the policy applies to is defined using a list of criteria. Each
-criterion defines a field type (like resource_type, resource_urn, domain), a list of field values to compare, and a
+criterion defines a field type (like type, urn, domain), a list of field values to compare, and a
 condition (like EQUALS). It essentially checks whether the field of a certain resource matches any of the input values.
 Note, that if there are no criteria or resource is not set, policy is applied to ALL resources.
 
@@ -149,7 +149,7 @@ For example, the following resource filter will apply the policy to datasets, ch
       "filter": {
         "criteria": [
           {
-            "field": "RESOURCE_TYPE",
+            "field": "TYPE",
             "condition": "EQUALS",
             "values": [
               "dataset",
@@ -175,8 +175,8 @@ Supported fields are as follows
 
 | Field Type    | Description            | Example                 |
 |---------------|------------------------|-------------------------|
-| resource_type | Type of the resource   | dataset, chart, dataJob |
-| resource_urn  | Urn of the resource    | urn:li:dataset:...      |
+| type | Type of the resource   | dataset, chart, dataJob |
+| urn  | Urn of the resource    | urn:li:dataset:...      |
 | domain        | Domain of the resource | urn:li:domain:domainX   |
 
 ## Managing Policies
diff --git a/docs/dev-guides/timeline.md b/docs/dev-guides/timeline.md
index 829aef1d3eefa..6a8e158d40ebf 100644
--- a/docs/dev-guides/timeline.md
+++ b/docs/dev-guides/timeline.md
@@ -228,7 +228,7 @@ http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3Adat
 	REMOVE GLOSSARY_TERM dataset:hive:testTimelineDataset (urn:li:glossaryTerm:SavingsAccount): The GlossaryTerm 'SavingsAccount' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed.
 ```
 
-# Explore the API
+## Explore the API
 
 The API is browse-able via the UI through through the dropdown.
 Here are a few screenshots showing how to navigate to it. You can try out the API and send example requests.
@@ -243,7 +243,7 @@ Here are a few screenshots showing how to navigate to it. You can try out the AP
 </p>
 
 
-# Future Work
+## Future Work
 
 - Supporting versions as start and end parameters as part of the call to the timeline API
 - Supporting entities beyond Datasets
diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 3af3b2bdda215..4d1535f28fa0a 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -4,6 +4,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ## Next
 
+- #9010 - In Redshift source's config `incremental_lineage` is set default to off.
+
 ### Breaking Changes
 
 - #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
@@ -11,11 +13,50 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
   by Looker and LookML source connectors.
 - #8853 - The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7. See the docs for more details.
 - #8853 - Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include `pip install 'acryl-datahub-airflow-plugin[plugin-v2]'`. To continue using the v1 plugin, set the `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN` environment variable to `true`.
-- #8943 The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
+- #8943 - The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
 This is currently enabled by default to preserve compatibility, but will be disabled by default and then removed in the future.
 If stateful ingestion is enabled, simply setting `include_metastore: false` will perform all required cleanup.
 Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
 `datahub delete --platform databricks --soft` and then reingesting with `include_metastore: false`.
+- #8846 - Changed enum values in resource filters used by policies. `RESOURCE_TYPE` became `TYPE` and `RESOURCE_URN` became `URN`.
+Any existing policies using these filters (i.e. defined for particular `urns` or `types` such as `dataset`) need to be upgraded
+manually, for example by retrieving their respective `dataHubPolicyInfo` aspect and changing part using filter i.e.
+```yaml
+   "resources": {
+     "filter": {
+       "criteria": [
+         {
+           "field": "RESOURCE_TYPE",
+           "condition": "EQUALS",
+           "values": [
+             "dataset"
+           ]
+         }
+       ]
+     }
+```
+into
+```yaml
+   "resources": {
+     "filter": {
+       "criteria": [
+         {
+           "field": "TYPE",
+           "condition": "EQUALS",
+           "values": [
+             "dataset"
+           ]
+         }
+       ]
+     }
+```
+for example, using `datahub put` command. Policies can be also removed and re-created via UI.
+- #9077 - The BigQuery ingestion source by default sets `match_fully_qualified_names: true`.
+This means that any `dataset_pattern` or `schema_pattern` specified will be matched on the fully
+qualified dataset name, i.e. `<project_name>.<dataset_name>`. We attempt to support the old
+pattern format by prepending `.*\\.` to dataset patterns lacking a period, so in most cases this
+should not cause any issues. However, if you have a complex dataset pattern, we recommend you
+manually convert it to the fully qualified format to avoid any potential issues.
 
 ### Potential Downtime
 
diff --git a/docs/managed-datahub/chrome-extension.md b/docs/managed-datahub/chrome-extension.md
index 0aa0860d03b67..a4560bc8cc09b 100644
--- a/docs/managed-datahub/chrome-extension.md
+++ b/docs/managed-datahub/chrome-extension.md
@@ -1,10 +1,8 @@
 ---
 description: Learn how to upload and use the Acryl DataHub Chrome extension (beta) locally before it's available on the Chrome store.
 ---
-import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
 # Acryl DataHub Chrome Extension
-<FeatureAvailability saasOnly />
 
 ## Installing the Extension
 
diff --git a/docs/managed-datahub/observe/column-assertions.md b/docs/managed-datahub/observe/column-assertions.md
new file mode 100644
index 0000000000000..99a764f771676
--- /dev/null
+++ b/docs/managed-datahub/observe/column-assertions.md
@@ -0,0 +1,358 @@
+---
+description: This page provides an overview of working with DataHub Column Assertions
+---
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+
+# Column Assertions
+
+<FeatureAvailability saasOnly />
+
+> ⚠️ The **Column Assertions** feature is currently in private beta, part of the **Acryl Observe** module, and may only 
+> be available to a limited set of design partners.
+>
+> If you are interested in trying it and providing feedback, please reach out to your Acryl Customer Success
+> representative.
+
+## Introduction
+
+Can you remember a time when an important warehouse table column changed dramatically, with little or no notice? Perhaps the number of null values suddenly spiked, or a new value was added to a fixed set of possible values. If the answer is yes, how did you initially find out? We'll take a guess - someone looking at an internal reporting dashboard or worse, a user using your your product, sounded an alarm when a number looked a bit out of the ordinary.
+
+There are many reasons why important columns in your Snowflake, Redshift, or BigQuery tables may change - application code bugs, new feature rollouts, etc. Oftentimes, these changes break important assumptions made about the data used in building key downstream data products like reporting dashboards or data-driven product features.
+
+What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data issues before anyone else? With Acryl DataHub Column  Assertions, you can.
+
+With Acryl DataHub, you can define **Column Value** assertions to ensure each value in a column matches specific constraints, and **Column Metric** assertions to ensure that computed metrics from columns align with your expectations. As soon as things go wrong, your team will be the first to know, before the data issue becomes a larger data incident. 
+
+In this guide, we'll cover the basics of Column Assertions - what they are, how to configure them, and more - so that you and your team can start building trust in your most important data assets.
+
+Let's dive in!
+
+## Support
+
+Column Assertions are currently supported for:
+
+1. Snowflake
+2. Redshift
+3. BigQuery
+
+Note that an Ingestion Source _must_ be configured with the data platform of your choice in
+Acryl DataHub's **Ingestion** tab.
+
+> Note that Column Assertions are not yet supported if you are connecting to your warehouse
+> using the DataHub CLI or a Remote Ingestion Executor.
+
+## What is a Column Assertion?
+
+A **Column Assertion** is a highly configurable Data Quality rule used to monitor specific columns of a Data Warehouse table for unexpected changes. 
+
+Column Assertions are defined to validate a specific column, and can be used to 
+
+1. Validate that the values of the column match some constraints (regex, allowed values, max, min, etc) across rows OR
+2. Validate that specific column aggregation metrics match some expectations across rows.
+
+Column Assertions can be particularly useful for documenting and enforcing column-level "contracts", i.e. formal specifications about the expected contents of a particular column that can be used for coordinating among producers and consumers of the data.  
+
+### Anatomy of Column Assertion
+
+Column Assertions can be divided into two main types: **Column Value** and **Column Metric** Assertions.
+
+A **Column Value Assertion** is used to monitor the value of a specific column in a table, and ensure that every row
+adheres to a specific condition. In comparison, a **Column Metric Assertion** is used to compute a metric for that column,
+and ensure that the value of that metric adheres to a specific condition.
+
+At the most basic level, both types consist of a few important parts:
+
+1. An **Evaluation Schedule**
+2. A **Column Selection**
+3. A **Evaluation Criteria**
+4. A **Row Evaluation Type**
+
+In this section, we'll give an overview of each.
+
+#### 1. Evaluation Schedule
+
+The **Evaluation Schedule**: This defines how often to evaluate the Column Assertion against the given warehouse table.
+This should usually be configured to match the expected change frequency of the table, although it can also be less
+frequently depending on your requirements. You can also specify specific days of the week, hours in the day, or even
+minutes in an hour.
+
+#### 2. Column Selection
+
+The **Column Selection**: This defines the column that should be monitored by the Column Assertion. You can choose from
+any of the columns from the table listed in the dropdown. Note that columns of struct / object type are not currently supported.
+
+#### 3. Evaluation Criteria
+
+The **Evaluation Criteria**: This defines the condition that must be satisfied in order for the Column
+Assertion to pass.
+
+For **Column Value Assertions**, you will be able to choose from a set of operators that can be applied to the column
+value. The options presented will vary based on the data type of the selected column. For example, if you've selected a numeric column, you
+can verify that the column value is greater than a particular value. For string types, you can check that the column value
+matches a particular regex pattern. Additionally, you are able to control the behavior of the check in the presence of NULL values. If the
+**Allow Nulls** option is _disabled_, then any null values encountered will be reported as a failure when evaluating the
+assertion. If **Allow Nulls** is enabled, then nulls will be ignored; the condition will be evaluated for rows where the column value is non-null.  
+
+For **Column Metric Assertions**, you will be able to choose from a list of common column metrics - MAX, MIN, MEAN, NULL COUNT, etc - and then compare these metric values to an expected value. The list of metrics will vary based on the type of the selected column. For example
+if you've selected a numeric column, you can choose to compute the MEAN value of the column, and then assert that it is greater than a
+specific number. For string types, you can choose to compute the MAX LENGTH of the string across all column values, and then assert that it
+is less than a specific number.
+
+#### 4. Row Selection Set
+
+The **Row Selection Set**: This defines which rows in the table the Column Assertion will be evaluated across. You can choose
+from the following options:
+
+- **All Table Rows**: Evaluate the Column Assertion across all rows in the table. This is the default option. Note that
+this may not be desirable for large tables.
+
+- **Only Rows That Have Changed**: Evaluate the Column Assertion only against rows that have changed since the last
+evaluation of the assertion. If you choose this option, you will need to specify a **High Watermark Column** to help determine which rows
+have changed. A **High Watermark Column** is a column that contains a constantly incrementing value - a date, a time, or
+another always-increasing number - that can be used to find the "new rows" that were added since previous evaluation. When selected, a query will be issued to the table to find only the rows that have changed since the previous assertion evaluation. 
+
+## Creating a Column Assertion
+
+### Prerequisites
+
+1. **Permissions**: To create or delete Column Assertions for a specific entity on DataHub, you'll need to be granted the
+   `Edit Assertions` and `Edit Monitors` privileges for the entity. This is granted to Entity owners by default.
+
+2. **Data Platform Connection**: In order to create a Column Assertion, you'll need to have an **Ingestion Source** 
+   configured to your Data Platform: Snowflake, BigQuery, or Redshift under the **Ingestion** tab.
+
+Once these are in place, you're ready to create your Column Assertions!
+
+### Steps
+
+1. Navigate to the Table that you want to monitor
+2. Click the **Validations** tab
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/profile-validation-tab.png"/>
+</p>
+
+3. Click **+ Create Assertion**
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-choose-type.png"/>
+</p>
+
+4. Choose **Column**
+
+5. Configure the evaluation **schedule**. This is the frequency at which the assertion will be evaluated to produce a
+   pass or fail result, and the times when the column values will be checked.
+
+6. Configure the **column assertion type**. You can choose from **Column Value** or **Column Metric**.
+   **Column Value** assertions are used to monitor the value of a specific column in a table, and ensure that every row
+   adheres to a specific condition. **Column Metric** assertions are used to compute a metric for that column, and then compare the value of that metric to your expectations.
+
+<p align="center">
+  <img width="45%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-assertion-type.png"/>
+</p>
+
+7. Configure the **column selection**. This defines the column that should be monitored by the Column Assertion. 
+   You can choose from any of the columns from the table listed in the dropdown.
+
+<p align="center">
+  <img width="45%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-field-selection.png"/>
+</p>
+
+8. Configure the **evaluation criteria**. This step varies based on the type of assertion you chose in the previous step.
+
+   - **Column Value Assertions**: You will be able to choose from a set of operators that can be applied to the column
+     value. The options presented will vary based on the data type of the selected column. For example with numeric types, you
+     can check that the column value is greater than a specific value. For string types, you can check that the column value
+     matches a particular regex pattern. You will also be able to control the behavior of null values in the column. If the
+     **Allow Nulls** option is _disabled_, any null values encountered will be reported as a failure when evaluating the
+     assertion.
+
+   - **Column Metric Assertions**: You will be able to choose from a list of common metrics and then specify the operator
+     and value to compare against. The list of metrics will vary based on the data type of the selected column. For example
+     with numeric types, you can choose to compute the average value of the column, and then assert that it is greater than a
+     specific number. For string types, you can choose to compute the max length of all column values, and then assert that it
+     is less than a specific number.
+
+9. Configure the **row evaluation type**. This defines which rows in the table the Column Assertion should evaluate. You can choose
+   from the following options:
+
+   - **All Table Rows**: Evaluate the Column Assertion against all rows in the table. This is the default option. Note that
+     this may not be desirable for large tables.
+
+   - **Only Rows That Have Changed**: Evaluate the Column Assertion only against rows that have changed since the last
+     evaluation. If you choose this option, you will need to specify a **High Watermark Column** to help determine which rows
+     have changed. A **High Watermark Column** is a column that contains a constantly-incrementing value - a date, a time, or
+     another always-increasing number. When selected, a query will be issued to the table find only the rows which have changed since the last assertion run. 
+
+<p align="center">
+  <img width="60%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-row-evaluation-type.png"/>
+</p>
+
+10. (Optional) Click **Advanced** to further customize the Column Assertion. The options listed here will vary based on the
+    type of assertion you chose in the previous step.
+
+    - **Invalid Values Threshold**: For **Column Value** assertions, you can configure the number of invalid values
+      (i.e. rows) that are allowed to fail before the assertion is marked as failing. This is useful if you want to allow a limited number
+      of invalid values in the column. By default this is 0, meaning the assertion will fail if any rows have an invalid column value.
+
+    - **Source**: For **Column Metric** assertions, you can choose the mechanism that will be used to obtain the column
+      metric. **Query** will issue a query to the dataset to compute the metric. **DataHub Dataset Profile** will use the
+      DataHub Dataset Profile metadata to compute the metric. Note that this option requires that dataset profiling
+      statistics are up-to-date as of the assertion run time.
+
+    - **Additional Filters**: You can choose to add additional filters to the query that will be used to evaluate the
+      assertion. This is useful if you want to limit the assertion to a subset of rows in the table. Note this option will not
+      be available if you choose **DataHub Dataset Profile** as the **source**.
+
+11. Click **Next**
+12. Configure actions that should be taken when the Column Assertion passes or fails
+
+<p align="left">
+  <img width="45%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-builder-actions.png"/>
+</p>
+
+- **Raise incident**: Automatically raise a new DataHub `Column` Incident for the Table whenever the Column Assertion is failing. This
+  may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when
+  an incident is created due to an Assertion failure.
+- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Column Assertion. Note that
+  any other incidents will not be impacted.
+
+10. Click **Save**.
+
+And that's it! DataHub will now begin to monitor your Column Assertion for the table.
+
+To view the time of the next Column Assertion evaluation, simply click **Column** and then click on your
+new Assertion:
+
+<p align="center">
+  <img width="40%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-next-evaluation-time.png"/>
+</p>
+
+Once your assertion has run, you will begin to see Success or Failure status for the Table
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/profile-passing-column-assertions-expanded.png"/>
+</p>
+
+## Stopping a Column Assertion
+
+In order to temporarily stop the evaluation of a Column Assertion:
+
+1. Navigate to the **Validations** tab of the table with the assertion
+2. Click **Column** to open the Column Assertions list
+3. Click the three-dot menu on the right side of the assertion you want to disable
+4. Click **Stop**
+
+<p align="left">
+  <img width="25%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/manage-assertion-menu.png"/>
+</p>
+
+To resume the Column Assertion, simply click **Turn On**.
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/stopped-assertion.png"/>
+</p>
+
+## Creating Column Assertions via API
+
+Under the hood, Acryl DataHub implements Column Assertion Monitoring using two "entity" concepts:
+
+- **Assertion**: The specific expectation for the column metric. e.g. "The value of an integer column is greater than 10 for all rows in the table." This is the "what".
+
+- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific
+  mechanisms. This is the "how".
+
+Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the
+`Edit Assertions` and `Edit Monitors` privileges for it.
+
+#### GraphQL
+
+In order to create a Column Assertion that is being monitored on a specific **Evaluation Schedule**, you'll need to use 2
+GraphQL mutation queries to create a Column Assertion entity and create an Assertion Monitor entity responsible for evaluating it.
+
+Start by creating the Column Assertion entity using the `createFieldAssertion` query and hang on to the 'urn' field of the Assertion entity
+you get back. Then continue by creating a Monitor entity using the `createAssertionMonitor`.
+
+##### Examples
+
+To create a Column Assertion Entity that checks that the value of an integer column is greater than 10:
+
+```json
+mutation createFieldAssertion {
+  createFieldAssertion(
+  input: {
+    entityUrn: "<urn of the table to be monitored>",
+    type: FIELD_VALUES,
+    fieldValuesAssertion: {
+      field: {
+        path: "<name of the column to be monitored>",
+        type: "NUMBER",
+        nativeType: "NUMBER(38,0)"
+      },
+      operator: GREATER_THAN,
+      parameters: {
+        value: {
+          type: NUMBER,
+          value: "10"
+        }
+      },
+      failThreshold: {
+        type: COUNT,
+        value: 0
+      },
+      excludeNulls: true
+    }
+  }
+  ) {
+  urn
+}
+}
+```
+
+To create an Assertion Monitor Entity that evaluates the column assertion every 8 hours using all rows in the table:
+
+```json
+mutation createAssertionMonitor {
+  createAssertionMonitor(
+    input: {
+      entityUrn: "<urn of entity being monitored>",
+      assertionUrn: "<urn of assertion created in first query>",
+      schedule: {
+        cron: "0 */8 * * *",
+        timezone: "America/Los_Angeles"
+      },
+      parameters: {
+        type: DATASET_FIELD,
+        datasetFieldParameters: {
+          sourceType: ALL_ROWS_QUERY
+        }
+      }
+    }
+  ) {
+    urn
+  }
+}
+```
+
+This entity defines _when_ to run the check (Using CRON format - every 8th hour) and _how_ to run the check (using a query against all rows of the table).
+
+After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone.
+
+You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`.
+
+### Tips
+
+:::info
+**Authorization**
+
+Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows:
+
+```
+Authorization: Bearer <personal-access-token>
+```
+
+**Exploring GraphQL API**
+
+Also, remember that you can play with an interactive version of the Acryl GraphQL API at `https://your-account-id.acryl.io/api/graphiql`
+:::
diff --git a/gradle/versioning/versioning.gradle b/gradle/versioning/versioning.gradle
index 1fac894d165a8..39a8a3faf8011 100644
--- a/gradle/versioning/versioning.gradle
+++ b/gradle/versioning/versioning.gradle
@@ -21,7 +21,7 @@ Produces the following variables and supports token replacement
 import org.apache.tools.ant.filters.ReplaceTokens
 
 def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def cliMajorVersion = "0.10.5" // base default cli major version
+def cliMajorVersion = "0.12.0" // base default cli major version
 def snapshotVersion = false
 if (project.hasProperty("releaseVersion")) {
   version = releaseVersion
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
index 46763f29a7040..1258d958f2092 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
@@ -4,6 +4,19 @@
  * List of entity field types to fetch for a given entity
  */
 public enum EntityFieldType {
+
+  /**
+   * Type of the entity (e.g. dataset, chart)
+   * @deprecated
+   */
+  @Deprecated
+  RESOURCE_URN,
+  /**
+   * Urn of the entity
+   * @deprecated
+   */
+  @Deprecated
+  RESOURCE_TYPE,
   /**
    * Type of the entity (e.g. dataset, chart)
    */
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
index 16585f70e820b..e1d53be7bae6b 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
@@ -98,7 +98,7 @@ def _get_dependencies(
         # It is possible to tie an external sensor to DAG if external_task_id is omitted but currently we can't tie
         # jobflow to anothet jobflow.
         external_task_upstreams = []
-        if task.task_type == "ExternalTaskSensor":
+        if isinstance(task, ExternalTaskSensor):
             task = cast(ExternalTaskSensor, task)
             if hasattr(task, "external_task_id") and task.external_task_id is not None:
                 external_task_upstreams = [
@@ -155,6 +155,8 @@ def generate_dataflow(
             "_concurrency",
             # "_default_view",
             "catchup",
+            "description",
+            "doc_md",
             "fileloc",
             "is_paused_upon_creation",
             "start_date",
@@ -431,6 +433,9 @@ def run_datajob(
         job_property_bag["operator"] = str(ti.operator)
         job_property_bag["priority_weight"] = str(ti.priority_weight)
         job_property_bag["log_url"] = ti.log_url
+        job_property_bag["orchestrator"] = "airflow"
+        job_property_bag["dag_id"] = str(dag.dag_id)
+        job_property_bag["task_id"] = str(ti.task_id)
         dpi.properties.update(job_property_bag)
         dpi.url = ti.log_url
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
index 046fbb5efaa03..f9a2119f51e32 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
@@ -23,6 +23,7 @@
 
 TASK_ON_FAILURE_CALLBACK = "on_failure_callback"
 TASK_ON_SUCCESS_CALLBACK = "on_success_callback"
+TASK_ON_RETRY_CALLBACK = "on_retry_callback"
 
 
 def get_task_inlets_advanced(task: BaseOperator, context: Any) -> Iterable[Any]:
@@ -259,6 +260,28 @@ def custom_on_success_callback(context):
     return custom_on_success_callback
 
 
+def _wrap_on_retry_callback(on_retry_callback):
+    def custom_on_retry_callback(context):
+        config = get_lineage_config()
+        if config.enabled:
+            context["_datahub_config"] = config
+            try:
+                datahub_task_status_callback(
+                    context, status=InstanceRunResult.UP_FOR_RETRY
+                )
+            except Exception as e:
+                if not config.graceful_exceptions:
+                    raise e
+                else:
+                    print(f"Exception: {traceback.format_exc()}")
+
+        # Call original policy
+        if on_retry_callback:
+            on_retry_callback(context)
+
+    return custom_on_retry_callback
+
+
 def task_policy(task: Union[BaseOperator, MappedOperator]) -> None:
     task.log.debug(f"Setting task policy for Dag: {task.dag_id} Task: {task.task_id}")
     # task.add_inlets(["auto"])
@@ -274,7 +297,14 @@ def task_policy(task: Union[BaseOperator, MappedOperator]) -> None:
         on_success_callback_prop: property = getattr(
             MappedOperator, TASK_ON_SUCCESS_CALLBACK
         )
-        if not on_failure_callback_prop.fset or not on_success_callback_prop.fset:
+        on_retry_callback_prop: property = getattr(
+            MappedOperator, TASK_ON_RETRY_CALLBACK
+        )
+        if (
+            not on_failure_callback_prop.fset
+            or not on_success_callback_prop.fset
+            or not on_retry_callback_prop.fset
+        ):
             task.log.debug(
                 "Using MappedOperator's partial_kwargs instead of callback properties"
             )
@@ -284,10 +314,14 @@ def task_policy(task: Union[BaseOperator, MappedOperator]) -> None:
             task.partial_kwargs[TASK_ON_SUCCESS_CALLBACK] = _wrap_on_success_callback(
                 task.on_success_callback
             )
+            task.partial_kwargs[TASK_ON_RETRY_CALLBACK] = _wrap_on_retry_callback(
+                task.on_retry_callback
+            )
             return
 
     task.on_failure_callback = _wrap_on_failure_callback(task.on_failure_callback)  # type: ignore
     task.on_success_callback = _wrap_on_success_callback(task.on_success_callback)  # type: ignore
+    task.on_retry_callback = _wrap_on_retry_callback(task.on_retry_callback)  # type: ignore
     # task.pre_execute = _wrap_pre_execution(task.pre_execute)
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
index 26aa2afaa831a..a4c17c73e9c7e 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -373,7 +375,10 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets",
+                "orchestrator": "airflow",
+                "dag_id": "basic_iolets",
+                "task_id": "run_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets",
             "name": "basic_iolets_run_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
index b2e3a1fe47da7..a0a95716a0993 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -302,7 +304,10 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "task_1"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag",
             "name": "simple_dag_task_1_manual_run_test",
@@ -433,6 +438,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -654,7 +661,10 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "run_another_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag",
             "name": "simple_dag_run_another_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
index 2e733c2ad40a9..1974f1f085df0 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -224,7 +226,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "basic_iolets",
+                "task_id": "run_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
             "name": "basic_iolets_run_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
index 44b288efda954..d02951bc9e82d 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -224,7 +226,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "basic_iolets",
+                "task_id": "run_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
             "name": "basic_iolets_run_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
index 454c509279e11..9acc47ec1321e 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -189,7 +191,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "task_1"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_task_1_manual_run_test",
@@ -523,7 +528,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "run_another_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_run_another_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
index 73b5765e96b7d..03299c483f57f 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -189,7 +191,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "task_1"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_task_1_manual_run_test",
@@ -435,6 +440,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -579,7 +586,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "run_another_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_run_another_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
index affc395d421da..11a0b17b45b95 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -234,7 +236,10 @@
                 "state": "running",
                 "operator": "SnowflakeOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "snowflake_operator",
+                "task_id": "transform_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1",
             "name": "snowflake_operator_transform_cost_table_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
index 1a32b38ce055d..19e4aac9fb95e 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -74,9 +76,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -98,7 +98,44 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -157,7 +194,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:24.632190+00:00",
+                "start_date": "2023-10-15 20:29:10.262813+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -166,13 +203,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "5",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "create_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_create_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056984632,
+                "time": 1697401750262,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -221,7 +261,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056984632,
+            "timestampMillis": 1697401750262,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -251,9 +291,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -275,7 +313,80 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -331,7 +442,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056984947,
+            "timestampMillis": 1697401750651,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -447,7 +558,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:28.605901+00:00",
+                "start_date": "2023-10-15 20:29:15.013834+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -456,13 +567,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "4",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "populate_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056988605,
+                "time": 1697401755013,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -511,7 +625,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056988605,
+            "timestampMillis": 1697401755013,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -621,7 +735,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056989098,
+            "timestampMillis": 1697401755600,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -807,7 +921,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:32.888165+00:00",
+                "start_date": "2023-10-15 20:29:20.216818+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -816,13 +930,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "3",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "transform_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056992888,
+                "time": 1697401760216,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -895,7 +1012,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056992888,
+            "timestampMillis": 1697401760216,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1131,7 +1248,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056993744,
+            "timestampMillis": 1697401761237,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1249,7 +1366,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:37.745717+00:00",
+                "start_date": "2023-10-15 20:29:26.243934+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1258,13 +1375,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056997745,
+                "time": 1697401766243,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1313,7 +1433,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056997745,
+            "timestampMillis": 1697401766243,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1425,7 +1545,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056998672,
+            "timestampMillis": 1697401767373,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1543,7 +1663,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:42.645806+00:00",
+                "start_date": "2023-10-15 20:29:32.075613+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1552,13 +1672,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_processed_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057002645,
+                "time": 1697401772075,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1607,7 +1730,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057002645,
+            "timestampMillis": 1697401772075,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1719,7 +1842,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057003759,
+            "timestampMillis": 1697401773454,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
index c082be693e30c..b67464b385335 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -74,9 +76,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -98,7 +98,44 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -157,7 +194,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:00:45.832554+00:00",
+                "start_date": "2023-10-15 20:27:26.883178+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -166,13 +203,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "5",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "create_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_create_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057245832,
+                "time": 1697401646883,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -221,7 +261,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057245832,
+            "timestampMillis": 1697401646883,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -251,9 +291,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -275,7 +313,80 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -331,7 +442,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057246734,
+            "timestampMillis": 1697401647826,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -354,6 +465,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -502,7 +615,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:00:49.653938+00:00",
+                "start_date": "2023-10-15 20:27:31.398799+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -511,13 +624,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "4",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "populate_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057249653,
+                "time": 1697401651398,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -566,7 +682,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057249653,
+            "timestampMillis": 1697401651398,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -676,7 +792,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057250831,
+            "timestampMillis": 1697401652651,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -699,6 +815,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -917,7 +1035,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:00:53.989264+00:00",
+                "start_date": "2023-10-15 20:27:37.697995+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -926,13 +1044,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "3",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "transform_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057253989,
+                "time": 1697401657697,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1005,7 +1126,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057253989,
+            "timestampMillis": 1697401657697,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1241,7 +1362,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057255628,
+            "timestampMillis": 1697401659496,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1264,6 +1385,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -1414,7 +1537,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:01:00.421177+00:00",
+                "start_date": "2023-10-15 20:27:45.670215+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1423,13 +1546,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057260421,
+                "time": 1697401665670,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1478,7 +1604,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057260421,
+            "timestampMillis": 1697401665670,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1590,7 +1716,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057262258,
+            "timestampMillis": 1697401667670,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1613,6 +1739,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -1763,7 +1891,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:01:05.540192+00:00",
+                "start_date": "2023-10-15 20:27:51.559194+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1772,13 +1900,16 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_processed_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057265540,
+                "time": 1697401671559,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1827,7 +1958,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057265540,
+            "timestampMillis": 1697401671559,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1939,7 +2070,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057267631,
+            "timestampMillis": 1697401673788,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion/adding-source.md b/metadata-ingestion/adding-source.md
index e4fc950a7cdbd..a0930102c6827 100644
--- a/metadata-ingestion/adding-source.md
+++ b/metadata-ingestion/adding-source.md
@@ -62,7 +62,7 @@ Some sources use the default `SourceReport` class, but others inherit and extend
 
 ### 3. Implement the source itself
 
-The core for the source is the `get_workunits` method, which produces a stream of metadata events (typically MCP objects) wrapped up in a MetadataWorkUnit.
+The core for the source is the `get_workunits_internal` method, which produces a stream of metadata events (typically MCP objects) wrapped up in a MetadataWorkUnit.
 The [file source](./src/datahub/ingestion/source/file.py) is a good and simple example.
 
 The MetadataChangeEventClass is defined in the metadata models which are generated
diff --git a/metadata-ingestion/examples/library/read_lineage_rest.py b/metadata-ingestion/examples/library/read_lineage_rest.py
index 34437ed86280d..bd9b4e8651dba 100644
--- a/metadata-ingestion/examples/library/read_lineage_rest.py
+++ b/metadata-ingestion/examples/library/read_lineage_rest.py
@@ -6,7 +6,7 @@
 
 # Query multiple aspects from entity
 query = """
-mutation searchAcrossLineage {
+query searchAcrossLineage {
   searchAcrossLineage(
     input: {
       query: "*"
diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py
index a9b9b4b20f5ac..021ebd4a31eb3 100644
--- a/metadata-ingestion/scripts/avro_codegen.py
+++ b/metadata-ingestion/scripts/avro_codegen.py
@@ -152,7 +152,8 @@ def add_name(self, name_attr, space_attr, new_schema):
     return encoded
 
 
-autogen_header = """# flake8: noqa
+autogen_header = """# mypy: ignore-errors
+# flake8: noqa
 
 # This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
 # Do not modify manually!
diff --git a/metadata-ingestion/scripts/modeldocgen.py b/metadata-ingestion/scripts/modeldocgen.py
index ffa80515dbafd..81b26145e620c 100644
--- a/metadata-ingestion/scripts/modeldocgen.py
+++ b/metadata-ingestion/scripts/modeldocgen.py
@@ -351,8 +351,8 @@ def strip_types(field_path: str) -> str:
             field_objects = []
             for f in entity_fields:
                 field = avro.schema.Field(
-                    type=f["type"],
-                    name=f["name"],
+                    f["type"],
+                    f["name"],
                     has_default=False,
                 )
                 field_objects.append(field)
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index c46409ecbf52f..7f7826abe2095 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -32,7 +32,7 @@
     "expandvars>=0.6.5",
     "avro-gen3==0.7.11",
     # "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3",
-    "avro>=1.10.2,<1.11",
+    "avro>=1.11.3,<1.12",
     "python-dateutil>=2.8.0",
     "tabulate",
     "progressbar2",
@@ -101,22 +101,36 @@
     "grpcio-tools>=1.44.0,<2",
 }
 
-sql_common = {
-    # Required for all SQL sources.
-    # This is temporary lower bound that we're open to loosening/tightening as requirements show up
-    "sqlalchemy>=1.4.39, <2",
-    # Required for SQL profiling.
-    "great-expectations>=0.15.12, <=0.15.50",
-    # scipy version restricted to reduce backtracking, used by great-expectations,
-    "scipy>=1.7.2",
-    # GE added handling for higher version of jinja2
-    # https://github.com/great-expectations/great_expectations/pull/5382/files
-    # datahub does not depend on traitlets directly but great expectations does.
-    # https://github.com/ipython/traitlets/issues/741
-    "traitlets<5.2.2",
-    "greenlet",
+usage_common = {
+    "sqlparse",
 }
 
+sqlglot_lib = {
+    # Using an Acryl fork of sqlglot.
+    # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
+    "acryl-sqlglot==18.5.2.dev45",
+}
+
+sql_common = (
+    {
+        # Required for all SQL sources.
+        # This is temporary lower bound that we're open to loosening/tightening as requirements show up
+        "sqlalchemy>=1.4.39, <2",
+        # Required for SQL profiling.
+        "great-expectations>=0.15.12, <=0.15.50",
+        # scipy version restricted to reduce backtracking, used by great-expectations,
+        "scipy>=1.7.2",
+        # GE added handling for higher version of jinja2
+        # https://github.com/great-expectations/great_expectations/pull/5382/files
+        # datahub does not depend on traitlets directly but great expectations does.
+        # https://github.com/ipython/traitlets/issues/741
+        "traitlets<5.2.2",
+        "greenlet",
+    }
+    | usage_common
+    | sqlglot_lib
+)
+
 sqllineage_lib = {
     "sqllineage==1.3.8",
     # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage.
@@ -125,12 +139,6 @@
     "sqlparse==0.4.4",
 }
 
-sqlglot_lib = {
-    # Using an Acryl fork of sqlglot.
-    # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
-    "acryl-sqlglot==18.5.2.dev45",
-}
-
 aws_common = {
     # AWS Python SDK
     "boto3",
@@ -243,10 +251,6 @@
 
 powerbi_report_server = {"requests", "requests_ntlm"}
 
-usage_common = {
-    "sqlparse",
-}
-
 databricks = {
     # 0.1.11 appears to have authentication issues with azure databricks
     "databricks-sdk>=0.9.0",
@@ -282,7 +286,8 @@
     # Source plugins
     # sqlalchemy-bigquery is included here since it provides an implementation of
     # a SQLalchemy-conform STRUCT type definition
-    "athena": sql_common | {"PyAthena[SQLAlchemy]>=2.6.0,<3.0.0", "sqlalchemy-bigquery>=1.4.1"},
+    "athena": sql_common
+    | {"PyAthena[SQLAlchemy]>=2.6.0,<3.0.0", "sqlalchemy-bigquery>=1.4.1"},
     "azure-ad": set(),
     "bigquery": sql_common
     | bigquery_common
@@ -354,9 +359,13 @@
     | {"psycopg2-binary", "pymysql>=1.0.2"},
     "pulsar": {"requests"},
     "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib,
-    "redshift": sql_common | redshift_common | usage_common | sqlglot_lib | {"redshift-connector"},
-    "redshift-legacy": sql_common | redshift_common,
-    "redshift-usage-legacy": sql_common | usage_common | redshift_common,
+    "redshift": sql_common
+    | redshift_common
+    | usage_common
+    | {"redshift-connector"}
+    | sqlglot_lib,
+    "redshift-legacy": sql_common | redshift_common | sqlglot_lib,
+    "redshift-usage-legacy": sql_common | redshift_common | sqlglot_lib | usage_common,
     "s3": {*s3_base, *data_lake_profiling},
     "gcs": {*s3_base, *data_lake_profiling},
     "sagemaker": aws_common,
@@ -435,7 +444,9 @@
 deepdiff_dep = "deepdiff"
 test_api_requirements = {pytest_dep, deepdiff_dep, "PyYAML"}
 
-debug_requirements = {"memray"}
+debug_requirements = {
+    "memray",
+}
 
 base_dev_requirements = {
     *base_requirements,
@@ -668,6 +679,7 @@
         "Documentation": "https://datahubproject.io/docs/",
         "Source": "https://github.com/datahub-project/datahub",
         "Changelog": "https://github.com/datahub-project/datahub/releases",
+        "Releases": "https://github.com/acryldata/datahub/releases",
     },
     license="Apache License 2.0",
     description="A CLI to work with DataHub metadata",
diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py
index c909b89eb0c2d..73ac4baac48c0 100644
--- a/metadata-ingestion/src/datahub/configuration/common.py
+++ b/metadata-ingestion/src/datahub/configuration/common.py
@@ -283,7 +283,7 @@ class VersionedConfig(ConfigModel):
 
 class LineageConfig(ConfigModel):
     incremental_lineage: bool = Field(
-        default=True,
+        default=False,
         description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
     )
 
diff --git a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
index 071d590f270f8..cedaa4fbbd7f6 100644
--- a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
@@ -106,6 +106,7 @@ def process_sql_parsing_result(
         user: Optional[UserUrn] = None,
         custom_operation_type: Optional[str] = None,
         include_urns: Optional[Set[DatasetUrn]] = None,
+        include_column_lineage: bool = True,
     ) -> Iterable[MetadataWorkUnit]:
         """Process a single query and yield any generated workunits.
 
@@ -130,7 +131,9 @@ def process_sql_parsing_result(
                 _merge_lineage_data(
                     downstream_urn=downstream_urn,
                     upstream_urns=result.in_tables,
-                    column_lineage=result.column_lineage,
+                    column_lineage=result.column_lineage
+                    if include_column_lineage
+                    else None,
                     upstream_edges=self._lineage_map[downstream_urn],
                     query_timestamp=query_timestamp,
                     is_view_ddl=is_view_ddl,
@@ -179,15 +182,16 @@ def add_lineage(
 
     def gen_workunits(self) -> Iterable[MetadataWorkUnit]:
         if self.generate_lineage:
-            yield from self._gen_lineage_workunits()
+            for mcp in self._gen_lineage_mcps():
+                yield mcp.as_workunit()
         if self.generate_usage_statistics:
             yield from self._gen_usage_statistics_workunits()
 
-    def _gen_lineage_workunits(self) -> Iterable[MetadataWorkUnit]:
+    def _gen_lineage_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
         for downstream_urn in self._lineage_map:
             upstreams: List[UpstreamClass] = []
             fine_upstreams: List[FineGrainedLineageClass] = []
-            for upstream_urn, edge in self._lineage_map[downstream_urn].items():
+            for edge in self._lineage_map[downstream_urn].values():
                 upstreams.append(edge.gen_upstream_aspect())
                 fine_upstreams.extend(edge.gen_fine_grained_lineage_aspects())
 
@@ -201,7 +205,7 @@ def _gen_lineage_workunits(self) -> Iterable[MetadataWorkUnit]:
             )
             yield MetadataChangeProposalWrapper(
                 entityUrn=downstream_urn, aspect=upstream_lineage
-            ).as_workunit()
+            )
 
     def _gen_usage_statistics_workunits(self) -> Iterable[MetadataWorkUnit]:
         yield from self._usage_aggregator.generate_workunits(
diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
index 9478c5cf7efa2..945b201ca5758 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
@@ -130,10 +130,13 @@ def auto_incremental_lineage(
                 if len(wu.metadata.proposedSnapshot.aspects) > 0:
                     yield wu
 
-            yield _lineage_wu_via_read_modify_write(
-                graph, urn, lineage_aspect, wu.metadata.systemMetadata
-            ) if lineage_aspect.fineGrainedLineages else _convert_upstream_lineage_to_patch(
-                urn, lineage_aspect, wu.metadata.systemMetadata
-            )
+            if lineage_aspect.fineGrainedLineages:
+                yield _lineage_wu_via_read_modify_write(
+                    graph, urn, lineage_aspect, wu.metadata.systemMetadata
+                )
+            elif lineage_aspect.upstreams:
+                yield _convert_upstream_lineage_to_patch(
+                    urn, lineage_aspect, wu.metadata.systemMetadata
+                )
         else:
             yield wu
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py
index b86844b1c4c83..8940642f7008a 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source.py
@@ -215,6 +215,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
             )
         ):
             auto_lowercase_dataset_urns = auto_lowercase_urns
+
         return [
             auto_lowercase_dataset_urns,
             auto_status_aspect,
diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
index 4acf99a50e50e..df0b732833fbe 100644
--- a/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
+++ b/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
@@ -1,6 +1,18 @@
 import json
 import logging
-from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Type,
+    Union,
+    cast,
+    overload,
+)
 
 import avro.schema
 
@@ -54,6 +66,8 @@
     avro.schema.PrimitiveSchema,
 ]
 
+SchemaOrField = Union[avro.schema.Schema, avro.schema.Field]
+
 FieldStack = List[avro.schema.Field]
 
 # The latest avro code contains this type definition in a compatibility module,
@@ -124,16 +138,22 @@ def __init__(
         self._meta_mapping_processor = meta_mapping_processor
         self._schema_tags_field = schema_tags_field
         self._tag_prefix = tag_prefix
+
         # Map of avro schema type to the conversion handler
-        self._avro_type_to_mce_converter_map: Dict[
-            avro.schema.Schema,
-            Callable[[ExtendedAvroNestedSchemas], Generator[SchemaField, None, None]],
+        # TODO: Clean up this type... perhaps refactor
+        self._avro_type_to_mce_converter_map: Mapping[
+            Union[
+                Type[avro.schema.Schema],
+                Type[avro.schema.Field],
+                Type[avro.schema.LogicalSchema],
+            ],
+            Callable[[SchemaOrField], Iterable[SchemaField]],
         ] = {
             avro.schema.RecordSchema: self._gen_from_non_field_nested_schemas,
             avro.schema.UnionSchema: self._gen_from_non_field_nested_schemas,
             avro.schema.ArraySchema: self._gen_from_non_field_nested_schemas,
             avro.schema.MapSchema: self._gen_from_non_field_nested_schemas,
-            avro.schema.Field: self._gen_nested_schema_from_field,
+            avro.schema.Field: self._gen_nested_schema_from_field,  # type: ignore
             avro.schema.PrimitiveSchema: self._gen_non_nested_to_mce_fields,
             avro.schema.FixedSchema: self._gen_non_nested_to_mce_fields,
             avro.schema.EnumSchema: self._gen_non_nested_to_mce_fields,
@@ -142,20 +162,22 @@ def __init__(
 
     @staticmethod
     def _get_type_name(
-        avro_schema: avro.schema.Schema, logical_if_present: bool = False
+        avro_schema: SchemaOrField, logical_if_present: bool = False
     ) -> str:
         logical_type_name: Optional[str] = None
         if logical_if_present:
-            logical_type_name = getattr(
-                avro_schema, "logical_type", None
-            ) or avro_schema.props.get("logicalType")
+            logical_type_name = cast(
+                Optional[str],
+                getattr(avro_schema, "logical_type", None)
+                or avro_schema.props.get("logicalType"),
+            )
         return logical_type_name or str(
             getattr(avro_schema.type, "type", avro_schema.type)
         )
 
     @staticmethod
     def _get_column_type(
-        avro_schema: avro.schema.Schema, logical_type: Optional[str]
+        avro_schema: SchemaOrField, logical_type: Optional[str]
     ) -> SchemaFieldDataType:
         type_name: str = AvroToMceSchemaConverter._get_type_name(avro_schema)
         TypeClass: Optional[Type] = AvroToMceSchemaConverter.field_type_mapping.get(
@@ -186,7 +208,7 @@ def _get_column_type(
             )
         return dt
 
-    def _is_nullable(self, schema: avro.schema.Schema) -> bool:
+    def _is_nullable(self, schema: SchemaOrField) -> bool:
         if isinstance(schema, avro.schema.Field):
             return self._is_nullable(schema.type)
         if isinstance(schema, avro.schema.UnionSchema):
@@ -208,7 +230,7 @@ def _strip_namespace(name_or_fullname: str) -> str:
         return name_or_fullname.rsplit(".", maxsplit=1)[-1]
 
     @staticmethod
-    def _get_simple_native_type(schema: ExtendedAvroNestedSchemas) -> str:
+    def _get_simple_native_type(schema: SchemaOrField) -> str:
         if isinstance(schema, (avro.schema.RecordSchema, avro.schema.Field)):
             # For Records, fields, always return the name.
             return AvroToMceSchemaConverter._strip_namespace(schema.name)
@@ -226,7 +248,7 @@ def _get_simple_native_type(schema: ExtendedAvroNestedSchemas) -> str:
         return schema.type
 
     @staticmethod
-    def _get_type_annotation(schema: ExtendedAvroNestedSchemas) -> str:
+    def _get_type_annotation(schema: SchemaOrField) -> str:
         simple_native_type = AvroToMceSchemaConverter._get_simple_native_type(schema)
         if simple_native_type.startswith("__struct_"):
             simple_native_type = "struct"
@@ -237,10 +259,24 @@ def _get_type_annotation(schema: ExtendedAvroNestedSchemas) -> str:
         else:
             return f"[type={simple_native_type}]"
 
+    @staticmethod
+    @overload
+    def _get_underlying_type_if_option_as_union(
+        schema: SchemaOrField, default: SchemaOrField
+    ) -> SchemaOrField:
+        ...
+
+    @staticmethod
+    @overload
+    def _get_underlying_type_if_option_as_union(
+        schema: SchemaOrField, default: Optional[SchemaOrField] = None
+    ) -> Optional[SchemaOrField]:
+        ...
+
     @staticmethod
     def _get_underlying_type_if_option_as_union(
-        schema: AvroNestedSchemas, default: Optional[AvroNestedSchemas] = None
-    ) -> AvroNestedSchemas:
+        schema: SchemaOrField, default: Optional[SchemaOrField] = None
+    ) -> Optional[SchemaOrField]:
         if isinstance(schema, avro.schema.UnionSchema) and len(schema.schemas) == 2:
             (first, second) = schema.schemas
             if first.type == AVRO_TYPE_NULL:
@@ -258,8 +294,8 @@ class SchemaFieldEmissionContextManager:
 
         def __init__(
             self,
-            schema: avro.schema.Schema,
-            actual_schema: avro.schema.Schema,
+            schema: SchemaOrField,
+            actual_schema: SchemaOrField,
             converter: "AvroToMceSchemaConverter",
             description: Optional[str] = None,
             default_value: Optional[str] = None,
@@ -275,7 +311,7 @@ def __enter__(self):
             self._converter._prefix_name_stack.append(type_annotation)
             return self
 
-        def emit(self) -> Generator[SchemaField, None, None]:
+        def emit(self) -> Iterable[SchemaField]:
             if (
                 not isinstance(
                     self._actual_schema,
@@ -307,7 +343,7 @@ def emit(self) -> Generator[SchemaField, None, None]:
 
                 description = self._description
                 if not description and actual_schema.props.get("doc"):
-                    description = actual_schema.props.get("doc")
+                    description = cast(Optional[str], actual_schema.props.get("doc"))
 
                 if self._default_value is not None:
                     description = f"{description if description else ''}\nField default value: {self._default_value}"
@@ -320,12 +356,12 @@ def emit(self) -> Generator[SchemaField, None, None]:
                     native_data_type = native_data_type[
                         slice(len(type_prefix), len(native_data_type) - 1)
                     ]
-                native_data_type = actual_schema.props.get(
-                    "native_data_type", native_data_type
+                native_data_type = cast(
+                    str, actual_schema.props.get("native_data_type", native_data_type)
                 )
 
                 field_path = self._converter._get_cur_field_path()
-                merged_props = {}
+                merged_props: Dict[str, Any] = {}
                 merged_props.update(self._schema.other_props)
                 merged_props.update(schema.other_props)
 
@@ -363,12 +399,13 @@ def emit(self) -> Generator[SchemaField, None, None]:
 
                 meta_terms_aspect = meta_aspects.get(Constants.ADD_TERM_OPERATION)
 
-                logical_type_name: Optional[str] = (
+                logical_type_name: Optional[str] = cast(
+                    Optional[str],
                     # logicalType nested inside type
                     getattr(actual_schema, "logical_type", None)
                     or actual_schema.props.get("logicalType")
                     # bare logicalType
-                    or self._actual_schema.props.get("logicalType")
+                    or self._actual_schema.props.get("logicalType"),
                 )
 
                 field = SchemaField(
@@ -392,14 +429,12 @@ def emit(self) -> Generator[SchemaField, None, None]:
         def __exit__(self, exc_type, exc_val, exc_tb):
             self._converter._prefix_name_stack.pop()
 
-    def _get_sub_schemas(
-        self, schema: ExtendedAvroNestedSchemas
-    ) -> Generator[avro.schema.Schema, None, None]:
+    def _get_sub_schemas(self, schema: SchemaOrField) -> Iterable[SchemaOrField]:
         """Responsible for generation for appropriate sub-schemas for every nested AVRO type."""
 
         def gen_items_from_list_tuple_or_scalar(
             val: Any,
-        ) -> Generator[avro.schema.Schema, None, None]:
+        ) -> Iterable[avro.schema.Schema]:
             if isinstance(val, (list, tuple)):
                 for i in val:
                     yield i
@@ -433,7 +468,7 @@ def gen_items_from_list_tuple_or_scalar(
     def _gen_nested_schema_from_field(
         self,
         field: avro.schema.Field,
-    ) -> Generator[SchemaField, None, None]:
+    ) -> Iterable[SchemaField]:
         """Handles generation of MCE SchemaFields for an AVRO Field type."""
         # NOTE: Here we only manage the field stack and trigger MCE Field generation from this field's type.
         # The actual emitting of a field happens when
@@ -447,7 +482,7 @@ def _gen_nested_schema_from_field(
 
     def _gen_from_last_field(
         self, schema_to_recurse: Optional[AvroNestedSchemas] = None
-    ) -> Generator[SchemaField, None, None]:
+    ) -> Iterable[SchemaField]:
         """Emits the field most-recent field, optionally triggering sub-schema generation under the field."""
         last_field_schema = self._fields_stack[-1]
         # Generate the custom-description for the field.
@@ -467,8 +502,8 @@ def _gen_from_last_field(
                     yield from self._to_mce_fields(sub_schema)
 
     def _gen_from_non_field_nested_schemas(
-        self, schema: AvroNestedSchemas
-    ) -> Generator[SchemaField, None, None]:
+        self, schema: SchemaOrField
+    ) -> Iterable[SchemaField]:
         """Handles generation of MCE SchemaFields for all standard AVRO nested types."""
         # Handle recursive record definitions
         recurse: bool = True
@@ -511,8 +546,8 @@ def _gen_from_non_field_nested_schemas(
                         yield from self._to_mce_fields(sub_schema)
 
     def _gen_non_nested_to_mce_fields(
-        self, schema: AvroNonNestedSchemas
-    ) -> Generator[SchemaField, None, None]:
+        self, schema: SchemaOrField
+    ) -> Iterable[SchemaField]:
         """Handles generation of MCE SchemaFields for non-nested AVRO types."""
         with AvroToMceSchemaConverter.SchemaFieldEmissionContextManager(
             schema,
@@ -521,9 +556,7 @@ def _gen_non_nested_to_mce_fields(
         ) as non_nested_emitter:
             yield from non_nested_emitter.emit()
 
-    def _to_mce_fields(
-        self, avro_schema: avro.schema.Schema
-    ) -> Generator[SchemaField, None, None]:
+    def _to_mce_fields(self, avro_schema: SchemaOrField) -> Iterable[SchemaField]:
         # Invoke the relevant conversion handler for the schema element type.
         schema_type = (
             type(avro_schema)
@@ -541,7 +574,7 @@ def to_mce_fields(
         meta_mapping_processor: Optional[OperationProcessor] = None,
         schema_tags_field: Optional[str] = None,
         tag_prefix: Optional[str] = None,
-    ) -> Generator[SchemaField, None, None]:
+    ) -> Iterable[SchemaField]:
         """
         Converts a key or value type AVRO schema string to appropriate MCE SchemaFields.
         :param avro_schema_string: String representation of the AVRO schema.
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
index 07b55e0e25a89..f2735c24ca19d 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
@@ -390,7 +390,15 @@ def run(self) -> None:
                         record_envelopes = self.extractor.get_records(wu)
                         for record_envelope in self.transform(record_envelopes):
                             if not self.dry_run:
-                                self.sink.write_record_async(record_envelope, callback)
+                                try:
+                                    self.sink.write_record_async(
+                                        record_envelope, callback
+                                    )
+                                except Exception as e:
+                                    # In case the sink's error handling is bad, we still want to report the error.
+                                    self.sink.report.report_failure(
+                                        f"Failed to write record: {e}"
+                                    )
 
                     except RuntimeError:
                         raise
diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py
index 39054c256a7fd..38ddadaafc862 100644
--- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py
@@ -9,7 +9,6 @@
     MetadataChangeEvent,
     MetadataChangeProposal,
 )
-from datahub.metadata.schema_classes import MetadataChangeProposalClass
 
 
 class KafkaSinkConfig(KafkaEmitterConfig):
@@ -58,27 +57,21 @@ def write_record_async(
         ],
         write_callback: WriteCallback,
     ) -> None:
-        record = record_envelope.record
-        if isinstance(record, MetadataChangeEvent):
-            self.emitter.emit_mce_async(
+        callback = _KafkaCallback(
+            self.report, record_envelope, write_callback
+        ).kafka_callback
+        try:
+            record = record_envelope.record
+            self.emitter.emit(
                 record,
-                callback=_KafkaCallback(
-                    self.report, record_envelope, write_callback
-                ).kafka_callback,
-            )
-        elif isinstance(
-            record, (MetadataChangeProposalWrapper, MetadataChangeProposalClass)
-        ):
-            self.emitter.emit_mcp_async(
-                record,
-                callback=_KafkaCallback(
-                    self.report, record_envelope, write_callback
-                ).kafka_callback,
-            )
-        else:
-            raise ValueError(
-                f"The datahub-kafka sink only supports MetadataChangeEvent/MetadataChangeProposal[Wrapper] classes, not {type(record)}"
+                callback=callback,
             )
+        except Exception as err:
+            # In case we throw an exception while trying to emit the record,
+            # catch it and report the failure. This might happen if the schema
+            # registry is down or otherwise misconfigured, in which case we'd
+            # fail when serializing the record.
+            callback(err, f"Failed to write record: {err}")
 
     def close(self) -> None:
         self.emitter.flush()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index 692d8c4f81bb6..6959a48313010 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -1050,8 +1050,14 @@ def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]:
                     for idx, field in enumerate(schema_fields):
                         # Remove all the [version=2.0].[type=struct]. tags to get the field path
                         if (
-                            re.sub(r"\[.*?\]\.", "", field.fieldPath, 0, re.MULTILINE)
-                            == col.field_path
+                            re.sub(
+                                r"\[.*?\]\.",
+                                "",
+                                field.fieldPath.lower(),
+                                0,
+                                re.MULTILINE,
+                            )
+                            == col.field_path.lower()
                         ):
                             field.description = col.comment
                             schema_fields[idx] = field
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index 944814b6936a4..f762d451849ab 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -119,8 +119,8 @@ class BigQueryV2Config(
     )
 
     match_fully_qualified_names: bool = Field(
-        default=False,
-        description="Whether `dataset_pattern` is matched against fully qualified dataset name `<project_id>.<dataset_name>`.",
+        default=True,
+        description="[deprecated] Whether `dataset_pattern` is matched against fully qualified dataset name `<project_id>.<dataset_name>`.",
     )
 
     include_external_url: bool = Field(
@@ -299,7 +299,7 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
                 "use project_id_pattern whenever possible. project_id will be deprecated, please use project_id_pattern only if possible."
             )
 
-        dataset_pattern = values.get("dataset_pattern")
+        dataset_pattern: Optional[AllowDenyPattern] = values.get("dataset_pattern")
         schema_pattern = values.get("schema_pattern")
         if (
             dataset_pattern == AllowDenyPattern.allow_all()
@@ -309,6 +309,7 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
                 "dataset_pattern is not set but schema_pattern is set, using schema_pattern as dataset_pattern. schema_pattern will be deprecated, please use dataset_pattern instead."
             )
             values["dataset_pattern"] = schema_pattern
+            dataset_pattern = schema_pattern
         elif (
             dataset_pattern != AllowDenyPattern.allow_all()
             and schema_pattern != AllowDenyPattern.allow_all()
@@ -327,9 +328,24 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
         ):
             logger.warning(
                 "Please update `dataset_pattern` to match against fully qualified schema name `<project_id>.<dataset_name>` and set config `match_fully_qualified_names : True`."
-                "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. "
-                "The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`."
+                "The config option `match_fully_qualified_names` is deprecated and will be removed in a future release."
             )
+        elif match_fully_qualified_names and dataset_pattern is not None:
+            adjusted = False
+            for lst in [dataset_pattern.allow, dataset_pattern.deny]:
+                for i, pattern in enumerate(lst):
+                    if "." not in pattern:
+                        if pattern.startswith("^"):
+                            lst[i] = r"^.*\." + pattern[1:]
+                        else:
+                            lst[i] = r".*\." + pattern
+                        adjusted = True
+            if adjusted:
+                logger.warning(
+                    "`dataset_pattern` was adjusted to match against fully qualified schema names,"
+                    " of the form `<project_id>.<dataset_name>`."
+                )
+
         return values
 
     def get_table_pattern(self, pattern: List[str]) -> str:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
index 98c8cbaf85eec..e9acf5ea86044 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
@@ -20,6 +20,7 @@
 from google.cloud.datacatalog import lineage_v1
 from google.cloud.logging_v2.client import Client as GCPLoggingClient
 
+from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
@@ -548,7 +549,7 @@ def _get_parsed_audit_log_events(self, project_id: str) -> Iterable[QueryEvent]:
         # handle the case where the read happens within our time range but the query
         # completion event is delayed and happens after the configured end time.
         corrected_start_time = self.start_time - self.config.max_query_duration
-        corrected_end_time = self.end_time + -self.config.max_query_duration
+        corrected_end_time = self.end_time + self.config.max_query_duration
         self.report.log_entry_start_time = corrected_start_time
         self.report.log_entry_end_time = corrected_end_time
 
@@ -683,8 +684,11 @@ def _create_lineage_map(
                 self.report.num_skipped_lineage_entries_missing_data[e.project_id] += 1
                 continue
 
-            if not self.config.dataset_pattern.allowed(
-                destination_table.table_identifier.dataset
+            if not is_schema_allowed(
+                self.config.dataset_pattern,
+                destination_table.table_identifier.dataset,
+                destination_table.table_identifier.project_id,
+                self.config.match_fully_qualified_names,
             ) or not self.config.table_pattern.allowed(
                 destination_table.table_identifier.get_table_name()
             ):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
index 201567e104a51..65b559550ffc5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
@@ -21,6 +21,7 @@
 
 import humanfriendly
 
+from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.configuration.time_window_config import (
     BaseTimeWindowConfig,
     get_time_bucket,
@@ -335,8 +336,13 @@ def get_time_window(self) -> Tuple[datetime, datetime]:
     def _is_table_allowed(self, table_ref: Optional[BigQueryTableRef]) -> bool:
         return (
             table_ref is not None
-            and self.config.dataset_pattern.allowed(table_ref.table_identifier.dataset)
-            and self.config.table_pattern.allowed(table_ref.table_identifier.table)
+            and is_schema_allowed(
+                self.config.dataset_pattern,
+                table_ref.table_identifier.dataset,
+                table_ref.table_identifier.project_id,
+                self.config.match_fully_qualified_names,
+            )
+            and self.config.table_pattern.allowed(str(table_ref.table_identifier))
         )
 
     def _should_ingest_usage(self) -> bool:
@@ -844,7 +850,7 @@ def _get_parsed_bigquery_log_events(
         # handle the case where the read happens within our time range but the query
         # completion event is delayed and happens after the configured end time.
         corrected_start_time = self.start_time - self.config.max_query_duration
-        corrected_end_time = self.end_time + -self.config.max_query_duration
+        corrected_end_time = self.end_time + self.config.max_query_duration
         self.report.audit_start_time = corrected_start_time
         self.report.audit_end_time = corrected_end_time
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
index 053d136305527..83958dc76754f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
@@ -1,3 +1,4 @@
+import os
 from typing import Optional
 
 from pydantic import Field, root_validator
@@ -67,9 +68,25 @@ class DataHubSourceConfig(StatefulIngestionConfigBase):
         ),
     )
 
+    pull_from_datahub_api: bool = Field(
+        default=False,
+        description="Use the DataHub API to fetch versioned aspects.",
+        hidden_from_docs=True,
+    )
+
+    max_workers: int = Field(
+        default=5 * (os.cpu_count() or 4),
+        description="Number of worker threads to use for datahub api ingestion.",
+        hidden_from_docs=True,
+    )
+
     @root_validator
     def check_ingesting_data(cls, values):
-        if not values.get("database_connection") and not values.get("kafka_connection"):
+        if (
+            not values.get("database_connection")
+            and not values.get("kafka_connection")
+            and not values.get("pull_from_datahub_api")
+        ):
             raise ValueError(
                 "Your current config will not ingest any data."
                 " Please specify at least one of `database_connection` or `kafka_connection`, ideally both."
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
new file mode 100644
index 0000000000000..7ee36736723b2
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
@@ -0,0 +1,49 @@
+import logging
+from concurrent import futures
+from typing import Dict, Iterable, List
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.graph.filters import RemovedStatusFilter
+from datahub.ingestion.source.datahub.config import DataHubSourceConfig
+from datahub.ingestion.source.datahub.report import DataHubSourceReport
+from datahub.metadata._schema_classes import _Aspect
+
+logger = logging.getLogger(__name__)
+
+# Should work for at least mysql, mariadb, postgres
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
+
+
+class DataHubApiReader:
+    def __init__(
+        self,
+        config: DataHubSourceConfig,
+        report: DataHubSourceReport,
+        graph: DataHubGraph,
+    ):
+        self.config = config
+        self.report = report
+        self.graph = graph
+
+    def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
+        urns = self.graph.get_urns_by_filter(
+            status=RemovedStatusFilter.ALL,
+            batch_size=self.config.database_query_batch_size,
+        )
+        tasks: List[futures.Future[Iterable[MetadataChangeProposalWrapper]]] = []
+        with futures.ThreadPoolExecutor(
+            max_workers=self.config.max_workers
+        ) as executor:
+            for urn in urns:
+                tasks.append(executor.submit(self._get_aspects_for_urn, urn))
+            for task in futures.as_completed(tasks):
+                yield from task.result()
+
+    def _get_aspects_for_urn(self, urn: str) -> Iterable[MetadataChangeProposalWrapper]:
+        aspects: Dict[str, _Aspect] = self.graph.get_entity_semityped(urn)  # type: ignore
+        for aspect in aspects.values():
+            yield MetadataChangeProposalWrapper(
+                entityUrn=urn,
+                aspect=aspect,
+            )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
index 2368febe1ff57..a2f43b8cc62cb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
@@ -15,6 +15,7 @@
 from datahub.ingestion.api.source_helpers import auto_workunit_reporter
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.datahub.config import DataHubSourceConfig
+from datahub.ingestion.source.datahub.datahub_api_reader import DataHubApiReader
 from datahub.ingestion.source.datahub.datahub_database_reader import (
     DataHubDatabaseReader,
 )
@@ -58,6 +59,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         logger.info(f"Ingesting DataHub metadata up until {self.report.stop_time}")
         state = self.stateful_ingestion_handler.get_last_run_state()
 
+        if self.config.pull_from_datahub_api:
+            yield from self._get_api_workunits()
+
         if self.config.database_connection is not None:
             yield from self._get_database_workunits(
                 from_createdon=state.database_createdon_datetime
@@ -139,6 +143,18 @@ def _get_kafka_workunits(
                     )
                 self._commit_progress(i)
 
+    def _get_api_workunits(self) -> Iterable[MetadataWorkUnit]:
+        if self.ctx.graph is None:
+            self.report.report_failure(
+                "datahub_api",
+                "Specify datahub_api on your ingestion recipe to ingest from the DataHub API",
+            )
+            return
+
+        reader = DataHubApiReader(self.config, self.report, self.ctx.graph)
+        for mcp in reader.get_aspects():
+            yield mcp.as_workunit()
+
     def _commit_progress(self, i: Optional[int] = None) -> None:
         """Commit progress to stateful storage, if there have been no errors.
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 48d2118a9b091..c4de24bf192f1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -280,6 +280,11 @@ class DBTCommonConfig(
         default=False,
         description="When enabled, dbt test warnings will be treated as failures.",
     )
+    # override fault value to True.
+    incremental_lineage: bool = Field(
+        default=True,
+        description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
+    )
 
     @validator("target_platform")
     def validate_target_platform_value(cls, target_platform: str) -> str:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
index d5039360da567..23770ff3cf812 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
@@ -3,7 +3,7 @@
 import logging
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Any, Dict, Iterable, List, Optional, Type
+from typing import Any, Dict, Iterable, List, Optional, Type, cast
 
 import avro.schema
 import confluent_kafka
@@ -316,13 +316,20 @@ def _extract_record(
             avro_schema = avro.schema.parse(
                 schema_metadata.platformSchema.documentSchema
             )
-            description = avro_schema.doc
+            description = getattr(avro_schema, "doc", None)
             # set the tags
             all_tags: List[str] = []
-            for tag in avro_schema.other_props.get(
-                self.source_config.schema_tags_field, []
-            ):
-                all_tags.append(self.source_config.tag_prefix + tag)
+            try:
+                schema_tags = cast(
+                    Iterable[str],
+                    avro_schema.other_props.get(
+                        self.source_config.schema_tags_field, []
+                    ),
+                )
+                for tag in schema_tags:
+                    all_tags.append(self.source_config.tag_prefix + tag)
+            except TypeError:
+                pass
 
             if self.source_config.enable_meta_mapping:
                 meta_aspects = self.meta_processor.process(avro_schema.other_props)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
index f02b6845e40b5..890c5c64bd5e6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
@@ -11,7 +11,11 @@
 from pymongo.mongo_client import MongoClient
 
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.source_common import EnvConfigMixin
+from datahub.configuration.source_common import (
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+)
+from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -55,7 +59,7 @@
 DENY_DATABASE_LIST = set(["admin", "config", "local"])
 
 
-class MongoDBConfig(EnvConfigMixin):
+class MongoDBConfig(PlatformInstanceConfigMixin, EnvConfigMixin):
     # See the MongoDB authentication docs for details and examples.
     # https://pymongo.readthedocs.io/en/stable/examples/authentication.html
     connect_uri: str = Field(
@@ -199,6 +203,7 @@ def construct_schema_pymongo(
 @platform_name("MongoDB")
 @config_class(MongoDBConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @dataclass
 class MongoDBSource(Source):
@@ -320,7 +325,12 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     self.report.report_dropped(dataset_name)
                     continue
 
-                dataset_urn = f"urn:li:dataset:(urn:li:dataPlatform:{platform},{dataset_name},{self.config.env})"
+                dataset_urn = make_dataset_urn_with_platform_instance(
+                    platform=platform,
+                    name=dataset_name,
+                    env=self.config.env,
+                    platform_instance=self.config.platform_instance,
+                )
 
                 dataset_snapshot = DatasetSnapshot(
                     urn=dataset_urn,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
index 2789b800940db..79b044841e054 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -133,7 +133,13 @@ class RedshiftConfig(
     )
 
     extract_column_level_lineage: bool = Field(
-        default=True, description="Whether to extract column level lineage."
+        default=True,
+        description="Whether to extract column level lineage. This config works with rest-sink only.",
+    )
+
+    incremental_lineage: bool = Field(
+        default=False,
+        description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.  This config works with rest-sink only.",
     )
 
     @root_validator(pre=True)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
index a1b6333a3775d..c7d01021773b1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -1,5 +1,6 @@
 import logging
 from collections import defaultdict
+from functools import partial
 from typing import Dict, Iterable, List, Optional, Type, Union
 
 import humanfriendly
@@ -25,6 +26,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
 from datahub.ingestion.api.source import (
     CapabilityReport,
     MetadataWorkUnitProcessor,
@@ -216,6 +218,9 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
     ] = {
         "BYTES": BytesType,
         "BOOL": BooleanType,
+        "BOOLEAN": BooleanType,
+        "DOUBLE": NumberType,
+        "DOUBLE PRECISION": NumberType,
         "DECIMAL": NumberType,
         "NUMERIC": NumberType,
         "BIGNUMERIC": NumberType,
@@ -242,6 +247,13 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
         "CHARACTER": StringType,
         "CHAR": StringType,
         "TIMESTAMP WITHOUT TIME ZONE": TimeType,
+        "REAL": NumberType,
+        "VARCHAR": StringType,
+        "TIMESTAMPTZ": TimeType,
+        "GEOMETRY": NullType,
+        "HLLSKETCH": NullType,
+        "TIMETZ": TimeType,
+        "VARBYTE": StringType,
     }
 
     def get_platform_instance_id(self) -> str:
@@ -369,6 +381,11 @@ def gen_database_container(self, database: str) -> Iterable[MetadataWorkUnit]:
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
+            partial(
+                auto_incremental_lineage,
+                self.ctx.graph,
+                self.config.incremental_lineage,
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
@@ -942,7 +959,9 @@ def generate_lineage(self, database: str) -> Iterable[MetadataWorkUnit]:
                 )
                 if lineage_info:
                     yield from gen_lineage(
-                        dataset_urn, lineage_info, self.config.incremental_lineage
+                        dataset_urn,
+                        lineage_info,
+                        incremental_lineage=False,  # incremental lineage generation is taken care by auto_incremental_lineage
                     )
 
         for schema in self.db_views[database]:
@@ -956,7 +975,9 @@ def generate_lineage(self, database: str) -> Iterable[MetadataWorkUnit]:
                 )
                 if lineage_info:
                     yield from gen_lineage(
-                        dataset_urn, lineage_info, self.config.incremental_lineage
+                        dataset_urn,
+                        lineage_info,
+                        incremental_lineage=False,  # incremental lineage generation is taken care by auto_incremental_lineage
                     )
 
     def add_config_to_report(self):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
index 0a15c352fc842..4219533dc217c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
@@ -20,12 +20,12 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage
 from datahub.ingestion.source.snowflake.constants import (
     LINEAGE_PERMISSION_ERROR,
     SnowflakeEdition,
-    SnowflakeObjectDomain,
 )
 from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config
 from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery
@@ -53,7 +53,6 @@
     sqlglot_lineage,
 )
 from datahub.utilities.time import ts_millis_to_datetime
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -136,7 +135,6 @@ def get_workunits(
             return
 
         self._populate_external_lineage_map(discovered_tables)
-
         if self.config.include_view_lineage:
             if len(discovered_views) > 0:
                 yield from self.get_view_upstream_workunits(
@@ -196,19 +194,6 @@ def get_table_upstream_workunits(
                 f"Upstream lineage detected for {self.report.num_tables_with_upstreams} tables.",
             )
 
-    def _gen_workunit_from_sql_parsing_result(
-        self,
-        dataset_identifier: str,
-        result: SqlParsingResult,
-    ) -> MetadataWorkUnit:
-        upstreams, fine_upstreams = self.get_upstreams_from_sql_parsing_result(
-            self.dataset_urn_builder(dataset_identifier), result
-        )
-        self.report.num_views_with_upstreams += 1
-        return self._create_upstream_lineage_workunit(
-            dataset_identifier, upstreams, fine_upstreams
-        )
-
     def _gen_workunits_from_query_result(
         self,
         discovered_assets: Collection[str],
@@ -242,18 +227,31 @@ def get_view_upstream_workunits(
         schema_resolver: SchemaResolver,
         view_definitions: MutableMapping[str, str],
     ) -> Iterable[MetadataWorkUnit]:
-        views_processed = set()
+        views_failed_parsing = set()
         if self.config.include_view_column_lineage:
             with PerfTimer() as timer:
+                builder = SqlParsingBuilder(
+                    generate_lineage=True,
+                    generate_usage_statistics=False,
+                    generate_operations=False,
+                )
                 for view_identifier, view_definition in view_definitions.items():
                     result = self._run_sql_parser(
                         view_identifier, view_definition, schema_resolver
                     )
-                    if result:
-                        views_processed.add(view_identifier)
-                        yield self._gen_workunit_from_sql_parsing_result(
-                            view_identifier, result
+                    if result and result.out_tables:
+                        self.report.num_views_with_upstreams += 1
+                        # This does not yield any workunits but we use
+                        # yield here to execute this method
+                        yield from builder.process_sql_parsing_result(
+                            result=result,
+                            query=view_definition,
+                            is_view_ddl=True,
                         )
+                    else:
+                        views_failed_parsing.add(view_identifier)
+
+                yield from builder.gen_workunits()
                 self.report.view_lineage_parse_secs = timer.elapsed_seconds()
 
         with PerfTimer() as timer:
@@ -261,7 +259,7 @@ def get_view_upstream_workunits(
 
             if results:
                 yield from self._gen_workunits_from_query_result(
-                    set(discovered_views) - views_processed,
+                    views_failed_parsing,
                     results,
                     upstream_for_view=True,
                 )
@@ -349,39 +347,6 @@ def get_upstreams_from_query_result_row(
 
         return upstreams, fine_upstreams
 
-    def get_upstreams_from_sql_parsing_result(
-        self, downstream_table_urn: str, result: SqlParsingResult
-    ) -> Tuple[List[UpstreamClass], List[FineGrainedLineage]]:
-        # Note: This ignores the out_tables section of the sql parsing result.
-        upstreams = [
-            UpstreamClass(dataset=upstream_table_urn, type=DatasetLineageTypeClass.VIEW)
-            for upstream_table_urn in set(result.in_tables)
-        ]
-
-        # Maps downstream_col -> [upstream_col]
-        fine_lineage: Dict[str, Set[SnowflakeColumnId]] = defaultdict(set)
-        for column_lineage in result.column_lineage or []:
-            out_column = column_lineage.downstream.column
-            for upstream_column_info in column_lineage.upstreams:
-                upstream_table_name = DatasetUrn.create_from_string(
-                    upstream_column_info.table
-                ).get_dataset_name()
-                fine_lineage[out_column].add(
-                    SnowflakeColumnId(
-                        columnName=upstream_column_info.column,
-                        objectName=upstream_table_name,
-                        objectDomain=SnowflakeObjectDomain.VIEW.value,
-                    )
-                )
-        fine_upstreams = [
-            self.build_finegrained_lineage(
-                downstream_table_urn, downstream_col, upstream_cols
-            )
-            for downstream_col, upstream_cols in fine_lineage.items()
-        ]
-
-        return upstreams, list(filter(None, fine_upstreams))
-
     def _populate_external_lineage_map(self, discovered_tables: List[str]) -> None:
         with PerfTimer() as timer:
             self.report.num_external_table_edges_scanned = 0
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
index dad61e5173166..75e8fe1d6f7a6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
@@ -31,22 +31,24 @@
     register_custom_type,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, make_sqlalchemy_uri
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     gen_database_container,
     gen_database_key,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
-from datahub.metadata.schema_classes import RecordTypeClass
+from datahub.metadata.schema_classes import MapTypeClass, RecordTypeClass
 from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
 from datahub.utilities.sqlalchemy_type_converter import (
+    MapType,
     get_schema_fields_for_sqlalchemy_column,
 )
 
 logger = logging.getLogger(__name__)
 
+assert STRUCT, "required type modules are not available"
 register_custom_type(STRUCT, RecordTypeClass)
+register_custom_type(MapType, MapTypeClass)
 
 
 class CustomAthenaRestDialect(AthenaRestDialect):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
index 63b21bc82eddd..d081acb6c1eff 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
@@ -1,15 +1,18 @@
 import json
 import logging
 import re
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 from pydantic.class_validators import validator
 from pydantic.fields import Field
 
 # This import verifies that the dependencies are available.
 from pyhive import hive  # noqa: F401
-from pyhive.sqlalchemy_hive import HiveDate, HiveDecimal, HiveTimestamp
+from pyhive.sqlalchemy_hive import HiveDate, HiveDecimal, HiveDialect, HiveTimestamp
+from sqlalchemy.engine.reflection import Inspector
 
+from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.decorators import (
     SourceCapability,
     SupportStatus,
@@ -18,8 +21,10 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.extractor import schema_util
-from datahub.ingestion.source.sql.sql_common import register_custom_type
+from datahub.ingestion.source.sql.sql_common import SqlWorkUnit, register_custom_type
+from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
 from datahub.ingestion.source.sql.two_tier_sql_source import (
     TwoTierSQLAlchemyConfig,
     TwoTierSQLAlchemySource,
@@ -31,6 +36,7 @@
     SchemaField,
     TimeTypeClass,
 )
+from datahub.metadata.schema_classes import ViewPropertiesClass
 from datahub.utilities import config_clean
 from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
 
@@ -90,19 +96,34 @@ def dbapi_get_columns_patched(self, connection, table_name, schema=None, **kw):
     logger.warning(f"Failed to patch method due to {e}")
 
 
+@reflection.cache  # type: ignore
+def get_view_names_patched(self, connection, schema=None, **kw):
+    query = "SHOW VIEWS"
+    if schema:
+        query += " IN " + self.identifier_preparer.quote_identifier(schema)
+    return [row[0] for row in connection.execute(query)]
+
+
+@reflection.cache  # type: ignore
+def get_view_definition_patched(self, connection, view_name, schema=None, **kw):
+    full_table = self.identifier_preparer.quote_identifier(view_name)
+    if schema:
+        full_table = "{}.{}".format(
+            self.identifier_preparer.quote_identifier(schema),
+            self.identifier_preparer.quote_identifier(view_name),
+        )
+    row = connection.execute("SHOW CREATE TABLE {}".format(full_table)).fetchone()
+    return row[0]
+
+
+HiveDialect.get_view_names = get_view_names_patched
+HiveDialect.get_view_definition = get_view_definition_patched
+
+
 class HiveConfig(TwoTierSQLAlchemyConfig):
     # defaults
     scheme = Field(default="hive", hidden_from_docs=True)
 
-    # Hive SQLAlchemy connector returns views as tables.
-    # See https://github.com/dropbox/PyHive/blob/b21c507a24ed2f2b0cf15b0b6abb1c43f31d3ee0/pyhive/sqlalchemy_hive.py#L270-L273.
-    # Disabling views helps us prevent this duplication.
-    include_views = Field(
-        default=False,
-        hidden_from_docs=True,
-        description="Hive SQLAlchemy connector returns views as tables. See https://github.com/dropbox/PyHive/blob/b21c507a24ed2f2b0cf15b0b6abb1c43f31d3ee0/pyhive/sqlalchemy_hive.py#L270-L273. Disabling views helps us prevent this duplication.",
-    )
-
     @validator("host_port")
     def clean_host_port(cls, v):
         return config_clean.remove_protocol(v)
@@ -174,3 +195,41 @@ def get_schema_fields_for_column(
             return new_fields
 
         return fields
+
+    # Hive SQLAlchemy connector returns views as tables in get_table_names.
+    # See https://github.com/dropbox/PyHive/blob/b21c507a24ed2f2b0cf15b0b6abb1c43f31d3ee0/pyhive/sqlalchemy_hive.py#L270-L273.
+    # This override makes sure that we ingest view definitions for views
+    def _process_view(
+        self,
+        dataset_name: str,
+        inspector: Inspector,
+        schema: str,
+        view: str,
+        sql_config: SQLCommonConfig,
+    ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]:
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            self.platform,
+            dataset_name,
+            self.config.platform_instance,
+            self.config.env,
+        )
+
+        try:
+            view_definition = inspector.get_view_definition(view, schema)
+            if view_definition is None:
+                view_definition = ""
+            else:
+                # Some dialects return a TextClause instead of a raw string,
+                # so we need to convert them to a string.
+                view_definition = str(view_definition)
+        except NotImplementedError:
+            view_definition = ""
+
+        if view_definition:
+            view_properties_aspect = ViewPropertiesClass(
+                materialized=False, viewLanguage="SQL", viewLogic=view_definition
+            )
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=view_properties_aspect,
+            ).as_workunit()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
index 685d4fb3074c9..710825c8ba55d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
@@ -530,7 +530,7 @@ def _get_procedure_inputs(
     def _get_procedure_code(
         conn: Connection, procedure: StoredProcedure
     ) -> Tuple[Optional[str], Optional[str]]:
-        query = f"EXEC [{procedure.db}].dbo.sp_helptext '{procedure.full_name}'"
+        query = f"EXEC [{procedure.db}].dbo.sp_helptext '{procedure.escape_full_name}'"
         try:
             code_data = conn.execute(query)
         except ProgrammingError:
@@ -567,7 +567,7 @@ def _get_procedure_properties(
                 create_date as date_created,
                 modify_date as date_modified
             FROM sys.procedures
-            WHERE object_id = object_id('{procedure.full_name}')
+            WHERE object_id = object_id('{procedure.escape_full_name}')
             """
         )
         properties = {}
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
index a6a9d8e2c8597..4f133c6459a0f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
@@ -103,10 +103,6 @@ class BasePostgresConfig(BasicSQLAlchemyConfig):
 
 
 class PostgresConfig(BasePostgresConfig):
-    include_view_lineage = Field(
-        default=False, description="Include table lineage for views"
-    )
-
     database_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
         description=(
@@ -183,9 +179,10 @@ def get_inspectors(self) -> Iterable[Inspector]:
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
         yield from super().get_workunits_internal()
 
-        for inspector in self.get_inspectors():
-            if self.config.include_view_lineage:
-                yield from self._get_view_lineage_workunits(inspector)
+        if self.views_failed_parsing:
+            for inspector in self.get_inspectors():
+                if self.config.include_view_lineage:
+                    yield from self._get_view_lineage_workunits(inspector)
 
     def _get_view_lineage_elements(
         self, inspector: Inspector
@@ -245,11 +242,14 @@ def _get_view_lineage_workunits(
             dependent_view, dependent_schema = key
 
             # Construct a lineage object.
+            view_identifier = self.get_identifier(
+                schema=dependent_schema, entity=dependent_view, inspector=inspector
+            )
+            if view_identifier not in self.views_failed_parsing:
+                return
             urn = mce_builder.make_dataset_urn_with_platform_instance(
                 platform=self.platform,
-                name=self.get_identifier(
-                    schema=dependent_schema, entity=dependent_view, inspector=inspector
-                ),
+                name=view_identifier,
                 platform_instance=self.config.platform_instance,
                 env=self.config.env,
             )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index 6524eea8222d4..51909eaf4ed55 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -2,12 +2,14 @@
 import logging
 import traceback
 from dataclasses import dataclass, field
+from functools import partial
 from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
     Iterable,
     List,
+    MutableMapping,
     Optional,
     Set,
     Tuple,
@@ -29,7 +31,9 @@
     make_tag_urn,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
 from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import (
@@ -37,7 +41,6 @@
     DatasetSubTypes,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     downgrade_schema_from_v2,
@@ -81,16 +84,22 @@
     DatasetLineageTypeClass,
     DatasetPropertiesClass,
     GlobalTagsClass,
-    MapTypeClass,
     SubTypesClass,
     TagAssociationClass,
     UpstreamClass,
     ViewPropertiesClass,
 )
 from datahub.telemetry import telemetry
+from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.registries.domain_registry import DomainRegistry
 from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport
+from datahub.utilities.sqlglot_lineage import (
+    SchemaResolver,
+    SqlParsingResult,
+    sqlglot_lineage,
+    view_definition_lineage_helper,
+)
 
 if TYPE_CHECKING:
     from datahub.ingestion.source.ge_data_profiler import (
@@ -112,6 +121,11 @@ class SQLSourceReport(StaleEntityRemovalSourceReport):
 
     query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None
 
+    num_view_definitions_parsed: int = 0
+    num_view_definitions_failed_parsing: int = 0
+    num_view_definitions_failed_column_parsing: int = 0
+    view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList)
+
     def report_entity_scanned(self, name: str, ent_type: str = "table") -> None:
         """
         Entity could be a view or a table
@@ -140,6 +154,7 @@ class SqlWorkUnit(MetadataWorkUnit):
 
 
 _field_type_mapping: Dict[Type[TypeEngine], Type] = {
+    # Note: to add dialect-specific types to this mapping, use the `register_custom_type` function.
     types.Integer: NumberTypeClass,
     types.Numeric: NumberTypeClass,
     types.Boolean: BooleanTypeClass,
@@ -156,8 +171,6 @@ class SqlWorkUnit(MetadataWorkUnit):
     types.DATETIME: TimeTypeClass,
     types.TIMESTAMP: TimeTypeClass,
     types.JSON: RecordTypeClass,
-    # additional type definitions that are used by the Athena source
-    MapType: MapTypeClass,  # type: ignore
     # Because the postgresql dialect is used internally by many other dialects,
     # we add some postgres types here. This is ok to do because the postgresql
     # dialect is built-in to sqlalchemy.
@@ -322,6 +335,18 @@ def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str)
                 cached_domains=[k for k in self.config.domain], graph=self.ctx.graph
             )
 
+        self.views_failed_parsing: Set[str] = set()
+        self.schema_resolver: SchemaResolver = SchemaResolver(
+            platform=self.platform,
+            platform_instance=self.config.platform_instance,
+            env=self.config.env,
+        )
+        self._view_definition_cache: MutableMapping[str, str]
+        if self.config.use_file_backed_cache:
+            self._view_definition_cache = FileBackedDict[str]()
+        else:
+            self._view_definition_cache = {}
+
     def warn(self, log: logging.Logger, key: str, reason: str) -> None:
         self.report.report_warning(key, reason[:100])
         log.warning(f"{key} => {reason}")
@@ -458,6 +483,11 @@ def get_schema_level_workunits(
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
+            partial(
+                auto_incremental_lineage,
+                self.ctx.graph,
+                self.config.incremental_lineage,
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
@@ -515,6 +545,35 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit
                     profile_requests, profiler, platform=self.platform
                 )
 
+        if self.config.include_view_lineage:
+            yield from self.get_view_lineage()
+
+    def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
+        builder = SqlParsingBuilder(
+            generate_lineage=True,
+            generate_usage_statistics=False,
+            generate_operations=False,
+        )
+        for dataset_name in self._view_definition_cache.keys():
+            view_definition = self._view_definition_cache[dataset_name]
+            result = self._run_sql_parser(
+                dataset_name,
+                view_definition,
+                self.schema_resolver,
+            )
+            if result and result.out_tables:
+                # This does not yield any workunits but we use
+                # yield here to execute this method
+                yield from builder.process_sql_parsing_result(
+                    result=result,
+                    query=view_definition,
+                    is_view_ddl=True,
+                    include_column_lineage=self.config.include_view_column_lineage,
+                )
+            else:
+                self.views_failed_parsing.add(dataset_name)
+        yield from builder.gen_workunits()
+
     def get_identifier(
         self, *, schema: str, entity: str, inspector: Inspector, **kwargs: Any
     ) -> str:
@@ -661,6 +720,8 @@ def _process_table(
             schema_fields,
         )
         dataset_snapshot.aspects.append(schema_metadata)
+        if self.config.include_view_lineage:
+            self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata)
         db_name = self.get_db_name(inspector)
 
         yield from self.add_table_to_schema_container(
@@ -865,6 +926,12 @@ def _process_view(
         view: str,
         sql_config: SQLCommonConfig,
     ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]:
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            self.platform,
+            dataset_name,
+            self.config.platform_instance,
+            self.config.env,
+        )
         try:
             columns = inspector.get_columns(view, schema)
         except KeyError:
@@ -880,6 +947,8 @@ def _process_view(
                 columns,
                 canonical_schema=schema_fields,
             )
+            if self.config.include_view_lineage:
+                self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata)
         description, properties, _ = self.get_table_properties(inspector, schema, view)
         try:
             view_definition = inspector.get_view_definition(view, schema)
@@ -893,12 +962,9 @@ def _process_view(
             view_definition = ""
         properties["view_definition"] = view_definition
         properties["is_view"] = "True"
-        dataset_urn = make_dataset_urn_with_platform_instance(
-            self.platform,
-            dataset_name,
-            self.config.platform_instance,
-            self.config.env,
-        )
+        if view_definition and self.config.include_view_lineage:
+            self._view_definition_cache[dataset_name] = view_definition
+
         dataset_snapshot = DatasetSnapshot(
             urn=dataset_urn,
             aspects=[StatusClass(removed=False)],
@@ -945,6 +1011,51 @@ def _process_view(
                 domain_registry=self.domain_registry,
             )
 
+    def _run_sql_parser(
+        self, view_identifier: str, query: str, schema_resolver: SchemaResolver
+    ) -> Optional[SqlParsingResult]:
+        try:
+            database, schema = self.get_db_schema(view_identifier)
+        except ValueError:
+            logger.warning(f"Invalid view identifier: {view_identifier}")
+            return None
+        raw_lineage = sqlglot_lineage(
+            query,
+            schema_resolver=schema_resolver,
+            default_db=database,
+            default_schema=schema,
+        )
+        view_urn = make_dataset_urn_with_platform_instance(
+            self.platform,
+            view_identifier,
+            self.config.platform_instance,
+            self.config.env,
+        )
+
+        if raw_lineage.debug_info.table_error:
+            logger.debug(
+                f"Failed to parse lineage for view {view_identifier}: "
+                f"{raw_lineage.debug_info.table_error}"
+            )
+            self.report.num_view_definitions_failed_parsing += 1
+            self.report.view_definitions_parsing_failures.append(
+                f"Table-level sql parsing error for view {view_identifier}: {raw_lineage.debug_info.table_error}"
+            )
+            return None
+
+        elif raw_lineage.debug_info.column_error:
+            self.report.num_view_definitions_failed_column_parsing += 1
+            self.report.view_definitions_parsing_failures.append(
+                f"Column-level sql parsing error for view {view_identifier}: {raw_lineage.debug_info.column_error}"
+            )
+        else:
+            self.report.num_view_definitions_parsed += 1
+        return view_definition_lineage_helper(raw_lineage, view_urn)
+
+    def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
+        database, schema, _view = dataset_identifier.split(".")
+        return database, schema
+
     def get_profiler_instance(self, inspector: Inspector) -> "DatahubGEProfiler":
         from datahub.ingestion.source.ge_data_profiler import DatahubGEProfiler
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 57aae32b361cf..095b8e6443171 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -6,7 +6,7 @@
 from pydantic import Field
 from sqlalchemy.engine import URL
 
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel, LineageConfig
 from datahub.configuration.source_common import (
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
@@ -28,6 +28,7 @@ class SQLCommonConfig(
     StatefulIngestionConfigBase,
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
+    LineageConfig,
 ):
     options: dict = pydantic.Field(
         default_factory=dict,
@@ -70,6 +71,22 @@ class SQLCommonConfig(
         description="If the source supports it, include table lineage to the underlying storage location.",
     )
 
+    include_view_lineage: bool = Field(
+        default=True,
+        description="Populates view->view and table->view lineage using DataHub's sql parser.",
+    )
+
+    include_view_column_lineage: bool = Field(
+        default=True,
+        description="Populates column-level lineage for  view->view and table->view lineage using DataHub's sql parser."
+        " Requires `include_view_lineage` to be enabled.",
+    )
+
+    use_file_backed_cache: bool = Field(
+        default=True,
+        description="Whether to use a file backed cache for the view definitions.",
+    )
+
     profiling: GEProfilingConfig = GEProfilingConfig()
     # Custom Stateful Ingestion settings
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
index 51626891e9fef..3b4a7e1dc0287 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
@@ -1,15 +1,13 @@
 import re
 from typing import Any, Dict, ValuesView
 
-from sqlalchemy import types
-
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     ArrayType,
     BooleanType,
     BytesType,
     DateType,
     EnumType,
-    MapType as MapTypeAvro,
+    MapType,
     NullType,
     NumberType,
     RecordType,
@@ -365,16 +363,10 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "time": TimeType,
     "timestamp": TimeType,
     "row": RecordType,
-    "map": MapTypeAvro,
+    "map": MapType,
     "array": ArrayType,
 }
 
-
-class MapType(types.TupleType):
-    # Wrapper class around SQLalchemy's TupleType to increase compatibility with DataHub
-    pass
-
-
 # https://docs.aws.amazon.com/athena/latest/ug/data-types.html
 # https://github.com/dbt-athena/dbt-athena/tree/main
 ATHENA_SQL_TYPES_MAP: Dict[str, Any] = {
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index e628e4dbd3446..899a7b6697c0a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -1,7 +1,7 @@
 import logging
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Iterable, MutableMapping, Optional, Union
+from typing import Iterable, Optional, Union
 
 # This import verifies that the dependencies are available.
 import teradatasqlalchemy  # noqa: F401
@@ -33,14 +33,11 @@
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
-from datahub.metadata._schema_classes import SchemaMetadataClass, ViewPropertiesClass
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BytesTypeClass,
     TimeTypeClass,
 )
-from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.sqlglot_lineage import SchemaResolver, sqlglot_lineage
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -87,11 +84,6 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         "This requires to have the table lineage feature enabled.",
     )
 
-    include_view_lineage = Field(
-        default=True,
-        description="Whether to include view lineage in the ingestion. "
-        "This requires to have the view lineage feature enabled.",
-    )
     usage: BaseUsageConfig = Field(
         description="The usage config to use when generating usage statistics",
         default=BaseUsageConfig(),
@@ -107,11 +99,6 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         description="Generate usage statistic.",
     )
 
-    use_file_backed_cache: bool = Field(
-        default=True,
-        description="Whether to use a file backed cache for the view definitions.",
-    )
-
 
 @platform_name("Teradata")
 @config_class(TeradataConfig)
@@ -143,8 +130,6 @@ class TeradataSource(TwoTierSQLAlchemySource):
      and "timestamp" < TIMESTAMP '{end_time}'
      """
 
-    _view_definition_cache: MutableMapping[str, str]
-
     def __init__(self, config: TeradataConfig, ctx: PipelineContext):
         super().__init__(config, ctx, "teradata")
 
@@ -167,34 +152,11 @@ def __init__(self, config: TeradataConfig, ctx: PipelineContext):
             env=self.config.env,
         )
 
-        if self.config.use_file_backed_cache:
-            self._view_definition_cache = FileBackedDict[str]()
-        else:
-            self._view_definition_cache = {}
-
     @classmethod
     def create(cls, config_dict, ctx):
         config = TeradataConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
-    def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
-        for key in self._view_definition_cache.keys():
-            view_definition = self._view_definition_cache[key]
-            dataset_urn = DatasetUrn.create_from_string(key)
-
-            db_name: Optional[str] = None
-            # We need to get the default db from the dataset urn otherwise the builder generates the wrong urns
-            if "." in dataset_urn.get_dataset_name():
-                db_name = dataset_urn.get_dataset_name().split(".", 1)[0]
-
-            self.report.num_view_ddl_parsed += 1
-            if self.report.num_view_ddl_parsed % 1000 == 0:
-                logger.info(f"Parsed {self.report.num_queries_parsed} view ddl")
-
-            yield from self.gen_lineage_from_query(
-                query=view_definition, default_database=db_name, is_view_ddl=True
-            )
-
     def get_audit_log_mcps(self) -> Iterable[MetadataWorkUnit]:
         engine = self.get_metadata_engine()
         for entry in engine.execute(
@@ -252,19 +214,7 @@ def get_metadata_engine(self) -> Engine:
 
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
         # Add all schemas to the schema resolver
-        for wu in super().get_workunits_internal():
-            urn = wu.get_urn()
-            schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass)
-            if schema_metadata:
-                self.schema_resolver.add_schema_metadata(urn, schema_metadata)
-            view_properties = wu.get_aspect_of_type(ViewPropertiesClass)
-            if view_properties and self.config.include_view_lineage:
-                self._view_definition_cache[urn] = view_properties.viewLogic
-            yield wu
-
-        if self.config.include_view_lineage:
-            self.report.report_ingestion_stage_start("view lineage extraction")
-            yield from self.get_view_lineage()
+        yield from super().get_workunits_internal()
 
         if self.config.include_table_lineage or self.config.include_usage_statistics:
             self.report.report_ingestion_stage_start("audit log extraction")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
index 7a49551dc1235..efb1d3ffe119f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
@@ -1,6 +1,6 @@
 import typing
 import urllib.parse
-from typing import Any, Dict, Iterable, Optional
+from typing import Any, Dict, Iterable, Optional, Tuple
 
 from pydantic.fields import Field
 from sqlalchemy import create_engine, inspect
@@ -71,6 +71,10 @@ def __init__(self, config, ctx, platform):
         super().__init__(config, ctx, platform)
         self.config: TwoTierSQLAlchemyConfig = config
 
+    def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
+        schema, _view = dataset_identifier.split(".", 1)
+        return None, schema
+
     def get_database_container_key(self, db_name: str, schema: str) -> ContainerKey:
         # Because our overridden get_allowed_schemas method returns db_name as the schema name,
         # the db_name and schema here will be the same. Hence, we just ignore the schema parameter.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
index a417cae2b1ab0..b89db755853bc 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
@@ -86,7 +86,7 @@ class VerticaConfig(BasicSQLAlchemyConfig):
         default=True, description="Whether Models should be ingested."
     )
 
-    include_view_lineage: Optional[bool] = pydantic.Field(
+    include_view_lineage: bool = pydantic.Field(
         default=True,
         description="If the source supports it, include view lineage to the underlying storage location.",
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
index be97e9380f1f5..7fb2cf9813cab 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
@@ -11,7 +11,6 @@
     ConfigModel,
     ConfigurationError,
     DynamicTypedConfig,
-    LineageConfig,
 )
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
@@ -100,7 +99,7 @@ class StatefulIngestionConfigBase(GenericModel, Generic[CustomConfig]):
     )
 
 
-class StatefulLineageConfigMixin(LineageConfig):
+class StatefulLineageConfigMixin:
     enable_stateful_lineage_ingestion: bool = Field(
         default=True,
         description="Enable stateful lineage ingestion."
diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
index 0d72fc52da0ca..c3e8c175f1de5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
@@ -166,13 +166,17 @@ def _check_oauth_config(oauth_config: Optional[OAuthConfiguration]) -> None:
                 "but should be set when using use_certificate false for oauth_config"
             )
 
-    @pydantic.validator("include_view_lineage")
-    def validate_include_view_lineage(cls, v, values):
-        if not values.get("include_table_lineage") and v:
+    @pydantic.root_validator()
+    def validate_include_view_lineage(cls, values):
+        if (
+            "include_table_lineage" in values
+            and not values.get("include_table_lineage")
+            and values.get("include_view_lineage")
+        ):
             raise ValueError(
                 "include_table_lineage must be True for include_view_lineage to be set."
             )
-        return v
+        return values
 
     def get_sql_alchemy_url(
         self,
diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
index b3b1331db768b..2b610947e9043 100644
--- a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
+++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
@@ -24,6 +24,7 @@ def assert_sql_result_with_resolver(
     *,
     expected_file: pathlib.Path,
     schema_resolver: SchemaResolver,
+    allow_table_error: bool = False,
     **kwargs: Any,
 ) -> None:
     # HACK: Our BigQuery source overwrites this value and doesn't undo it.
@@ -36,6 +37,14 @@ def assert_sql_result_with_resolver(
         **kwargs,
     )
 
+    if res.debug_info.table_error:
+        if allow_table_error:
+            logger.info(
+                f"SQL parser table error: {res.debug_info.table_error}",
+                exc_info=res.debug_info.table_error,
+            )
+        else:
+            raise res.debug_info.table_error
     if res.debug_info.column_error:
         logger.warning(
             f"SQL parser column error: {res.debug_info.column_error}",
diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py
index eb2d975ee607f..f91c01d901ac1 100644
--- a/metadata-ingestion/src/datahub/utilities/mapping.py
+++ b/metadata-ingestion/src/datahub/utilities/mapping.py
@@ -4,7 +4,7 @@
 import re
 import time
 from functools import reduce
-from typing import Any, Dict, List, Match, Optional, Union, cast
+from typing import Any, Dict, List, Mapping, Match, Optional, Union, cast
 
 from datahub.emitter import mce_builder
 from datahub.emitter.mce_builder import OwnerType
@@ -111,7 +111,7 @@ def __init__(
         self.owner_source_type = owner_source_type
         self.match_nested_props = match_nested_props
 
-    def process(self, raw_props: Dict[str, Any]) -> Dict[str, Any]:
+    def process(self, raw_props: Mapping[str, Any]) -> Dict[str, Any]:
         # Defining the following local variables -
         # operations_map - the final resulting map when operations are processed.
         # Against each operation the values to be applied are stored.
diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
index a431f262a85fd..5d2fc6872c7bd 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
@@ -4,15 +4,24 @@
 from typing import Any, Dict, List, Optional, Type, Union
 
 from sqlalchemy import types
-from sqlalchemy_bigquery import STRUCT
 
 from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
 from datahub.metadata.schema_classes import NullTypeClass, SchemaFieldDataTypeClass
 
 logger = logging.getLogger(__name__)
 
+try:
+    # This is used for both BigQuery and Athena.
+    from sqlalchemy_bigquery import STRUCT
+except ImportError:
+    STRUCT = None
+
+
+class MapType(types.TupleType):
+    # Wrapper class around SQLalchemy's TupleType to increase compatibility with DataHub
+    pass
+
 
 class SqlAlchemyColumnToAvroConverter:
     """Helper class that collects some methods to convert SQLalchemy columns to Avro schema."""
@@ -38,7 +47,9 @@ def get_avro_type(
     ) -> Dict[str, Any]:
         """Determines the concrete AVRO schema type for a SQLalchemy-typed column"""
 
-        if type(column_type) in cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE.keys():
+        if isinstance(
+            column_type, tuple(cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE.keys())
+        ):
             return {
                 "type": cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE[type(column_type)],
                 "native_data_type": str(column_type),
@@ -84,7 +95,7 @@ def get_avro_type(
                 "key_type": cls.get_avro_type(column_type=key_type, nullable=nullable),
                 "key_native_data_type": str(key_type),
             }
-        if isinstance(column_type, STRUCT):
+        if STRUCT and isinstance(column_type, STRUCT):
             fields = []
             for field_def in column_type._STRUCT_fields:
                 field_name, field_type = field_def
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index c830ec8c02fd4..1d74b20569814 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -241,9 +241,9 @@ class SqlParsingResult(_ParserBaseModel):
     )
 
 
-def _parse_statement(sql: str, dialect: str) -> sqlglot.Expression:
-    statement = sqlglot.parse_one(
-        sql, read=dialect, error_level=sqlglot.ErrorLevel.RAISE
+def _parse_statement(sql: sqlglot.exp.ExpOrStr, dialect: str) -> sqlglot.Expression:
+    statement: sqlglot.Expression = sqlglot.maybe_parse(
+        sql, dialect=dialect, error_level=sqlglot.ErrorLevel.RAISE
     )
     return statement
 
@@ -467,14 +467,20 @@ def _column_level_lineage(  # noqa: C901
     default_db: Optional[str],
     default_schema: Optional[str],
 ) -> List[_ColumnLineageInfo]:
-    if not isinstance(
-        statement,
-        _SupportedColumnLineageTypesTuple,
+    is_create_ddl = _is_create_table_ddl(statement)
+    if (
+        not isinstance(
+            statement,
+            _SupportedColumnLineageTypesTuple,
+        )
+        and not is_create_ddl
     ):
         raise UnsupportedStatementTypeError(
             f"Can only generate column-level lineage for select-like inner statements, not {type(statement)}"
         )
 
+    column_lineage: List[_ColumnLineageInfo] = []
+
     use_case_insensitive_cols = dialect in {
         # Column identifiers are case-insensitive in BigQuery, so we need to
         # do a normalization step beforehand to make sure it's resolved correctly.
@@ -580,16 +586,46 @@ def _schema_aware_fuzzy_column_resolve(
         ) from e
     logger.debug("Qualified sql %s", statement.sql(pretty=True, dialect=dialect))
 
+    # Handle the create DDL case.
+    if is_create_ddl:
+        assert (
+            output_table is not None
+        ), "output_table must be set for create DDL statements"
+
+        create_schema: sqlglot.exp.Schema = statement.this
+        sqlglot_columns = create_schema.expressions
+
+        for column_def in sqlglot_columns:
+            if not isinstance(column_def, sqlglot.exp.ColumnDef):
+                # Ignore things like constraints.
+                continue
+
+            output_col = _schema_aware_fuzzy_column_resolve(
+                output_table, column_def.name
+            )
+            output_col_type = column_def.args.get("kind")
+
+            column_lineage.append(
+                _ColumnLineageInfo(
+                    downstream=_DownstreamColumnRef(
+                        table=output_table,
+                        column=output_col,
+                        column_type=output_col_type,
+                    ),
+                    upstreams=[],
+                )
+            )
+
+        return column_lineage
+
     # Try to figure out the types of the output columns.
     try:
         statement = sqlglot.optimizer.annotate_types.annotate_types(
             statement, schema=sqlglot_db_schema
         )
-    except sqlglot.errors.OptimizeError as e:
+    except (sqlglot.errors.OptimizeError, sqlglot.errors.ParseError) as e:
         # This is not a fatal error, so we can continue.
-        logger.debug("sqlglot failed to annotate types: %s", e)
-
-    column_lineage = []
+        logger.debug("sqlglot failed to annotate or parse types: %s", e)
 
     try:
         assert isinstance(statement, _SupportedColumnLineageTypesTuple)
@@ -599,7 +635,6 @@ def _schema_aware_fuzzy_column_resolve(
             (select_col.alias_or_name, select_col) for select_col in statement.selects
         ]
         logger.debug("output columns: %s", [col[0] for col in output_columns])
-        output_col: str
         for output_col, original_col_expression in output_columns:
             if output_col == "*":
                 # If schema information is available, the * will be expanded to the actual columns.
@@ -628,7 +663,7 @@ def _schema_aware_fuzzy_column_resolve(
 
             # Generate SELECT lineage.
             # Using a set here to deduplicate upstreams.
-            direct_col_upstreams: Set[_ColumnRef] = set()
+            direct_raw_col_upstreams: Set[_ColumnRef] = set()
             for node in lineage_node.walk():
                 if node.downstream:
                     # We only want the leaf nodes.
@@ -643,8 +678,9 @@ def _schema_aware_fuzzy_column_resolve(
                     if node.subfield:
                         normalized_col = f"{normalized_col}.{node.subfield}"
 
-                    col = _schema_aware_fuzzy_column_resolve(table_ref, normalized_col)
-                    direct_col_upstreams.add(_ColumnRef(table=table_ref, column=col))
+                    direct_raw_col_upstreams.add(
+                        _ColumnRef(table=table_ref, column=normalized_col)
+                    )
                 else:
                     # This branch doesn't matter. For example, a count(*) column would go here, and
                     # we don't get any column-level lineage for that.
@@ -665,7 +701,16 @@ def _schema_aware_fuzzy_column_resolve(
             if original_col_expression.type:
                 output_col_type = original_col_expression.type
 
-            if not direct_col_upstreams:
+            # Fuzzy resolve upstream columns.
+            direct_resolved_col_upstreams = {
+                _ColumnRef(
+                    table=edge.table,
+                    column=_schema_aware_fuzzy_column_resolve(edge.table, edge.column),
+                )
+                for edge in direct_raw_col_upstreams
+            }
+
+            if not direct_resolved_col_upstreams:
                 logger.debug(f'  "{output_col}" has no upstreams')
             column_lineage.append(
                 _ColumnLineageInfo(
@@ -674,12 +719,12 @@ def _schema_aware_fuzzy_column_resolve(
                         column=output_col,
                         column_type=output_col_type,
                     ),
-                    upstreams=sorted(direct_col_upstreams),
+                    upstreams=sorted(direct_resolved_col_upstreams),
                     # logic=column_logic.sql(pretty=True, dialect=dialect),
                 )
             )
 
-        # TODO: Also extract referenced columns (e.g. non-SELECT lineage)
+        # TODO: Also extract referenced columns (aka auxillary / non-SELECT lineage)
     except (sqlglot.errors.OptimizeError, ValueError) as e:
         raise SqlUnderstandingError(
             f"sqlglot failed to compute some lineage: {e}"
@@ -700,6 +745,53 @@ def _extract_select_from_create(
         return statement
 
 
+_UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT: Set[str] = set(
+    sqlglot.exp.Update.arg_types.keys()
+) - set(sqlglot.exp.Select.arg_types.keys())
+
+
+def _extract_select_from_update(
+    statement: sqlglot.exp.Update,
+) -> sqlglot.exp.Select:
+    statement = statement.copy()
+
+    # The "SET" expressions need to be converted.
+    # For the update command, it'll be a list of EQ expressions, but the select
+    # should contain aliased columns.
+    new_expressions = []
+    for expr in statement.expressions:
+        if isinstance(expr, sqlglot.exp.EQ) and isinstance(
+            expr.left, sqlglot.exp.Column
+        ):
+            new_expressions.append(
+                sqlglot.exp.Alias(
+                    this=expr.right,
+                    alias=expr.left.this,
+                )
+            )
+        else:
+            # If we don't know how to convert it, just leave it as-is. If this causes issues,
+            # they'll get caught later.
+            new_expressions.append(expr)
+
+    return sqlglot.exp.Select(
+        **{
+            **{
+                k: v
+                for k, v in statement.args.items()
+                if k not in _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
+            },
+            "expressions": new_expressions,
+        }
+    )
+
+
+def _is_create_table_ddl(statement: sqlglot.exp.Expression) -> bool:
+    return isinstance(statement, sqlglot.exp.Create) and isinstance(
+        statement.this, sqlglot.exp.Schema
+    )
+
+
 def _try_extract_select(
     statement: sqlglot.exp.Expression,
 ) -> sqlglot.exp.Expression:
@@ -716,6 +808,9 @@ def _try_extract_select(
     elif isinstance(statement, sqlglot.exp.Insert):
         # TODO Need to map column renames in the expressions part of the statement.
         statement = statement.expression
+    elif isinstance(statement, sqlglot.exp.Update):
+        # Assumption: the output table is already captured in the modified tables list.
+        statement = _extract_select_from_update(statement)
     elif isinstance(statement, sqlglot.exp.Create):
         # TODO May need to map column renames.
         # Assumption: the output table is already captured in the modified tables list.
@@ -766,6 +861,7 @@ def _translate_sqlglot_type(
 def _translate_internal_column_lineage(
     table_name_urn_mapping: Dict[_TableName, str],
     raw_column_lineage: _ColumnLineageInfo,
+    dialect: str,
 ) -> ColumnLineageInfo:
     downstream_urn = None
     if raw_column_lineage.downstream.table:
@@ -779,7 +875,9 @@ def _translate_internal_column_lineage(
             )
             if raw_column_lineage.downstream.column_type
             else None,
-            native_column_type=raw_column_lineage.downstream.column_type.sql()
+            native_column_type=raw_column_lineage.downstream.column_type.sql(
+                dialect=dialect
+            )
             if raw_column_lineage.downstream.column_type
             and raw_column_lineage.downstream.column_type.this
             != sqlglot.exp.DataType.Type.UNKNOWN
@@ -800,12 +898,14 @@ def _get_dialect(platform: str) -> str:
     # TODO: convert datahub platform names to sqlglot dialect
     if platform == "presto-on-hive":
         return "hive"
+    if platform == "mssql":
+        return "tsql"
     else:
         return platform
 
 
 def _sqlglot_lineage_inner(
-    sql: str,
+    sql: sqlglot.exp.ExpOrStr,
     schema_resolver: SchemaResolver,
     default_db: Optional[str] = None,
     default_schema: Optional[str] = None,
@@ -886,19 +986,25 @@ def _sqlglot_lineage_inner(
     )
 
     # Simplify the input statement for column-level lineage generation.
-    select_statement = _try_extract_select(statement)
+    try:
+        select_statement = _try_extract_select(statement)
+    except Exception as e:
+        logger.debug(f"Failed to extract select from statement: {e}", exc_info=True)
+        debug_info.column_error = e
+        select_statement = None
 
     # Generate column-level lineage.
     column_lineage: Optional[List[_ColumnLineageInfo]] = None
     try:
-        column_lineage = _column_level_lineage(
-            select_statement,
-            dialect=dialect,
-            input_tables=table_name_schema_mapping,
-            output_table=downstream_table,
-            default_db=default_db,
-            default_schema=default_schema,
-        )
+        if select_statement is not None:
+            column_lineage = _column_level_lineage(
+                select_statement,
+                dialect=dialect,
+                input_tables=table_name_schema_mapping,
+                output_table=downstream_table,
+                default_db=default_db,
+                default_schema=default_schema,
+            )
     except UnsupportedStatementTypeError as e:
         # Inject details about the outer statement type too.
         e.args = (f"{e.args[0]} (outer statement type: {type(statement)})",)
@@ -918,7 +1024,7 @@ def _sqlglot_lineage_inner(
     if column_lineage:
         column_lineage_urns = [
             _translate_internal_column_lineage(
-                table_name_urn_mapping, internal_col_lineage
+                table_name_urn_mapping, internal_col_lineage, dialect=dialect
             )
             for internal_col_lineage in column_lineage
         ]
@@ -1050,3 +1156,20 @@ def create_lineage_sql_parsed_result(
     finally:
         if needs_close:
             schema_resolver.close()
+
+
+def view_definition_lineage_helper(
+    result: SqlParsingResult, view_urn: str
+) -> SqlParsingResult:
+    if result.query_type is QueryType.SELECT:
+        # Some platforms (e.g. postgres) store only <select statement> from view definition
+        # `create view V as <select statement>` . For such view definitions, `result.out_tables` and
+        # `result.column_lineage[].downstream` are empty in `sqlglot_lineage` response, whereas upstream
+        # details and downstream column details are extracted correctly.
+        # Here, we inject view V's urn in `result.out_tables` and `result.column_lineage[].downstream`
+        # to get complete lineage result.
+        result.out_tables = [view_urn]
+        if result.column_lineage:
+            for col_result in result.column_lineage:
+                col_result.downstream.table = view_urn
+    return result
diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
index f3b6d2b8138cc..6774d4c7055b9 100644
--- a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
+++ b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -93,7 +98,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -111,7 +117,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore",
@@ -121,7 +127,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -187,7 +193,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,7 +211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -224,7 +232,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -239,7 +248,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -257,17 +267,19 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: COLUMN_STATS_ACCURATE": "{\\\"BASIC_STATS\\\":\\\"true\\\"}",
+                            "Table Parameters: another.comment": "This table has no partitions",
+                            "Table Parameters: comment": "This table has array of structs",
                             "Table Parameters: numFiles": "1",
                             "Table Parameters: numRows": "1",
                             "Table Parameters: rawDataSize": "32",
                             "Table Parameters: totalSize": "33",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721976",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -278,6 +290,7 @@
                             "Storage Desc Params: serialization.format": "1"
                         },
                         "name": "array_struct_test",
+                        "description": "This table has array of structs",
                         "tags": []
                     }
                 },
@@ -304,6 +317,7 @@
                             {
                                 "fieldPath": "property_id",
                                 "nullable": true,
+                                "description": "id of property",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -316,6 +330,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
                                 "nullable": true,
+                                "description": "service types and providers",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.ArrayType": {
@@ -368,7 +383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -385,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -405,7 +422,189 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Table Type:": "VIRTUAL_VIEW",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "null",
+                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.array_struct_test",
+                            "View Expanded Text:": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "array_struct_test_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.array_struct_test_view",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "record"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<struct<type:string,provider:array<int>>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<struct<type:string,provider:array<int>>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -420,7 +619,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -438,7 +638,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test",
@@ -448,7 +648,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -518,7 +718,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -535,7 +736,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -555,7 +757,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -570,7 +773,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -588,7 +792,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test",
@@ -598,7 +802,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -717,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -734,7 +939,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -754,7 +960,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -769,7 +976,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -787,16 +995,17 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:22 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:08 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: numFiles": "1",
+                            "Table Parameters: numPartitions": "1",
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "5812",
-                            "Table Parameters: transient_lastDdlTime": "1688578704",
+                            "Table Parameters: transient_lastDdlTime": "1697721968",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -853,6 +1062,18 @@
                                 "nativeDataType": "string",
                                 "recursive": false,
                                 "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "baz",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false
                             }
                         ]
                     }
@@ -862,7 +1083,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -879,7 +1101,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +1122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +1138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -932,7 +1157,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test",
@@ -942,7 +1167,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -1039,7 +1264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1056,7 +1282,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1076,7 +1303,188 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized",
+                            "Table Type:": "MATERIALIZED_VIEW",
+                            "Table Parameters: numFiles": "0",
+                            "Table Parameters: totalSize": "0",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.struct_test",
+                            "View Expanded Text:": "null",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "struct_test_view_materialized",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.struct_test_view_materialized",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
+                                    }
+                                },
+                                "nativeDataType": "struct<type:string,provider:array<int>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"struct<type:string,provider:array<int>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1091,7 +1499,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1109,7 +1518,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test",
@@ -1119,10 +1528,10 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
-                            "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
-                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
-                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
                             "Compressed:": "No",
                             "Num Buckets:": "-1",
                             "Bucket Columns:": "[]",
@@ -1285,7 +1694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1302,7 +1712,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1322,7 +1733,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW `db1.array_struct_test_view` AS select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1342,7 +1772,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1357,7 +1788,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1372,7 +1804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1389,7 +1822,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1404,7 +1838,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1419,7 +1854,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1437,7 +1873,7 @@
                         "customProperties": {
                             "Database:": "db2",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:24 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:10 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db2.db/pokes",
@@ -1446,7 +1882,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "5812",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721971",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -1454,10 +1890,7 @@
                             "Num Buckets:": "-1",
                             "Bucket Columns:": "[]",
                             "Sort Columns:": "[]",
-                            "Storage Desc Params: serialization.format": "1",
-                            "Table:": "db2.pokes",
-                            "Constraint Name:": "pk_1173723383_1683022998392_0",
-                            "Column Names:": "foo"
+                            "Storage Desc Params: serialization.format": "1"
                         },
                         "name": "pokes",
                         "tags": []
@@ -1515,7 +1948,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1552,7 +1987,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1572,7 +2008,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1587,7 +2024,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1602,7 +2040,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1619,7 +2058,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1634,7 +2074,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json
index 08f281f398909..e93924049f626 100644
--- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json
+++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -93,7 +98,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -111,7 +117,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore",
@@ -121,7 +127,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -187,7 +193,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,7 +211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -224,7 +232,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -239,7 +248,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -257,17 +267,19 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: COLUMN_STATS_ACCURATE": "{\\\"BASIC_STATS\\\":\\\"true\\\"}",
+                            "Table Parameters: another.comment": "This table has no partitions",
+                            "Table Parameters: comment": "This table has array of structs",
                             "Table Parameters: numFiles": "1",
                             "Table Parameters: numRows": "1",
                             "Table Parameters: rawDataSize": "32",
                             "Table Parameters: totalSize": "33",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721976",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -278,6 +290,7 @@
                             "Storage Desc Params: serialization.format": "1"
                         },
                         "name": "array_struct_test",
+                        "description": "This table has array of structs",
                         "tags": []
                     }
                 },
@@ -304,6 +317,7 @@
                             {
                                 "fieldPath": "property_id",
                                 "nullable": true,
+                                "description": "id of property",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -316,6 +330,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
                                 "nullable": true,
+                                "description": "service types and providers",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.ArrayType": {
@@ -368,7 +383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -385,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -405,7 +422,189 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Table Type:": "VIRTUAL_VIEW",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "null",
+                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.array_struct_test",
+                            "View Expanded Text:": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "array_struct_test_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.array_struct_test_view",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "record"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<struct<type:string,provider:array<int>>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<struct<type:string,provider:array<int>>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -420,7 +619,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -438,7 +638,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test",
@@ -448,7 +648,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -518,7 +718,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -535,7 +736,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -555,7 +757,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -570,7 +773,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -588,7 +792,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test",
@@ -598,7 +802,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -717,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -734,7 +939,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -754,7 +960,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -769,7 +976,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -787,16 +995,17 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:22 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:08 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: numFiles": "1",
+                            "Table Parameters: numPartitions": "1",
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "5812",
-                            "Table Parameters: transient_lastDdlTime": "1688578704",
+                            "Table Parameters: transient_lastDdlTime": "1697721968",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -853,6 +1062,18 @@
                                 "nativeDataType": "string",
                                 "recursive": false,
                                 "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "baz",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false
                             }
                         ]
                     }
@@ -862,7 +1083,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -879,7 +1101,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +1122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +1138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -932,7 +1157,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test",
@@ -942,7 +1167,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -1039,7 +1264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1056,7 +1282,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1076,7 +1303,188 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized",
+                            "Table Type:": "MATERIALIZED_VIEW",
+                            "Table Parameters: numFiles": "0",
+                            "Table Parameters: totalSize": "0",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.struct_test",
+                            "View Expanded Text:": "null",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "struct_test_view_materialized",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.struct_test_view_materialized",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
+                                    }
+                                },
+                                "nativeDataType": "struct<type:string,provider:array<int>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"struct<type:string,provider:array<int>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1091,7 +1499,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1109,7 +1518,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test",
@@ -1119,10 +1528,10 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
-                            "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
-                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
-                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
                             "Compressed:": "No",
                             "Num Buckets:": "-1",
                             "Bucket Columns:": "[]",
@@ -1285,7 +1694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1302,7 +1712,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1322,7 +1733,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW `db1.array_struct_test_view` AS select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/hive/hive_setup.sql b/metadata-ingestion/tests/integration/hive/hive_setup.sql
index 8fb8498894bc0..323a78e24d10b 100644
--- a/metadata-ingestion/tests/integration/hive/hive_setup.sql
+++ b/metadata-ingestion/tests/integration/hive/hive_setup.sql
@@ -1,10 +1,10 @@
 CREATE DATABASE IF NOT EXISTS db1;
 CREATE DATABASE IF NOT EXISTS db2;
 -- Setup a "pokes" example table.
-CREATE TABLE IF NOT EXISTS db1.pokes (foo INT, bar STRING);
-LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE db1.pokes;
+CREATE TABLE IF NOT EXISTS db1.pokes (foo INT, bar STRING) PARTITIONED BY (baz STRING);
+LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE db1.pokes PARTITION (baz='dummy');
 
-CREATE TABLE IF NOT EXISTS db2.pokes (foo INT, bar STRING, CONSTRAINT pk_1173723383_1683022998392_0 primary key(foo) DISABLE NOVALIDATE NORELY);
+CREATE TABLE IF NOT EXISTS db2.pokes (foo INT, bar STRING);
 LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE db2.pokes;
 
 -- Setup a table with a special character.
@@ -23,12 +23,12 @@ CREATE TABLE IF NOT EXISTS db1.struct_test
 
 CREATE TABLE IF NOT EXISTS db1.array_struct_test
 (
- property_id INT,
+ property_id INT COMMENT 'id of property',
  service array<STRUCT<
                 type: STRING
                ,provider: ARRAY<INT>
-               >>
-);
+               >> COMMENT 'service types and providers'
+)  TBLPROPERTIES ('comment' = 'This table has array of structs', 'another.comment' = 'This table has no partitions');;
 
 WITH
 test_data as (
@@ -39,6 +39,9 @@ test_data as (
 INSERT INTO TABLE db1.array_struct_test
 select * from test_data;
 
+CREATE MATERIALIZED VIEW db1.struct_test_view_materialized as select * from db1.struct_test;
+CREATE VIEW db1.array_struct_test_view as select * from db1.array_struct_test;
+
 CREATE TABLE IF NOT EXISTS db1.nested_struct_test
 (
  property_id INT,
@@ -50,9 +53,6 @@ CREATE TABLE IF NOT EXISTS db1.nested_struct_test
 
 CREATE TABLE db1.union_test(
     foo UNIONTYPE<int, double, array<string>, struct<a:int,b:string>, struct<c:int,d:double>>
-);
+) STORED AS ORC ;
 
-CREATE TABLE db1.map_test(
-    KeyValue String, 
-    RecordId map<int,string>
-); 
\ No newline at end of file
+CREATE TABLE db1.map_test(KeyValue String, RecordId map<int,string>); 
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
index 1f662cfe514e2..e16101b137ac9 100644
--- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
+++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
@@ -2,7 +2,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.emptyCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -41,7 +41,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.firstCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -345,7 +345,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.largeCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -3988,7 +3988,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.secondCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -4135,7 +4135,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.emptyCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4150,7 +4150,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.firstCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4165,7 +4165,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.largeCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4180,7 +4180,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.secondCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py
index 5228c21223e24..56fb471d4c9f1 100644
--- a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py
+++ b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py
@@ -25,6 +25,7 @@ def test_mongodb_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time
                         "username": "mongoadmin",
                         "password": "examplepass",
                         "maxDocumentSize": 25000,
+                        "platform_instance": "instance",
                     },
                 },
                 "sink": {
diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
index 4aaefb48d33e1..38b03ce238d1c 100644
--- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
+++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -93,7 +98,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -213,7 +219,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -230,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -250,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -265,7 +274,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -361,7 +371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -378,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -398,7 +410,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +567,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -969,7 +983,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -989,7 +1004,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1004,7 +1020,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1019,7 +1036,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1036,7 +1054,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1053,7 +1072,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1068,7 +1088,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1083,7 +1104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1215,7 +1237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1232,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1249,7 +1273,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1269,7 +1294,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1284,7 +1310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1445,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1435,7 +1463,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1452,7 +1481,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1472,7 +1502,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1487,7 +1518,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1586,7 +1618,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1603,7 +1636,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1620,7 +1654,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1637,7 +1672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1657,7 +1693,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1677,7 +1714,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1692,7 +1730,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1707,7 +1746,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1724,7 +1764,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1739,7 +1780,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1754,7 +1796,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1874,7 +1917,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1891,7 +1935,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1911,7 +1956,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1926,7 +1972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2022,7 +2069,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2039,7 +2087,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2059,7 +2108,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2182,7 +2232,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2233,7 +2284,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2253,7 +2305,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2268,7 +2321,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2283,7 +2337,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2300,7 +2355,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2315,7 +2371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2330,7 +2387,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2390,7 +2448,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2407,7 +2466,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2427,7 +2487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2442,7 +2503,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2502,7 +2564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2519,7 +2582,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2539,7 +2603,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2568,7 +2633,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2597,7 +2663,79 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),path)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json
index 535ce964c6058..b9b2a3b2141a8 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json
+++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -99,7 +104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -181,7 +191,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -201,7 +212,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -216,7 +228,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -231,7 +244,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -248,7 +262,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -263,7 +278,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -284,7 +300,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -299,7 +316,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -314,7 +332,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -331,7 +350,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -346,7 +366,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -366,7 +387,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -381,7 +403,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -537,7 +560,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +578,186 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+                            "is_view": "True"
+                        },
+                        "name": "metadata_aspect_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "postgrestest.public.metadata_aspect_view",
+                        "platform": "urn:li:dataPlatform:postgres",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "urn",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=500)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "aspect",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=200)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -634,31 +837,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
-                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
-                },
-                {
-                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
-                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -675,29 +855,39 @@
                         "actor": "urn:li:corpuser:unknown"
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
-                    "type": "TRANSFORMED"
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml b/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml
index b390d9246677e..2bfa39a65363b 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml
+++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml
@@ -25,7 +25,7 @@ source:
       include_field_distinct_value_frequencies: false
       include_field_histogram: false
       catch_exceptions: true
-    include_views: false
+    include_views: true
 sink:
   type: file
   config:
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json
index bf36a39a8c103..f6fa0a0ed032e 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json
+++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -99,7 +104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -181,7 +191,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -196,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -352,7 +364,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +382,186 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+                            "is_view": "True"
+                        },
+                        "name": "metadata_aspect_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "postgrestest.public.metadata_aspect_view",
+                        "platform": "urn:li:dataPlatform:postgres",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "urn",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=500)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "aspect",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=200)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -391,31 +583,57 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
-                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
                 },
                 {
-                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
-                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml b/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml
index a489877d52a23..4a2cc543f2d01 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml
+++ b/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml
@@ -13,7 +13,7 @@ source:
       profile_table_row_count_estimate_only: true
       turn_off_expensive_profiling_metrics: true
       catch_exceptions: true
-    include_views: false
+    include_views: true
 sink:
   type: file
   config:
diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py
index 81e307a78ae9e..ff448eca01071 100644
--- a/metadata-ingestion/tests/integration/snowflake/common.py
+++ b/metadata-ingestion/tests/integration/snowflake/common.py
@@ -94,7 +94,7 @@ def default_query_results(  # noqa: C901
                 "name": "VIEW_{}".format(view_idx),
                 "created_on": datetime(2021, 6, 8, 0, 0, 0, 0),
                 "comment": "Comment for View",
-                "text": None,
+                "text": f"create view view_{view_idx} as select * from table_{view_idx}",
             }
             for view_idx in range(1, num_views + 1)
         ]
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
index a424b258e68ff..c7273fee5a2e5 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
@@ -24,7 +24,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -39,7 +40,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -54,7 +56,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -71,7 +74,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -90,7 +94,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -105,7 +110,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -121,7 +127,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -150,7 +157,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -165,7 +173,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -180,7 +189,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -197,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -216,7 +227,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -231,7 +243,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -251,7 +264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -462,7 +477,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -488,7 +504,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -503,7 +520,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -520,7 +538,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -544,7 +563,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -559,7 +579,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -755,7 +776,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +819,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -813,7 +837,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -837,7 +862,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -852,7 +878,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1048,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1074,7 +1102,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1089,7 +1118,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1106,7 +1136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1130,7 +1161,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1145,7 +1177,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1341,7 +1374,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1367,7 +1401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1382,7 +1417,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1399,7 +1435,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1423,7 +1460,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1438,7 +1476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1634,7 +1673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1660,7 +1700,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1675,7 +1716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1692,7 +1734,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1716,7 +1759,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1731,7 +1775,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1927,7 +1972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1953,7 +1999,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1968,7 +2015,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1985,7 +2033,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2009,7 +2058,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2024,7 +2074,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2220,7 +2271,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2246,7 +2298,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2261,7 +2314,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2278,7 +2332,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2302,7 +2357,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2317,7 +2373,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2513,7 +2570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2539,7 +2597,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2554,7 +2613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2571,7 +2631,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2595,7 +2656,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2610,7 +2672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2806,7 +2869,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2832,7 +2896,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2847,7 +2912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2864,7 +2930,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2888,7 +2955,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2903,7 +2971,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3099,7 +3168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3125,7 +3195,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3140,7 +3211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3157,7 +3229,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3181,7 +3254,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3197,7 +3271,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3212,7 +3287,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3382,7 +3458,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3408,7 +3485,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3423,7 +3501,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3440,7 +3519,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_1 as select * from table_1",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3464,7 +3562,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3480,7 +3579,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3496,7 +3596,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3512,7 +3613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3527,7 +3629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3690,7 +3793,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3716,7 +3820,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3731,7 +3836,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3748,7 +3854,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3773,7 +3880,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3797,7 +3905,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_2 as select * from table_2",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3819,7 +3946,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3841,7 +3969,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3863,7 +3992,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3885,7 +4015,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3907,7 +4038,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3929,7 +4061,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3951,7 +4084,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3973,7 +4107,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3995,7 +4130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4017,7 +4153,145 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4034,14 +4308,127 @@
                         "actor": "urn:li:corpuser:unknown"
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4204,7 +4591,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4340,7 +4728,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4476,7 +4865,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4612,7 +5002,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4748,7 +5139,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4884,7 +5276,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5020,7 +5413,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5156,7 +5550,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5292,7 +5687,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5428,7 +5824,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5456,7 +5853,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5484,7 +5882,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5512,7 +5911,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5540,7 +5940,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5568,7 +5969,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5596,7 +5998,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5624,7 +6027,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5652,7 +6056,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5680,7 +6085,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5708,7 +6114,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5736,7 +6143,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5764,7 +6172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5786,7 +6195,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5808,7 +6218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5830,7 +6241,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5852,7 +6264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5874,7 +6287,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5896,7 +6310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5918,7 +6333,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5940,7 +6356,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5962,7 +6379,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5984,7 +6402,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5999,7 +6418,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6014,7 +6434,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6029,7 +6450,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6044,7 +6466,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6059,7 +6482,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
index 5057dacd5b0c8..5e55860483d24 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
@@ -1,13 +1,14 @@
 [
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "snowflake",
+                "instance": "instance1",
                 "env": "PROD",
                 "database": "test_db"
             },
@@ -29,13 +30,14 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "removed": false
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -46,12 +48,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:snowflake"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -61,15 +63,13 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Database"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -80,12 +80,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": []
+            "typeNames": [
+                "Database"
+            ]
         }
     },
     "systemMetadata": {
@@ -96,26 +98,17 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "containerProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "platform": "snowflake",
-                "env": "PROD",
-                "database": "test_db",
-                "schema": "test_schema"
-            },
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/",
-            "name": "TEST_SCHEMA",
-            "description": "comment for TEST_DB.TEST_SCHEMA",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            }
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -126,12 +119,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "removed": false
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -141,13 +135,24 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:snowflake"
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
+            "name": "TABLE_3",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -157,14 +162,14 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "Schema"
+                "Table"
             ]
         }
     },
@@ -175,32 +180,24 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -212,13 +209,28 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "containerProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "platform": "snowflake",
+                "instance": "instance1",
+                "env": "PROD",
+                "database": "test_db",
+                "schema": "test_schema"
+            },
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/",
+            "name": "TEST_SCHEMA",
+            "description": "comment for TEST_DB.TEST_SCHEMA",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            }
         }
     },
     "systemMetadata": {
@@ -229,12 +241,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_1",
+            "schemaName": "test_db.test_schema.table_3",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -392,24 +404,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
-            "name": "TABLE_1",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -420,12 +421,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -435,14 +436,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "Table"
+                "Schema"
             ]
         }
     },
@@ -453,20 +454,53 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 }
             ]
         }
@@ -479,7 +513,85 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_1 as select * from table_1",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/",
+            "name": "VIEW_1",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1",
+            "description": "Comment for View",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -495,12 +607,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_2",
+            "schemaName": "test_db.test_schema.table_1",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -659,23 +771,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
-            "name": "TABLE_2",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "typeNames": [
+                "View"
+            ]
         }
     },
     "systemMetadata": {
@@ -686,12 +789,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -702,39 +805,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
+            "name": "TABLE_1",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -745,12 +832,25 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -761,12 +861,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_3",
+            "schemaName": "test_db.test_schema.view_1",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -925,23 +1025,43 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
-            "name": "TABLE_3",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -952,12 +1072,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -968,14 +1088,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -986,19 +1105,131 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
+            "upstreams": [
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
@@ -1011,7 +1242,40 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1027,12 +1291,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_4",
+            "schemaName": "test_db.test_schema.table_2",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -1191,15 +1455,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
-            "name": "TABLE_4",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
+            "name": "TABLE_10",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
             "description": "Comment for Table",
             "created": {
                 "time": 1623110400000
@@ -1218,66 +1482,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1293,7 +1498,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
@@ -1457,7 +1662,39 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -1484,12 +1721,41 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
+            "name": "TABLE_2",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -1500,7 +1766,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1518,19 +1784,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -1543,12 +1813,25 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1559,12 +1842,76 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "schemaMetadata",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_6",
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "test_db.test_schema.table_10",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -1723,93 +2070,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
-            "name": "TABLE_6",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1825,12 +2086,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_7",
+            "schemaName": "test_db.test_schema.table_6",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -1989,15 +2250,48 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
-            "name": "TABLE_7",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
+            "name": "TABLE_6",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
             "description": "Comment for Table",
             "created": {
                 "time": 1623110400000
@@ -2016,12 +2310,29 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2032,7 +2343,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2050,19 +2361,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -2075,7 +2390,40 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2091,12 +2439,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_8",
+            "schemaName": "test_db.test_schema.table_4",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -2255,15 +2603,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
-            "name": "TABLE_8",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
+            "name": "TABLE_7",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
             "description": "Comment for Table",
             "created": {
                 "time": 1623110400000
@@ -2282,12 +2630,39 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
+            "name": "TABLE_4",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -2298,7 +2673,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2316,19 +2691,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -2341,12 +2720,43 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2357,12 +2767,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_9",
+            "schemaName": "test_db.test_schema.table_7",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -2521,93 +2931,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
-            "name": "TABLE_9",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2623,12 +2947,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_10",
+            "schemaName": "test_db.test_schema.table_8",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -2787,23 +3111,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
-            "name": "TABLE_10",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2814,12 +3128,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -2830,14 +3144,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
+            "name": "TABLE_8",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -2848,21 +3171,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -2873,21 +3187,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2898,20 +3204,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -2923,19 +3222,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -2948,21 +3251,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -2973,21 +3267,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2998,21 +3284,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
+            "name": "TABLE_9",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -3023,20 +3311,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -3048,19 +3329,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -3073,20 +3358,300 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "schemaName": "test_db.test_schema.table_9",
+            "platform": "urn:li:dataPlatform:snowflake",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "col_1",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "NUMBER(38,0)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_2",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_3",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_4",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_5",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_6",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_7",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_8",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_9",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_10",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_2 as select * from table_2",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/",
+            "name": "VIEW_2",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2",
+            "description": "Comment for View",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
             ]
         }
     },
@@ -3098,7 +3663,187 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "test_db.test_schema.view_2",
+            "platform": "urn:li:dataPlatform:snowflake",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "col_1",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "NUMBER(38,0)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_2",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_3",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_4",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_5",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_6",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_7",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_8",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_9",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_10",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -3109,8 +3854,120 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
index 3dafe85ef950a..4c00e48ede9fb 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
@@ -211,11 +211,12 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_
                         include_technical_schema=True,
                         include_table_lineage=True,
                         include_column_lineage=False,
-                        include_views=False,
-                        include_view_lineage=False,
+                        include_views=True,
+                        include_view_lineage=True,
                         include_usage_stats=False,
                         incremental_lineage=False,
                         include_operational_stats=False,
+                        platform_instance="instance1",
                         start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace(
                             tzinfo=timezone.utc
                         ),
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
index cd53b8f7db4f6..4b0dd2b1045a3 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
@@ -287,8 +287,9 @@ def test_snowflake_unexpected_snowflake_view_lineage_error_causes_pipeline_warni
             SnowflakeV2Config,
             cast(PipelineConfig, snowflake_pipeline_config1).source.config,
         )
+        config.include_table_lineage = True
         config.include_view_lineage = True
-        config.incremental_lineage = False
+
         pipeline = Pipeline(snowflake_pipeline_config1)
         pipeline.run()
         pipeline.raise_from_status()  # pipeline should not fail
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
index a495d04c4e398..2fe7a76fd01ae 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -80,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -91,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "1df94c0f-15fd-4b68-8ca3-6053a0332362",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-03-10 16:27:54.970000",
-                "date_modified": "2023-03-10 16:27:55.097000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -110,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -127,22 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                },
+                {
+                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
+                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1830,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1843,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-03-10 16:27:54.907000",
-                "date_modified": "2023-03-10 16:27:54.907000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1858,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1875,31 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                },
-                {
-                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
-                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,7 +2297,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2208,7 +2318,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2228,7 +2339,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2243,7 +2355,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2258,7 +2371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2275,7 +2389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2290,7 +2405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2311,7 +2427,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2326,7 +2443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2341,7 +2459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2358,7 +2477,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2373,7 +2493,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2393,7 +2514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2414,7 +2536,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2429,7 +2552,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2444,7 +2568,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2461,7 +2586,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2476,7 +2602,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2496,7 +2623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2517,7 +2645,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2532,7 +2661,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2547,7 +2677,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2564,7 +2695,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2579,7 +2711,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2599,7 +2732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2620,7 +2754,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2635,7 +2770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2650,7 +2786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2667,7 +2804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2682,7 +2820,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2702,7 +2841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2723,7 +2863,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2738,7 +2879,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2753,7 +2895,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2770,7 +2913,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2785,7 +2929,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2805,7 +2950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2826,7 +2972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2841,7 +2988,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2856,7 +3004,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2873,7 +3022,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2888,7 +3038,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2908,7 +3059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2929,7 +3081,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2944,7 +3097,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2959,7 +3113,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2976,7 +3131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2991,7 +3147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3011,7 +3168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3032,7 +3190,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3047,7 +3206,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3062,7 +3222,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3079,7 +3240,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3094,7 +3256,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3114,7 +3277,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3135,7 +3299,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3150,7 +3315,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3165,7 +3331,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3182,7 +3349,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3197,7 +3365,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3217,7 +3386,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3238,7 +3408,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3253,7 +3424,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3268,7 +3440,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3285,7 +3458,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3300,7 +3474,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3320,7 +3495,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3335,7 +3511,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3407,7 +3584,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3424,7 +3602,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3448,7 +3627,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3469,7 +3649,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3484,7 +3665,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3499,7 +3681,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3516,7 +3699,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3531,7 +3715,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3551,7 +3736,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3566,7 +3752,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3638,7 +3825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3655,7 +3843,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3679,7 +3868,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3694,7 +3884,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3790,7 +3981,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3807,7 +3999,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3831,7 +4024,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3852,7 +4046,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3867,7 +4062,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3882,7 +4078,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3899,7 +4096,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3914,7 +4112,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3934,7 +4133,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3955,7 +4155,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3970,7 +4171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3985,7 +4187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4002,7 +4205,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4017,7 +4221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4037,7 +4242,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4058,7 +4264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4073,7 +4280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4088,7 +4296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4105,7 +4314,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4120,27 +4330,34 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59",
+                    "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59"
+                }
+            ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4150,12 +4367,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4165,12 +4383,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4180,27 +4399,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b",
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59",
-                    "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
index 8277ff8bf7e89..c1984828750eb 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -80,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -91,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "1df94c0f-15fd-4b68-8ca3-6053a0332362",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-03-10 16:27:54.970000",
-                "date_modified": "2023-03-10 16:27:55.097000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -110,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -127,22 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                },
+                {
+                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
+                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1830,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1843,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-03-10 16:27:54.907000",
-                "date_modified": "2023-03-10 16:27:54.907000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1858,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1875,31 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                },
-                {
-                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
-                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,27 +2297,34 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                }
+            ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2218,12 +2334,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2233,12 +2350,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2248,27 +2366,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
index f3714bba6364d..804a8d74d0d51 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -80,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -91,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "1df94c0f-15fd-4b68-8ca3-6053a0332362",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-03-10 16:27:54.970000",
-                "date_modified": "2023-03-10 16:27:55.097000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -110,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -127,22 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                },
+                {
+                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
+                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1830,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1843,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-03-10 16:27:54.907000",
-                "date_modified": "2023-03-10 16:27:54.907000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1858,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1875,31 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.SalesReason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                },
-                {
-                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
-                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,27 +2297,34 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                }
+            ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2218,12 +2334,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2233,12 +2350,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2248,27 +2366,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
index d25d23daae2ea..9d1b288057a16 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -95,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -106,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "b6a0c1e2-f90a-4c86-a226-bf7ca59ad79f",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-08-06 21:01:05.157000",
-                "date_modified": "2023-08-06 21:01:05.283000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -125,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -142,7 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1837,7 +1926,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1854,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1867,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-08-06 21:01:05.093000",
-                "date_modified": "2023-08-06 21:01:05.093000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1882,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1899,7 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,7 +2297,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2208,7 +2318,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2223,7 +2334,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2238,7 +2350,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2253,12 +2366,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2268,7 +2382,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
index c1347a7c8caca..a17d52f9a39b1 100644
--- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
+++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
@@ -45,7 +45,7 @@ CREATE TABLE Foo.SalesReason
    )
 ;
 GO
-CREATE PROCEDURE Foo.DBs @ID INT
+CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT
 AS
     SELECT @ID AS ThatDB;
 GO
diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json
index 19961e48b4a33..c43223c68a6b6 100644
--- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json
+++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -99,7 +104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -181,7 +191,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -196,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -219,7 +231,7 @@
                             "numrows": "1",
                             "rawdatasize": "32",
                             "totalsize": "33",
-                            "transient_lastddltime": "1688422059"
+                            "transient_lastddltime": "1698223433"
                         },
                         "name": "array_struct_test",
                         "description": "This table has array of structs",
@@ -315,7 +327,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -332,7 +345,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -356,7 +370,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -371,7 +386,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -392,7 +408,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422063"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "map_test",
                         "tags": []
@@ -454,7 +470,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -471,7 +488,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -495,7 +513,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -510,7 +529,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -531,7 +551,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422062"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "nested_struct_test",
                         "tags": []
@@ -642,7 +662,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -659,7 +680,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -683,7 +705,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -698,7 +721,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -714,7 +738,7 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "transient_lastddltime": "1688421792"
+                            "transient_lastddltime": "1698223429"
                         },
                         "name": "pokes",
                         "tags": []
@@ -784,7 +808,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -801,7 +826,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -825,7 +851,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -840,7 +867,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -861,7 +889,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688421808"
+                            "transient_lastddltime": "1698223431"
                         },
                         "name": "struct_test",
                         "tags": []
@@ -950,7 +978,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -967,7 +996,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -991,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1006,7 +1037,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1024,7 +1056,7 @@
                         "customProperties": {
                             "numfiles": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422062"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "struct_test_view_materialized",
                         "tags": []
@@ -1113,7 +1145,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1130,7 +1163,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1154,7 +1188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1169,7 +1204,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1190,7 +1226,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688421807"
+                            "transient_lastddltime": "1698223431"
                         },
                         "name": "_test_table_underscore",
                         "tags": []
@@ -1248,7 +1284,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1265,7 +1302,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1289,7 +1327,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1304,7 +1343,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1325,7 +1365,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422062"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "union_test",
                         "tags": []
@@ -1467,7 +1507,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1484,7 +1525,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1550,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1523,7 +1566,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1539,7 +1583,7 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "transient_lastddltime": "1688422062",
+                            "transient_lastddltime": "1698223435",
                             "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"",
                             "is_view": "True"
                         },
@@ -1634,7 +1678,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1651,7 +1696,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1668,7 +1714,57 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD),property_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),property_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD),service)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),service)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1632398400000,
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1692,7 +1788,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
index 54a22d860285c..e8485106c6a81 100644
--- a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
@@ -104,6 +104,27 @@ def test_incremental_table_lineage(tmp_path, pytestconfig):
     )
 
 
+def test_incremental_table_lineage_empty_upstreams(tmp_path, pytestconfig):
+
+    urn = make_dataset_urn(platform, "dataset1")
+    aspect = make_lineage_aspect(
+        "dataset1",
+        upstreams=[],
+    )
+
+    processed_wus = auto_incremental_lineage(
+        graph=None,
+        incremental_lineage=True,
+        stream=[
+            MetadataChangeProposalWrapper(
+                entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
+            ).as_workunit()
+        ],
+    )
+
+    assert [wu.metadata for wu in processed_wus] == []
+
+
 @pytest.mark.parametrize(
     "gms_aspect,current_aspect,output_aspect",
     [
diff --git a/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py b/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py
index cbd5be9e7d832..4a69deb572fbd 100644
--- a/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py
+++ b/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py
@@ -1,14 +1,14 @@
 import tempfile
 from typing import List, Type
 
-import avro.schema
 import pandas as pd
 import ujson
 from avro import schema as avro_schema
 from avro.datafile import DataFileWriter
 from avro.io import DatumWriter
 
-from datahub.ingestion.source.schema_inference import avro, csv_tsv, json, parquet
+from datahub.ingestion.source.schema_inference import csv_tsv, json, parquet
+from datahub.ingestion.source.schema_inference.avro import AvroInferrer
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BooleanTypeClass,
     NumberTypeClass,
@@ -123,7 +123,7 @@ def test_infer_schema_avro():
 
         file.seek(0)
 
-        fields = avro.AvroInferrer().infer_schema(file)
+        fields = AvroInferrer().infer_schema(file)
         fields.sort(key=lambda x: x.fieldPath)
 
         assert_field_paths_match(fields, expected_field_paths_avro)
diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py
index d116f1f5473fa..d2d6a0bdda5b9 100644
--- a/metadata-ingestion/tests/unit/serde/test_serde.py
+++ b/metadata-ingestion/tests/unit/serde/test_serde.py
@@ -238,7 +238,7 @@ def test_missing_optional_simple() -> None:
                 "criteria": [
                     {
                         "condition": "EQUALS",
-                        "field": "RESOURCE_TYPE",
+                        "field": "TYPE",
                         "values": ["notebook", "dataset", "dashboard"],
                     }
                 ]
@@ -252,7 +252,7 @@ def test_missing_optional_simple() -> None:
             "criteria": [
                 {
                     "condition": "EQUALS",
-                    "field": "RESOURCE_TYPE",
+                    "field": "TYPE",
                     "values": ["notebook", "dataset", "dashboard"],
                 }
             ]
@@ -267,13 +267,13 @@ def test_missing_optional_simple() -> None:
 def test_missing_optional_in_union() -> None:
     # This one doesn't contain any optional fields and should work fine.
     revised_json = json.loads(
-        '{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}'
+        '{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}'
     )
     revised = models.DataHubPolicyInfoClass.from_obj(revised_json)
 
     # This one is missing the optional filters.allResources field.
     original_json = json.loads(
-        '{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}'
+        '{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}'
     )
     original = models.DataHubPolicyInfoClass.from_obj(original_json)
 
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
index f0175b4dc8892..d610b0a83f229 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
@@ -18,7 +18,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -36,7 +36,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -54,7 +54,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -72,7 +72,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
index b7df5444987f2..2d3d188d28316 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -32,7 +32,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
index 67e306bebf545..41ae0885941b0 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -32,7 +32,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
index b7df5444987f2..2d3d188d28316 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -32,7 +32,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
index b393b2445d6c4..26f8f8f59a3ff 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
@@ -16,7 +16,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -34,7 +34,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -52,7 +52,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
index 53fb94300e804..83365c09f69c2 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
@@ -17,7 +17,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -39,7 +39,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json
index 4773974545bfa..cf31b71cb50f6 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json
@@ -4,5 +4,58 @@
     "out_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)"
     ],
-    "column_lineage": null
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "id",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INTEGER"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "month",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "total_cost",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "REAL"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "area",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "REAL"
+            },
+            "upstreams": []
+        }
+    ]
 }
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
index ff452467aa5bd..8a6b60d0f1bde 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
@@ -30,7 +30,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "BIGINT"
+                "native_column_type": "NUMBER"
             },
             "upstreams": []
         },
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
index 5ad847e252497..2424fcda34752 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "NUMERIC"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
index 6ee3d2e61c39b..8dd2633eff612 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "NUMERIC"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
new file mode 100644
index 0000000000000..e2baa34e7fe28
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
@@ -0,0 +1,56 @@
+{
+    "query_type": "UPDATE",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)"
+    ],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+                    "column": "col1"
+                },
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+                    "column": "col2"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+                    "column": "col1"
+                },
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)",
+                    "column": "col2"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
new file mode 100644
index 0000000000000..b41ed61b37cdb
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
@@ -0,0 +1,35 @@
+{
+    "query_type": "UPDATE",
+    "in_tables": [],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                "column": "orderkey",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                "column": "totalprice",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": []
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
index b0351a7e07ad2..ee80285d87f60 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
@@ -12,6 +12,7 @@
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)",
                 "column": "PatientId",
+                "column_type": null,
                 "native_column_type": "INTEGER()"
             },
             "upstreams": [
@@ -25,6 +26,7 @@
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)",
                 "column": "BMI",
+                "column_type": null,
                 "native_column_type": "FLOAT()"
             },
             "upstreams": [
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index 059add8db67e4..dfc5b486abd35 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -3,6 +3,7 @@
 import pytest
 
 from datahub.testing.check_sql_parser_result import assert_sql_result
+from datahub.utilities.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
 
 RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens"
 
@@ -672,3 +673,98 @@ def test_teradata_default_normalization():
         },
         expected_file=RESOURCE_DIR / "test_teradata_default_normalization.json",
     )
+
+
+def test_snowflake_update_hardcoded():
+    assert_sql_result(
+        """
+UPDATE snowflake_sample_data.tpch_sf1.orders
+SET orderkey = 1, totalprice = 2
+WHERE orderkey = 3
+""",
+        dialect="snowflake",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": {
+                "orderkey": "NUMBER(38,0)",
+                "totalprice": "NUMBER(12,2)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_update_hardcoded.json",
+    )
+
+
+def test_update_from_select():
+    assert _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT == {"returning", "this"}
+
+
+def test_snowflake_update_from_table():
+    # Can create these tables with the following SQL:
+    """
+    -- Create or replace my_table
+    CREATE OR REPLACE TABLE my_table (
+        id INT IDENTITY PRIMARY KEY,
+        col1 VARCHAR(50),
+        col2 VARCHAR(50)
+    );
+
+    -- Create or replace table1
+    CREATE OR REPLACE TABLE table1 (
+        id INT IDENTITY PRIMARY KEY,
+        col1 VARCHAR(50),
+        col2 VARCHAR(50)
+    );
+
+    -- Create or replace table2
+    CREATE OR REPLACE TABLE table2 (
+        id INT IDENTITY PRIMARY KEY,
+        col2 VARCHAR(50)
+    );
+
+    -- Insert data into my_table
+    INSERT INTO my_table (col1, col2)
+    VALUES ('foo', 'bar'),
+           ('baz', 'qux');
+
+    -- Insert data into table1
+    INSERT INTO table1 (col1, col2)
+    VALUES ('foo', 'bar'),
+           ('baz', 'qux');
+
+    -- Insert data into table2
+    INSERT INTO table2 (col2)
+    VALUES ('bar'),
+           ('qux');
+    """
+
+    assert_sql_result(
+        """
+UPDATE my_table
+SET
+    col1 = t1.col1 || t1.col2,
+    col2 = t1.col1 || t2.col2
+FROM table1 t1
+JOIN table2 t2 ON t1.id = t2.id
+WHERE my_table.id = t1.id;
+""",
+        dialect="snowflake",
+        default_db="my_db",
+        default_schema="my_schema",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)": {
+                "id": "NUMBER(38,0)",
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)": {
+                "id": "NUMBER(38,0)",
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)": {
+                "id": "NUMBER(38,0)",
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_update_from_table.json",
+    )
diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py
index 6d3ed20eafde2..23dd7dd5a6e45 100644
--- a/metadata-ingestion/tests/unit/test_athena_source.py
+++ b/metadata-ingestion/tests/unit/test_athena_source.py
@@ -9,7 +9,7 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.aws.s3_util import make_s3_urn
 from datahub.ingestion.source.sql.athena import CustomAthenaRestDialect
-from datahub.ingestion.source.sql.sql_types import MapType
+from datahub.utilities.sqlalchemy_type_converter import MapType
 
 FROZEN_TIME = "2020-04-14 07:00:00"
 
diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py
index 5a11a933c8595..4cfa5c48d2377 100644
--- a/metadata-ingestion/tests/unit/test_bigquery_source.py
+++ b/metadata-ingestion/tests/unit/test_bigquery_source.py
@@ -53,6 +53,59 @@ def test_bigquery_uri_on_behalf():
     assert config.get_sql_alchemy_url() == "bigquery://test-project-on-behalf"
 
 
+def test_bigquery_dataset_pattern():
+    config = BigQueryV2Config.parse_obj(
+        {
+            "dataset_pattern": {
+                "allow": [
+                    "test-dataset",
+                    "test-project.test-dataset",
+                    ".*test-dataset",
+                ],
+                "deny": [
+                    "^test-dataset-2$",
+                    "project\\.second_dataset",
+                ],
+            },
+        }
+    )
+    assert config.dataset_pattern.allow == [
+        r".*\.test-dataset",
+        r"test-project.test-dataset",
+        r".*test-dataset",
+    ]
+    assert config.dataset_pattern.deny == [
+        r"^.*\.test-dataset-2$",
+        r"project\.second_dataset",
+    ]
+
+    config = BigQueryV2Config.parse_obj(
+        {
+            "dataset_pattern": {
+                "allow": [
+                    "test-dataset",
+                    "test-project.test-dataset",
+                    ".*test-dataset",
+                ],
+                "deny": [
+                    "^test-dataset-2$",
+                    "project\\.second_dataset",
+                ],
+            },
+            "match_fully_qualified_names": False,
+        }
+    )
+    assert config.dataset_pattern.allow == [
+        r"test-dataset",
+        r"test-project.test-dataset",
+        r".*test-dataset",
+    ]
+    assert config.dataset_pattern.deny == [
+        r"^test-dataset-2$",
+        r"project\.second_dataset",
+    ]
+
+
 def test_bigquery_uri_with_credential():
     expected_credential_json = {
         "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
diff --git a/metadata-ingestion/tests/unit/test_redshift_config.py b/metadata-ingestion/tests/unit/test_redshift_config.py
new file mode 100644
index 0000000000000..8a165e7f5f3fe
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_redshift_config.py
@@ -0,0 +1,6 @@
+from datahub.ingestion.source.redshift.config import RedshiftConfig
+
+
+def test_incremental_lineage_default_to_false():
+    config = RedshiftConfig(host_port="localhost:5439", database="test")
+    assert config.incremental_lineage is False
diff --git a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
index 959da0987a825..6c719d351c4c2 100644
--- a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
+++ b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
@@ -3,7 +3,6 @@
 from sqlalchemy import types
 from sqlalchemy_bigquery import STRUCT
 
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     MapTypeClass,
@@ -12,6 +11,7 @@
     RecordTypeClass,
 )
 from datahub.utilities.sqlalchemy_type_converter import (
+    MapType,
     get_schema_fields_for_sqlalchemy_column,
 )
 
diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java
index 42884241d9f7c..d890c373f1299 100644
--- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java
+++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java
@@ -259,7 +259,9 @@ private FieldDescriptorProto getNestedTypeFields(List<Integer> pathList, Descrip
             messageType = messageType.getNestedType(value);
         }
 
-        if (pathList.get(pathSize - 2) == DescriptorProto.FIELD_FIELD_NUMBER) {
+        if (pathList.get(pathSize - 2) == DescriptorProto.FIELD_FIELD_NUMBER
+                && pathList.get(pathSize - 1) != DescriptorProto.RESERVED_RANGE_FIELD_NUMBER
+                && pathList.get(pathSize - 1) != DescriptorProto.RESERVED_NAME_FIELD_NUMBER) {
             return messageType.getField(pathList.get(pathSize - 1));
         } else {
             return null;
diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle
index 53e7765152aef..bd8052283e168 100644
--- a/metadata-models/build.gradle
+++ b/metadata-models/build.gradle
@@ -23,6 +23,7 @@ dependencies {
       }
     }
   api project(':li-utils')
+  api project(path: ':li-utils', configuration: "dataTemplate")
   dataModel project(':li-utils')
 
   compileOnly externalDependency.lombok
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
index 4553139e3ca54..e30fb93109915 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
@@ -19,6 +19,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -53,6 +55,7 @@ public enum AuthorizationMode {
   // Maps privilege name to the associated set of policies for fast access.
   // Not concurrent data structure because writes are always against the entire thing.
   private final Map<String, List<DataHubPolicyInfo>> _policyCache = new HashMap<>(); // Shared Policy Cache.
+  private final ReadWriteLock _lockPolicyCache = new ReentrantReadWriteLock();
 
   private final ScheduledExecutorService _refreshExecutorService = Executors.newScheduledThreadPool(1);
   private final PolicyRefreshRunnable _policyRefreshRunnable;
@@ -71,7 +74,7 @@ public DataHubAuthorizer(
     _systemAuthentication = Objects.requireNonNull(systemAuthentication);
     _mode = Objects.requireNonNull(mode);
     _policyEngine = new PolicyEngine(systemAuthentication, Objects.requireNonNull(entityClient));
-    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache);
+    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache, _lockPolicyCache);
     _refreshExecutorService.scheduleAtFixedRate(_policyRefreshRunnable, delayIntervalSeconds, refreshIntervalSeconds, TimeUnit.SECONDS);
   }
 
@@ -90,31 +93,41 @@ public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request
 
     Optional<ResolvedEntitySpec> resolvedResourceSpec = request.getResourceSpec().map(_entitySpecResolver::resolve);
 
-    // 1. Fetch the policies relevant to the requested privilege.
-    final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>());
-
-    // 2. Evaluate each policy.
-    for (DataHubPolicyInfo policy : policiesToEvaluate) {
-      if (isRequestGranted(policy, request, resolvedResourceSpec)) {
-        // Short circuit if policy has granted privileges to this actor.
-        return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW,
-            String.format("Granted by policy with type: %s", policy.getType()));
+    _lockPolicyCache.readLock().lock();
+    try {
+      // 1. Fetch the policies relevant to the requested privilege.
+      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>());
+
+      // 2. Evaluate each policy.
+      for (DataHubPolicyInfo policy : policiesToEvaluate) {
+        if (isRequestGranted(policy, request, resolvedResourceSpec)) {
+          // Short circuit if policy has granted privileges to this actor.
+          return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW,
+              String.format("Granted by policy with type: %s", policy.getType()));
+        }
       }
+      return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
+    } finally {
+      _lockPolicyCache.readLock().unlock();
     }
-    return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
   }
 
   public List<String> getGrantedPrivileges(final String actor, final Optional<EntitySpec> resourceSpec) {
 
-    // 1. Fetch all policies
-    final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>());
+    _lockPolicyCache.readLock().lock();
+    try {
+      // 1. Fetch all policies
+      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>());
 
-    Urn actorUrn = UrnUtils.getUrn(actor);
-    final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
+      Urn actorUrn = UrnUtils.getUrn(actor);
+      final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
 
-    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
+      Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-    return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
+       return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
+    } finally {
+      _lockPolicyCache.readLock().unlock();
+    }
   }
 
   /**
@@ -124,36 +137,42 @@ public List<String> getGrantedPrivileges(final String actor, final Optional<Enti
   public AuthorizedActors authorizedActors(
       final String privilege,
       final Optional<EntitySpec> resourceSpec) {
-    // Step 1: Find policies granting the privilege.
-    final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>());
-
-    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
     final List<Urn> authorizedUsers = new ArrayList<>();
     final List<Urn> authorizedGroups = new ArrayList<>();
     boolean allUsers = false;
     boolean allGroups = false;
 
-    // Step 2: For each policy, determine whether the resource is a match.
-    for (DataHubPolicyInfo policy : policiesToEvaluate) {
-      if (!PoliciesConfig.ACTIVE_POLICY_STATE.equals(policy.getState())) {
-        // Policy is not active, skip.
-        continue;
-      }
+    _lockPolicyCache.readLock().lock();
+    try {
+      // Step 1: Find policies granting the privilege.
+      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>());
 
-      final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec);
+      Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-      // Step 3: For each matching policy, add actors that are authorized.
-      authorizedUsers.addAll(matchingActors.getUsers());
-      authorizedGroups.addAll(matchingActors.getGroups());
-      if (matchingActors.allUsers()) {
-        allUsers = true;
-      }
-      if (matchingActors.allGroups()) {
-        allGroups = true;
+
+      // Step 2: For each policy, determine whether the resource is a match.
+      for (DataHubPolicyInfo policy : policiesToEvaluate) {
+        if (!PoliciesConfig.ACTIVE_POLICY_STATE.equals(policy.getState())) {
+          // Policy is not active, skip.
+          continue;
+        }
+
+        final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec);
+
+        // Step 3: For each matching policy, add actors that are authorized.
+        authorizedUsers.addAll(matchingActors.getUsers());
+        authorizedGroups.addAll(matchingActors.getGroups());
+        if (matchingActors.allUsers()) {
+          allUsers = true;
+        }
+        if (matchingActors.allGroups()) {
+          allGroups = true;
+        }
       }
+    } finally {
+      _lockPolicyCache.readLock().unlock();
     }
-
     // Step 4: Return all authorized users and groups.
     return new AuthorizedActors(privilege, authorizedUsers, authorizedGroups, allUsers, allGroups);
   }
@@ -228,6 +247,7 @@ static class PolicyRefreshRunnable implements Runnable {
     private final Authentication _systemAuthentication;
     private final PolicyFetcher _policyFetcher;
     private final Map<String, List<DataHubPolicyInfo>> _policyCache;
+    private final ReadWriteLock _lockPolicyCache;
 
     @Override
     public void run() {
@@ -253,10 +273,13 @@ public void run() {
                 "Failed to retrieve policy urns! Skipping updating policy cache until next refresh. start: {}, count: {}", start, count, e);
             return;
           }
-          synchronized (_policyCache) {
-            _policyCache.clear();
-            _policyCache.putAll(newCache);
-          }
+        }
+        _lockPolicyCache.writeLock().lock();
+        try {
+          _policyCache.clear();
+          _policyCache.putAll(newCache);
+        } finally {
+          _lockPolicyCache.writeLock().unlock();
         }
         log.debug(String.format("Successfully fetched %s policies.", total));
       } catch (Exception e) {
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
index 4ad14ed59c9c0..65b0329a9c4f2 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
@@ -1,15 +1,16 @@
 package com.datahub.authorization;
 
-import com.datahub.authorization.fieldresolverprovider.DataPlatformInstanceFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
 import com.datahub.authentication.Authentication;
+import com.datahub.authorization.fieldresolverprovider.DataPlatformInstanceFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.DomainFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.EntityFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.GroupMembershipFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.util.Pair;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -34,7 +35,7 @@ public ResolvedEntitySpec resolve(EntitySpec entitySpec) {
 
   private Map<EntityFieldType, FieldResolver> getFieldResolvers(EntitySpec entitySpec) {
     return _entityFieldResolverProviders.stream()
-        .collect(Collectors.toMap(EntityFieldResolverProvider::getFieldType,
-            hydrator -> hydrator.getFieldResolver(entitySpec)));
+        .flatMap(resolver -> resolver.getFieldTypes().stream().map(fieldType -> Pair.of(fieldType, resolver)))
+        .collect(Collectors.toMap(Pair::getKey, pair -> pair.getValue().getFieldResolver(entitySpec)));
   }
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
index 27cb8fcee8138..cbb237654e969 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
@@ -1,8 +1,5 @@
 package com.datahub.authorization.fieldresolverprovider;
 
-import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME;
-import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ENTITY_NAME;
-
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
@@ -14,10 +11,13 @@
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.client.EntityClient;
 import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import static com.linkedin.metadata.Constants.*;
+
 /**
  * Provides field resolver for domain given resourceSpec
  */
@@ -29,8 +29,8 @@ public class DataPlatformInstanceFieldResolverProvider implements EntityFieldRes
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.DATA_PLATFORM_INSTANCE;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.DATA_PLATFORM_INSTANCE);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
index 25c2165f02b94..15d821b75c0bd 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
@@ -14,6 +14,7 @@
 
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -37,8 +38,8 @@ public class DomainFieldResolverProvider implements EntityFieldResolverProvider
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.DOMAIN;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.DOMAIN);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
index a76db0ecb5102..227d403a9cd1d 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
@@ -3,6 +3,7 @@
 import com.datahub.authorization.FieldResolver;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
+import java.util.List;
 
 
 /**
@@ -11,9 +12,10 @@
 public interface EntityFieldResolverProvider {
 
   /**
-   * Field that this hydrator is hydrating
+   * List of fields that this hydrator is hydrating.
+   * @return
    */
-  EntityFieldType getFieldType();
+  List<EntityFieldType> getFieldTypes();
 
   /**
    * Return resolver for fetching the field values given the entity
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
index 187f696904947..addac84c68b18 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
@@ -3,16 +3,19 @@
 import com.datahub.authorization.FieldResolver;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
+import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList;
 import java.util.Collections;
+import java.util.List;
 
 
 /**
  * Provides field resolver for entity type given entitySpec
  */
 public class EntityTypeFieldResolverProvider implements EntityFieldResolverProvider {
+
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.TYPE;
+  public List<EntityFieldType> getFieldTypes() {
+    return ImmutableList.of(EntityFieldType.TYPE, EntityFieldType.RESOURCE_TYPE);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
index 2f5c4a7c6c961..32960de687839 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
@@ -3,16 +3,19 @@
 import com.datahub.authorization.FieldResolver;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
+import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList;
 import java.util.Collections;
+import java.util.List;
 
 
 /**
  * Provides field resolver for entity urn given entitySpec
  */
 public class EntityUrnFieldResolverProvider implements EntityFieldResolverProvider {
+
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.URN;
+  public List<EntityFieldType> getFieldTypes() {
+    return ImmutableList.of(EntityFieldType.URN, EntityFieldType.RESOURCE_URN);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
index 8db029632d7e2..b1202d9f4bbd3 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
@@ -13,6 +13,7 @@
 import com.linkedin.identity.NativeGroupMembership;
 import com.linkedin.metadata.Constants;
 import com.linkedin.identity.GroupMembership;
+import java.util.Collections;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -35,8 +36,8 @@ public class GroupMembershipFieldResolverProvider implements EntityFieldResolver
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.GROUP_MEMBERSHIP;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.GROUP_MEMBERSHIP);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
index bdd652d1d3871..3c27f9e6ce8d7 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
@@ -12,6 +12,7 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
 import java.util.Collections;
+import java.util.List;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -28,8 +29,8 @@ public class OwnerFieldResolverProvider implements EntityFieldResolverProvider {
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.OWNER;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.OWNER);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
index b2343bbb01509..5c7d87f1c05a9 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
@@ -56,7 +56,7 @@ public void setup() {
 
   @Test
   public void shouldReturnDataPlatformInstanceType() {
-    assertEquals(EntityFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldType());
+    assertEquals(EntityFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldTypes().get(0));
   }
 
   @Test
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
index 54675045b4413..af547f14cd3fc 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
@@ -53,7 +53,7 @@ public void setup() {
 
   @Test
   public void shouldReturnGroupsMembershipType() {
-    assertEquals(EntityFieldType.GROUP_MEMBERSHIP, groupMembershipFieldResolverProvider.getFieldType());
+    assertEquals(EntityFieldType.GROUP_MEMBERSHIP, groupMembershipFieldResolverProvider.getFieldTypes().get(0));
   }
 
   @Test
diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json
index 410596cc30cbe..18cb48bfcf1f0 100644
--- a/metadata-service/war/src/main/resources/boot/policies.json
+++ b/metadata-service/war/src/main/resources/boot/policies.json
@@ -64,7 +64,8 @@
         "GET_TIMELINE_PRIVILEGE",
         "PRODUCE_PLATFORM_EVENT_PRIVILEGE",
         "MANAGE_DATA_PRODUCTS",
-        "MANAGE_GLOBAL_OWNERSHIP_TYPES"
+        "MANAGE_GLOBAL_OWNERSHIP_TYPES",
+        "DELETE_ENTITY"
       ],
       "displayName":"Root User - Edit and View All Resources",
       "description":"Grants full edit and view privileges for all resources to root 'datahub' root user.",
@@ -263,7 +264,8 @@
         "GET_ENTITY_PRIVILEGE",
         "GET_TIMELINE_PRIVILEGE",
         "PRODUCE_PLATFORM_EVENT_PRIVILEGE",
-        "MANAGE_DATA_PRODUCTS"
+        "MANAGE_DATA_PRODUCTS",
+        "DELETE_ENTITY"
       ],
       "displayName":"Admins - Metadata Policy",
       "description":"Admins have all metadata privileges.",
diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js
new file mode 100644
index 0000000000000..37ca62c8d1229
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js
@@ -0,0 +1,68 @@
+import { aliasQuery } from "../utils";
+const DATASET_ENTITY_TYPE = 'dataset';
+const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)';
+const DOWNSTREAM_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)";
+const upstreamColumn = '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)-Upstream"] text';
+const downstreamColumn = '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)-Downstream"] text';
+
+const verifyColumnPathModal = (from, to) => {
+  cy.get('[data-testid="entity-paths-modal"]').contains(from).should("be.visible");
+  cy.get('[data-testid="entity-paths-modal"]').contains(to).should("be.visible");
+};
+
+describe("column-Level lineage and impact analysis path test", () => {
+    beforeEach(() => {
+        cy.on('uncaught:exception', (err, runnable) => { return false; });
+        cy.intercept("POST", "/api/v2/graphql", (req) => {
+          aliasQuery(req, "appConfig");
+        });
+      });
+
+    it("verify column-level lineage path at lineage praph and impact analysis ", () => {
+      // Open dataset with column-level lineage configured an navigate to lineage tab -> visualize lineage
+      cy.loginWithCredentials();
+      cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN);
+
+      // Enable “show columns” toggle
+      cy.waitTextVisible("SampleCypressHdfs");
+      cy.clickOptionWithTestId("column-toggle");
+      cy.waitTextVisible("shipment_info");
+
+      // Verify functionality of column lineage
+      cy.get(upstreamColumn).eq(3).click();
+      cy.get(upstreamColumn).eq(3).prev().should('not.have.attr', 'fill', 'white');
+      cy.get(downstreamColumn).eq(2).prev().should('not.have.attr', 'stroke', 'transparent');
+      cy.get(downstreamColumn).eq(2).click();
+      cy.get(downstreamColumn).eq(2).prev().should('not.have.attr', 'fill', 'white');
+      cy.get(upstreamColumn).eq(3).prev().should('not.have.attr', 'stroke', 'transparent');
+
+      // Open dataset impact analysis view, enable column lineage
+      cy.goToDataset(DATASET_URN, "SampleCypressHdfsDataset");
+      cy.openEntityTab("Lineage");
+      cy.clickOptionWithText("Column Lineage");
+      cy.clickOptionWithText("Downstream");
+
+      // Verify upstream column lineage, test column path modal
+      cy.clickOptionWithText("Upstream");
+      cy.waitTextVisible("SampleCypressKafkaDataset");
+      cy.ensureTextNotPresent("field_bar");
+      cy.contains("Select column").click({ force: true}).wait(1000);
+      cy.get(".rc-virtual-list").contains("shipment_info").click(); 
+      cy.waitTextVisible("field_bar");
+      cy.clickOptionWithText("field_bar");
+      verifyColumnPathModal("shipment_info", "field_bar");
+      cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click();
+    
+      // Verify downstream column lineage, test column path modal
+      cy.goToDataset(DOWNSTREAM_DATASET_URN, "SampleCypressKafkaDataset");
+      cy.openEntityTab("Lineage");
+      cy.clickOptionWithText("Column Lineage");
+      cy.ensureTextNotPresent("shipment_info");
+      cy.contains("Select column").click({ force: true}).wait(1000);
+      cy.get(".rc-virtual-list").contains("field_bar").click(); 
+      cy.waitTextVisible("shipment_info");
+      cy.clickOptionWithText("shipment_info");
+      verifyColumnPathModal("shipment_info", "field_bar");
+      cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click();
+    });
+}); 
\ No newline at end of file
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
index 24a24cc21138d..3d052695e818f 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
@@ -7,7 +7,7 @@ describe("run managed ingestion", () => {
     it("create run managed ingestion source", () => {
         let number = Math.floor(Math.random() * 100000);
         let testName = `cypress test source ${number}`
-        let cli_version = "0.10.5.4";
+        let cli_version = "0.12.0";
         cy.login();
         cy.goToIngestionPage();
         cy.clickOptionWithText("Create new source");