From d3346a04e486fa098129b626e61013cab4f69350 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 4 Oct 2023 10:22:45 -0400
Subject: [PATCH 001/259] feat(ingest/unity): Ingest notebooks and their
 lineage (#8940)

---
 .../sources/databricks/unity-catalog_pre.md   |   1 +
 metadata-ingestion/setup.py                   |   2 +-
 .../src/datahub/emitter/mcp_builder.py        |  12 ++
 .../ingestion/source/common/subtypes.py       |   3 +
 .../datahub/ingestion/source/unity/config.py  |   9 +-
 .../datahub/ingestion/source/unity/proxy.py   |  89 +++++++----
 .../ingestion/source/unity/proxy_types.py     |  45 +++++-
 .../datahub/ingestion/source/unity/report.py  |   8 +-
 .../datahub/ingestion/source/unity/source.py  | 148 ++++++++++++++----
 .../datahub/ingestion/source/unity/usage.py   |  12 +-
 10 files changed, 257 insertions(+), 72 deletions(-)

diff --git a/metadata-ingestion/docs/sources/databricks/unity-catalog_pre.md b/metadata-ingestion/docs/sources/databricks/unity-catalog_pre.md
index 2be8846b87bea..ae2883343d7e8 100644
--- a/metadata-ingestion/docs/sources/databricks/unity-catalog_pre.md
+++ b/metadata-ingestion/docs/sources/databricks/unity-catalog_pre.md
@@ -13,6 +13,7 @@
     * Ownership of or `SELECT` privilege on any tables and views you want to ingest
     * [Ownership documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/ownership.html)
     * [Privileges documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/privileges.html)
+  + To ingest your workspace's notebooks and respective lineage, your service principal must have `CAN_READ` privileges on the folders containing the notebooks you want to ingest: [guide](https://docs.databricks.com/en/security/auth-authz/access-control/workspace-acl.html#folder-permissions).
   + To `include_usage_statistics` (enabled by default), your service principal must have `CAN_MANAGE` permissions on any SQL Warehouses you want to ingest: [guide](https://docs.databricks.com/security/auth-authz/access-control/sql-endpoint-acl.html).
   + To ingest `profiling` information with `call_analyze` (enabled by default), your service principal must have ownership or `MODIFY` privilege on any tables you want to profile.
     * Alternatively, you can run [ANALYZE TABLE](https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-analyze-table.html) yourself on any tables you want to profile, then set `call_analyze` to `false`.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 34afa8cdb39a4..fe8e3be4632c4 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -250,7 +250,7 @@
 
 databricks = {
     # 0.1.11 appears to have authentication issues with azure databricks
-    "databricks-sdk>=0.1.1, != 0.1.11",
+    "databricks-sdk>=0.9.0",
     "pyspark",
     "requests",
 }
diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
index 844a29f1c78a3..7419577b367aa 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -9,6 +9,7 @@
     make_container_urn,
     make_data_platform_urn,
     make_dataplatform_instance_urn,
+    make_dataset_urn_with_platform_instance,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
@@ -125,6 +126,17 @@ class BucketKey(ContainerKey):
     bucket_name: str
 
 
+class NotebookKey(DatahubKey):
+    notebook_id: int
+    platform: str
+    instance: Optional[str]
+
+    def as_urn(self) -> str:
+        return make_dataset_urn_with_platform_instance(
+            platform=self.platform, platform_instance=self.instance, name=self.guid()
+        )
+
+
 class DatahubKeyJSONEncoder(json.JSONEncoder):
     # overload method default
     def default(self, obj: Any) -> Any:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
index a2d89d26112f4..741b4789bef21 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
@@ -16,6 +16,9 @@ class DatasetSubTypes(str, Enum):
     SALESFORCE_STANDARD_OBJECT = "Object"
     POWERBI_DATASET_TABLE = "PowerBI Dataset Table"
 
+    # TODO: Create separate entity...
+    NOTEBOOK = "Notebook"
+
 
 class DatasetContainerSubTypes(str, Enum):
     # Generic SubTypes
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index 94ff755e3b254..a49c789a82f27 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -127,11 +127,16 @@ class UnityCatalogSourceConfig(
         description='Attach domains to catalogs, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like "Marketing".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.',
     )
 
-    include_table_lineage: Optional[bool] = pydantic.Field(
+    include_table_lineage: bool = pydantic.Field(
         default=True,
         description="Option to enable/disable lineage generation.",
     )
 
+    include_notebooks: bool = pydantic.Field(
+        default=False,
+        description="Ingest notebooks, represented as DataHub datasets.",
+    )
+
     include_ownership: bool = pydantic.Field(
         default=False,
         description="Option to enable/disable ownership generation for metastores, catalogs, schemas, and tables.",
@@ -141,7 +146,7 @@ class UnityCatalogSourceConfig(
         "include_table_ownership", "include_ownership"
     )
 
-    include_column_lineage: Optional[bool] = pydantic.Field(
+    include_column_lineage: bool = pydantic.Field(
         default=True,
         description="Option to enable/disable lineage generation. Currently we have to call a rest call per column to get column level lineage due to the Databrick api which can slow down ingestion. ",
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
index e92f4ff07b1ad..2401f1c3d163c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
@@ -23,6 +23,7 @@
     QueryStatementType,
     QueryStatus,
 )
+from databricks.sdk.service.workspace import ObjectType
 
 import datahub
 from datahub.ingestion.source.unity.proxy_profiling import (
@@ -33,6 +34,7 @@
     Catalog,
     Column,
     Metastore,
+    Notebook,
     Query,
     Schema,
     ServicePrincipal,
@@ -137,6 +139,21 @@ def service_principals(self) -> Iterable[ServicePrincipal]:
         for principal in self._workspace_client.service_principals.list():
             yield self._create_service_principal(principal)
 
+    def workspace_notebooks(self) -> Iterable[Notebook]:
+        for obj in self._workspace_client.workspace.list("/", recursive=True):
+            if obj.object_type == ObjectType.NOTEBOOK:
+                yield Notebook(
+                    id=obj.object_id,
+                    path=obj.path,
+                    language=obj.language,
+                    created_at=datetime.fromtimestamp(
+                        obj.created_at / 1000, tz=timezone.utc
+                    ),
+                    modified_at=datetime.fromtimestamp(
+                        obj.modified_at / 1000, tz=timezone.utc
+                    ),
+                )
+
     def query_history(
         self,
         start_time: datetime,
@@ -153,7 +170,7 @@ def query_history(
                     "start_time_ms": start_time.timestamp() * 1000,
                     "end_time_ms": end_time.timestamp() * 1000,
                 },
-                "statuses": [QueryStatus.FINISHED.value],
+                "statuses": [QueryStatus.FINISHED],
                 "statement_types": [typ.value for typ in ALLOWED_STATEMENT_TYPES],
             }
         )
@@ -196,61 +213,75 @@ def _query_history(
                 method, path, body={**body, "page_token": response["next_page_token"]}
             )
 
-    def list_lineages_by_table(self, table_name: str) -> dict:
+    def list_lineages_by_table(
+        self, table_name: str, include_entity_lineage: bool
+    ) -> dict:
         """List table lineage by table name."""
         return self._workspace_client.api_client.do(
             method="GET",
-            path="/api/2.0/lineage-tracking/table-lineage/get",
-            body={"table_name": table_name},
+            path="/api/2.0/lineage-tracking/table-lineage",
+            body={
+                "table_name": table_name,
+                "include_entity_lineage": include_entity_lineage,
+            },
         )
 
     def list_lineages_by_column(self, table_name: str, column_name: str) -> dict:
         """List column lineage by table name and column name."""
         return self._workspace_client.api_client.do(
             "GET",
-            "/api/2.0/lineage-tracking/column-lineage/get",
+            "/api/2.0/lineage-tracking/column-lineage",
             body={"table_name": table_name, "column_name": column_name},
         )
 
-    def table_lineage(self, table: Table) -> None:
+    def table_lineage(
+        self, table: Table, include_entity_lineage: bool
+    ) -> Optional[dict]:
         # Lineage endpoint doesn't exists on 2.1 version
         try:
             response: dict = self.list_lineages_by_table(
-                table_name=f"{table.schema.catalog.name}.{table.schema.name}.{table.name}"
+                table_name=table.ref.qualified_table_name,
+                include_entity_lineage=include_entity_lineage,
             )
-            table.upstreams = {
-                TableReference(
-                    table.schema.catalog.metastore.id,
-                    item["catalog_name"],
-                    item["schema_name"],
-                    item["name"],
-                ): {}
-                for item in response.get("upstream_tables", [])
-            }
+
+            for item in response.get("upstreams") or []:
+                if "tableInfo" in item:
+                    table_ref = TableReference.create_from_lineage(
+                        item["tableInfo"], table.schema.catalog.metastore.id
+                    )
+                    if table_ref:
+                        table.upstreams[table_ref] = {}
+                for notebook in item.get("notebookInfos") or []:
+                    table.upstream_notebooks.add(notebook["notebook_id"])
+
+            for item in response.get("downstreams") or []:
+                for notebook in item.get("notebookInfos") or []:
+                    table.downstream_notebooks.add(notebook["notebook_id"])
+
+            return response
         except Exception as e:
             logger.error(f"Error getting lineage: {e}")
+            return None
 
-    def get_column_lineage(self, table: Table) -> None:
+    def get_column_lineage(self, table: Table, include_entity_lineage: bool) -> None:
         try:
-            table_lineage_response: dict = self.list_lineages_by_table(
-                table_name=f"{table.schema.catalog.name}.{table.schema.name}.{table.name}"
+            table_lineage = self.table_lineage(
+                table, include_entity_lineage=include_entity_lineage
             )
-            if table_lineage_response:
+            if table_lineage:
                 for column in table.columns:
                     response: dict = self.list_lineages_by_column(
-                        table_name=f"{table.schema.catalog.name}.{table.schema.name}.{table.name}",
+                        table_name=table.ref.qualified_table_name,
                         column_name=column.name,
                     )
                     for item in response.get("upstream_cols", []):
-                        table_ref = TableReference(
-                            table.schema.catalog.metastore.id,
-                            item["catalog_name"],
-                            item["schema_name"],
-                            item["table_name"],
+                        table_ref = TableReference.create_from_lineage(
+                            item, table.schema.catalog.metastore.id
                         )
-                        table.upstreams.setdefault(table_ref, {}).setdefault(
-                            column.name, []
-                        ).append(item["name"])
+                        if table_ref:
+                            table.upstreams.setdefault(table_ref, {}).setdefault(
+                                column.name, []
+                            ).append(item["name"])
 
         except Exception as e:
             logger.error(f"Error getting lineage: {e}")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
index 2b943d8c98e7d..d57f20245913f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
@@ -1,8 +1,10 @@
 # Supported types are available at
 # https://api-docs.databricks.com/rest/latest/unity-catalog-api-specification-2-1.html?_ga=2.151019001.1795147704.1666247755-2119235717.1666247755
+import dataclasses
+import logging
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Dict, FrozenSet, List, Optional, Set
 
 from databricks.sdk.service.catalog import (
     CatalogType,
@@ -11,6 +13,7 @@
     TableType,
 )
 from databricks.sdk.service.sql import QueryStatementType
+from databricks.sdk.service.workspace import Language
 
 from datahub.metadata.schema_classes import (
     ArrayTypeClass,
@@ -26,6 +29,8 @@
     TimeTypeClass,
 )
 
+logger = logging.getLogger(__name__)
+
 DATA_TYPE_REGISTRY: dict = {
     ColumnTypeName.BOOLEAN: BooleanTypeClass,
     ColumnTypeName.BYTE: BytesTypeClass,
@@ -66,6 +71,9 @@
 ALLOWED_STATEMENT_TYPES = {*OPERATION_STATEMENT_TYPES.keys(), QueryStatementType.SELECT}
 
 
+NotebookId = int
+
+
 @dataclass
 class CommonProperty:
     id: str
@@ -136,6 +144,19 @@ def create(cls, table: "Table") -> "TableReference":
             table.name,
         )
 
+    @classmethod
+    def create_from_lineage(cls, d: dict, metastore: str) -> Optional["TableReference"]:
+        try:
+            return cls(
+                metastore,
+                d["catalog_name"],
+                d["schema_name"],
+                d.get("table_name", d["name"]),  # column vs table query output
+            )
+        except Exception as e:
+            logger.warning(f"Failed to create TableReference from {d}: {e}")
+            return None
+
     def __str__(self) -> str:
         return f"{self.metastore}.{self.catalog}.{self.schema}.{self.table}"
 
@@ -166,6 +187,8 @@ class Table(CommonProperty):
     view_definition: Optional[str]
     properties: Dict[str, str]
     upstreams: Dict[TableReference, Dict[str, List[str]]] = field(default_factory=dict)
+    upstream_notebooks: Set[NotebookId] = field(default_factory=set)
+    downstream_notebooks: Set[NotebookId] = field(default_factory=set)
 
     ref: TableReference = field(init=False)
 
@@ -228,3 +251,23 @@ def __bool__(self):
                 self.max is not None,
             )
         )
+
+
+@dataclass
+class Notebook:
+    id: NotebookId
+    path: str
+    language: Language
+    created_at: datetime
+    modified_at: datetime
+
+    upstreams: FrozenSet[TableReference] = field(default_factory=frozenset)
+
+    @classmethod
+    def add_upstream(cls, upstream: TableReference, notebook: "Notebook") -> "Notebook":
+        return cls(
+            **{  # type: ignore
+                **dataclasses.asdict(notebook),
+                "upstreams": frozenset([*notebook.upstreams, upstream]),
+            }
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
index 8382b31a56add..808172a136bb3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
@@ -5,21 +5,23 @@
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalSourceReport,
 )
+from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.utilities.lossy_collections import LossyDict, LossyList
 
 
 @dataclass
-class UnityCatalogReport(StaleEntityRemovalSourceReport):
+class UnityCatalogReport(IngestionStageReport, StaleEntityRemovalSourceReport):
     metastores: EntityFilterReport = EntityFilterReport.field(type="metastore")
     catalogs: EntityFilterReport = EntityFilterReport.field(type="catalog")
     schemas: EntityFilterReport = EntityFilterReport.field(type="schema")
     tables: EntityFilterReport = EntityFilterReport.field(type="table/view")
     table_profiles: EntityFilterReport = EntityFilterReport.field(type="table profile")
+    notebooks: EntityFilterReport = EntityFilterReport.field(type="notebook")
 
     num_queries: int = 0
     num_queries_dropped_parse_failure: int = 0
-    num_queries_dropped_missing_table: int = 0  # Can be due to pattern filter
-    num_queries_dropped_duplicate_table: int = 0
+    num_queries_missing_table: int = 0  # Can be due to pattern filter
+    num_queries_duplicate_table: int = 0
     num_queries_parsed_by_spark_plan: int = 0
 
     # Distinguish from Operations emitted for created / updated timestamps
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
index 493acb939c3bb..f2da1aece9fd4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
@@ -2,7 +2,7 @@
 import re
 import time
 from datetime import timedelta
-from typing import Dict, Iterable, List, Optional, Set
+from typing import Dict, Iterable, List, Optional, Set, Union
 from urllib.parse import urljoin
 
 from datahub.emitter.mce_builder import (
@@ -18,6 +18,7 @@
     CatalogKey,
     ContainerKey,
     MetastoreKey,
+    NotebookKey,
     UnitySchemaKey,
     add_dataset_to_container,
     gen_containers,
@@ -56,6 +57,8 @@
     Catalog,
     Column,
     Metastore,
+    Notebook,
+    NotebookId,
     Schema,
     ServicePrincipal,
     Table,
@@ -69,6 +72,7 @@
     ViewProperties,
 )
 from datahub.metadata.schema_classes import (
+    BrowsePathsClass,
     DataPlatformInstanceClass,
     DatasetLineageTypeClass,
     DatasetPropertiesClass,
@@ -88,6 +92,7 @@
     UpstreamClass,
     UpstreamLineageClass,
 )
+from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.hive_schema_to_avro import get_schema_fields_for_hive_column
 from datahub.utilities.registries.domain_registry import DomainRegistry
 
@@ -157,6 +162,7 @@ def __init__(self, ctx: PipelineContext, config: UnityCatalogSourceConfig):
         # Global set of table refs
         self.table_refs: Set[TableReference] = set()
         self.view_refs: Set[TableReference] = set()
+        self.notebooks: FileBackedDict[Notebook] = FileBackedDict()
 
     @staticmethod
     def test_connection(config_dict: dict) -> TestConnectionReport:
@@ -176,6 +182,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        self.report.report_ingestion_stage_start("Start warehouse")
         wait_on_warehouse = None
         if self.config.is_profiling_enabled():
             # Can take several minutes, so start now and wait later
@@ -187,10 +194,23 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 )
                 return
 
+        self.report.report_ingestion_stage_start("Ingest service principals")
         self.build_service_principal_map()
+        if self.config.include_notebooks:
+            self.report.report_ingestion_stage_start("Ingest notebooks")
+            yield from self.process_notebooks()
+
         yield from self.process_metastores()
 
+        if self.config.include_notebooks:
+            self.report.report_ingestion_stage_start("Notebook lineage")
+            for notebook in self.notebooks.values():
+                wu = self._gen_notebook_lineage(notebook)
+                if wu:
+                    yield wu
+
         if self.config.include_usage_statistics:
+            self.report.report_ingestion_stage_start("Ingest usage")
             usage_extractor = UnityCatalogUsageExtractor(
                 config=self.config,
                 report=self.report,
@@ -203,6 +223,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             )
 
         if self.config.is_profiling_enabled():
+            self.report.report_ingestion_stage_start("Wait on warehouse")
             assert wait_on_warehouse
             timeout = timedelta(seconds=self.config.profiling.max_wait_secs)
             wait_on_warehouse.result(timeout)
@@ -212,6 +233,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 self.unity_catalog_api_proxy,
                 self.gen_dataset_urn,
             )
+            self.report.report_ingestion_stage_start("Profiling")
             yield from profiling_extractor.get_workunits(self.table_refs)
 
     def build_service_principal_map(self) -> None:
@@ -223,6 +245,56 @@ def build_service_principal_map(self) -> None:
                 "service-principals", f"Unable to fetch service principals: {e}"
             )
 
+    def process_notebooks(self) -> Iterable[MetadataWorkUnit]:
+        for notebook in self.unity_catalog_api_proxy.workspace_notebooks():
+            self.notebooks[str(notebook.id)] = notebook
+            yield from self._gen_notebook_aspects(notebook)
+
+    def _gen_notebook_aspects(self, notebook: Notebook) -> Iterable[MetadataWorkUnit]:
+        mcps = MetadataChangeProposalWrapper.construct_many(
+            entityUrn=self.gen_notebook_urn(notebook),
+            aspects=[
+                DatasetPropertiesClass(
+                    name=notebook.path.rsplit("/", 1)[-1],
+                    customProperties={
+                        "path": notebook.path,
+                        "language": notebook.language.value,
+                    },
+                    externalUrl=urljoin(
+                        self.config.workspace_url, f"#notebook/{notebook.id}"
+                    ),
+                    created=TimeStampClass(int(notebook.created_at.timestamp() * 1000)),
+                    lastModified=TimeStampClass(
+                        int(notebook.modified_at.timestamp() * 1000)
+                    ),
+                ),
+                SubTypesClass(typeNames=[DatasetSubTypes.NOTEBOOK]),
+                BrowsePathsClass(paths=notebook.path.split("/")),
+                # TODO: Add DPI aspect
+            ],
+        )
+        for mcp in mcps:
+            yield mcp.as_workunit()
+
+        self.report.notebooks.processed(notebook.path)
+
+    def _gen_notebook_lineage(self, notebook: Notebook) -> Optional[MetadataWorkUnit]:
+        if not notebook.upstreams:
+            return None
+
+        return MetadataChangeProposalWrapper(
+            entityUrn=self.gen_notebook_urn(notebook),
+            aspect=UpstreamLineageClass(
+                upstreams=[
+                    UpstreamClass(
+                        dataset=self.gen_dataset_urn(upstream_ref),
+                        type=DatasetLineageTypeClass.COPY,
+                    )
+                    for upstream_ref in notebook.upstreams
+                ]
+            ),
+        ).as_workunit()
+
     def process_metastores(self) -> Iterable[MetadataWorkUnit]:
         metastore = self.unity_catalog_api_proxy.assigned_metastore()
         yield from self.gen_metastore_containers(metastore)
@@ -247,6 +319,7 @@ def process_schemas(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]:
                 self.report.schemas.dropped(schema.id)
                 continue
 
+            self.report.report_ingestion_stage_start(f"Ingest schema {schema.id}")
             yield from self.gen_schema_containers(schema)
             yield from self.process_tables(schema)
 
@@ -282,13 +355,21 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn
         ownership = self._create_table_ownership_aspect(table)
         data_platform_instance = self._create_data_platform_instance_aspect(table)
 
-        lineage: Optional[UpstreamLineageClass] = None
         if self.config.include_column_lineage:
-            self.unity_catalog_api_proxy.get_column_lineage(table)
-            lineage = self._generate_column_lineage_aspect(dataset_urn, table)
+            self.unity_catalog_api_proxy.get_column_lineage(
+                table, include_entity_lineage=self.config.include_notebooks
+            )
         elif self.config.include_table_lineage:
-            self.unity_catalog_api_proxy.table_lineage(table)
-            lineage = self._generate_lineage_aspect(dataset_urn, table)
+            self.unity_catalog_api_proxy.table_lineage(
+                table, include_entity_lineage=self.config.include_notebooks
+            )
+        lineage = self._generate_lineage_aspect(dataset_urn, table)
+
+        if self.config.include_notebooks:
+            for notebook_id in table.downstream_notebooks:
+                self.notebooks[str(notebook_id)] = Notebook.add_upstream(
+                    table.ref, self.notebooks[str(notebook_id)]
+                )
 
         yield from [
             mcp.as_workunit()
@@ -308,7 +389,7 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn
             )
         ]
 
-    def _generate_column_lineage_aspect(
+    def _generate_lineage_aspect(
         self, dataset_urn: str, table: Table
     ) -> Optional[UpstreamLineageClass]:
         upstreams: List[UpstreamClass] = []
@@ -318,6 +399,7 @@ def _generate_column_lineage_aspect(
         ):
             upstream_urn = self.gen_dataset_urn(upstream_ref)
 
+            # Should be empty if config.include_column_lineage is False
             finegrained_lineages.extend(
                 FineGrainedLineage(
                     upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
@@ -331,38 +413,28 @@ def _generate_column_lineage_aspect(
                 for d_col, u_cols in sorted(downstream_to_upstream_cols.items())
             )
 
-            upstream_table = UpstreamClass(
-                upstream_urn,
-                DatasetLineageTypeClass.TRANSFORMED,
-            )
-            upstreams.append(upstream_table)
-
-        if upstreams:
-            return UpstreamLineageClass(
-                upstreams=upstreams, fineGrainedLineages=finegrained_lineages
-            )
-        else:
-            return None
-
-    def _generate_lineage_aspect(
-        self, dataset_urn: str, table: Table
-    ) -> Optional[UpstreamLineageClass]:
-        upstreams: List[UpstreamClass] = []
-        for upstream in sorted(table.upstreams.keys()):
-            upstream_urn = make_dataset_urn_with_platform_instance(
-                self.platform,
-                f"{table.schema.catalog.metastore.id}.{upstream}",
-                self.platform_instance_name,
+            upstreams.append(
+                UpstreamClass(
+                    dataset=upstream_urn,
+                    type=DatasetLineageTypeClass.TRANSFORMED,
+                )
             )
 
-            upstream_table = UpstreamClass(
-                upstream_urn,
-                DatasetLineageTypeClass.TRANSFORMED,
+        for notebook in table.upstream_notebooks:
+            upstreams.append(
+                UpstreamClass(
+                    dataset=self.gen_notebook_urn(notebook),
+                    type=DatasetLineageTypeClass.TRANSFORMED,
+                )
             )
-            upstreams.append(upstream_table)
 
         if upstreams:
-            return UpstreamLineageClass(upstreams=upstreams)
+            return UpstreamLineageClass(
+                upstreams=upstreams,
+                fineGrainedLineages=finegrained_lineages
+                if self.config.include_column_lineage
+                else None,
+            )
         else:
             return None
 
@@ -389,6 +461,14 @@ def gen_dataset_urn(self, table_ref: TableReference) -> str:
             name=str(table_ref),
         )
 
+    def gen_notebook_urn(self, notebook: Union[Notebook, NotebookId]) -> str:
+        notebook_id = notebook.id if isinstance(notebook, Notebook) else notebook
+        return NotebookKey(
+            notebook_id=notebook_id,
+            platform=self.platform,
+            instance=self.config.platform_instance,
+        ).as_urn()
+
     def gen_schema_containers(self, schema: Schema) -> Iterable[MetadataWorkUnit]:
         domain_urn = self._gen_domain_urn(f"{schema.catalog.name}.{schema.name}")
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py
index 49f56b46fb012..ab21c1a318659 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py
@@ -214,12 +214,15 @@ def _resolve_tables(
         self, tables: List[str], table_map: TableMap
     ) -> List[TableReference]:
         """Resolve tables to TableReferences, filtering out unrecognized or unresolvable table names."""
+
+        missing_table = False
+        duplicate_table = False
         output = []
         for table in tables:
             table = str(table)
             if table not in table_map:
                 logger.debug(f"Dropping query with unrecognized table: {table}")
-                self.report.num_queries_dropped_missing_table += 1
+                missing_table = True
             else:
                 refs = table_map[table]
                 if len(refs) == 1:
@@ -228,6 +231,11 @@ def _resolve_tables(
                     logger.warning(
                         f"Could not resolve table ref for {table}: {len(refs)} duplicates."
                     )
-                    self.report.num_queries_dropped_duplicate_table += 1
+                    duplicate_table = True
+
+        if missing_table:
+            self.report.num_queries_missing_table += 1
+        if duplicate_table:
+            self.report.num_queries_duplicate_table += 1
 
         return output

From 301d3e6b1ccffaf946f128766578faddbc7ac44e Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 4 Oct 2023 10:23:13 -0400
Subject: [PATCH 002/259] test(ingest/unity): Add Unity Catalog memory
 performance testing (#8932)

---
 .../ingestion/source/unity/proxy_types.py     |   1 -
 .../tests/performance/bigquery/__init__.py    |   0
 .../bigquery_events.py}                       |   0
 .../{ => bigquery}/test_bigquery_usage.py     |  22 +--
 .../tests/performance/data_generation.py      |  53 ++++-
 .../tests/performance/data_model.py           |  31 ++-
 .../tests/performance/databricks/__init__.py  |   0
 .../performance/databricks/test_unity.py      |  71 +++++++
 .../databricks/unity_proxy_mock.py            | 183 ++++++++++++++++++
 .../tests/performance/helpers.py              |  21 ++
 .../tests/unit/test_bigquery_usage.py         |   7 +-
 11 files changed, 356 insertions(+), 33 deletions(-)
 create mode 100644 metadata-ingestion/tests/performance/bigquery/__init__.py
 rename metadata-ingestion/tests/performance/{bigquery.py => bigquery/bigquery_events.py} (100%)
 rename metadata-ingestion/tests/performance/{ => bigquery}/test_bigquery_usage.py (80%)
 create mode 100644 metadata-ingestion/tests/performance/databricks/__init__.py
 create mode 100644 metadata-ingestion/tests/performance/databricks/test_unity.py
 create mode 100644 metadata-ingestion/tests/performance/databricks/unity_proxy_mock.py
 create mode 100644 metadata-ingestion/tests/performance/helpers.py

diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
index d57f20245913f..54ac2e90d7c7e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
@@ -175,7 +175,6 @@ class Table(CommonProperty):
     columns: List[Column]
     storage_location: Optional[str]
     data_source_format: Optional[DataSourceFormat]
-    comment: Optional[str]
     table_type: TableType
     owner: Optional[str]
     generation: Optional[int]
diff --git a/metadata-ingestion/tests/performance/bigquery/__init__.py b/metadata-ingestion/tests/performance/bigquery/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/performance/bigquery.py b/metadata-ingestion/tests/performance/bigquery/bigquery_events.py
similarity index 100%
rename from metadata-ingestion/tests/performance/bigquery.py
rename to metadata-ingestion/tests/performance/bigquery/bigquery_events.py
diff --git a/metadata-ingestion/tests/performance/test_bigquery_usage.py b/metadata-ingestion/tests/performance/bigquery/test_bigquery_usage.py
similarity index 80%
rename from metadata-ingestion/tests/performance/test_bigquery_usage.py
rename to metadata-ingestion/tests/performance/bigquery/test_bigquery_usage.py
index 7e05ef070b45d..bbc3378450bff 100644
--- a/metadata-ingestion/tests/performance/test_bigquery_usage.py
+++ b/metadata-ingestion/tests/performance/bigquery/test_bigquery_usage.py
@@ -2,13 +2,11 @@
 import os
 import random
 from datetime import timedelta
-from typing import Iterable, Tuple
 
 import humanfriendly
 import psutil
 
 from datahub.emitter.mce_builder import make_dataset_urn
-from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.bigquery_v2.bigquery_config import (
     BigQueryUsageConfig,
     BigQueryV2Config,
@@ -16,12 +14,13 @@
 from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report
 from datahub.ingestion.source.bigquery_v2.usage import BigQueryUsageExtractor
 from datahub.utilities.perf_timer import PerfTimer
-from tests.performance.bigquery import generate_events, ref_from_table
+from tests.performance.bigquery.bigquery_events import generate_events, ref_from_table
 from tests.performance.data_generation import (
     NormalDistribution,
     generate_data,
     generate_queries,
 )
+from tests.performance.helpers import workunit_sink
 
 
 def run_test():
@@ -33,7 +32,7 @@ def run_test():
         num_views=2000,
         time_range=timedelta(days=7),
     )
-    all_tables = seed_metadata.tables + seed_metadata.views
+    all_tables = seed_metadata.all_tables
 
     config = BigQueryV2Config(
         start_time=seed_metadata.start_time,
@@ -88,21 +87,6 @@ def run_test():
     print(f"Hash collisions: {report.num_usage_query_hash_collisions}")
 
 
-def workunit_sink(workunits: Iterable[MetadataWorkUnit]) -> Tuple[int, int]:
-    peak_memory_usage = psutil.Process(os.getpid()).memory_info().rss
-    i: int = 0
-    for i, wu in enumerate(workunits):
-        if i % 10_000 == 0:
-            peak_memory_usage = max(
-                peak_memory_usage, psutil.Process(os.getpid()).memory_info().rss
-            )
-    peak_memory_usage = max(
-        peak_memory_usage, psutil.Process(os.getpid()).memory_info().rss
-    )
-
-    return i, peak_memory_usage
-
-
 if __name__ == "__main__":
     root_logger = logging.getLogger()
     root_logger.setLevel(logging.INFO)
diff --git a/metadata-ingestion/tests/performance/data_generation.py b/metadata-ingestion/tests/performance/data_generation.py
index c530848f27f5c..67b156896909a 100644
--- a/metadata-ingestion/tests/performance/data_generation.py
+++ b/metadata-ingestion/tests/performance/data_generation.py
@@ -11,11 +11,14 @@
 import uuid
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
-from typing import Iterable, List, TypeVar
+from typing import Iterable, List, TypeVar, Union, cast
 
 from faker import Faker
 
 from tests.performance.data_model import (
+    Column,
+    ColumnMapping,
+    ColumnType,
     Container,
     FieldAccess,
     Query,
@@ -52,15 +55,21 @@ def sample_with_floor(self, floor: int = 1) -> int:
 
 @dataclass
 class SeedMetadata:
-    containers: List[Container]
+    # Each list is a layer of containers, e.g. [[databases], [schemas]]
+    containers: List[List[Container]]
+
     tables: List[Table]
     views: List[View]
     start_time: datetime
     end_time: datetime
 
+    @property
+    def all_tables(self) -> List[Table]:
+        return self.tables + cast(List[Table], self.views)
+
 
 def generate_data(
-    num_containers: int,
+    num_containers: Union[List[int], int],
     num_tables: int,
     num_views: int,
     columns_per_table: NormalDistribution = NormalDistribution(5, 2),
@@ -68,32 +77,52 @@ def generate_data(
     view_definition_length: NormalDistribution = NormalDistribution(150, 50),
     time_range: timedelta = timedelta(days=14),
 ) -> SeedMetadata:
-    containers = [Container(f"container-{i}") for i in range(num_containers)]
+    # Assemble containers
+    if isinstance(num_containers, int):
+        num_containers = [num_containers]
+
+    containers: List[List[Container]] = []
+    for i, num_in_layer in enumerate(num_containers):
+        layer = [
+            Container(
+                f"{i}-container-{j}",
+                parent=random.choice(containers[-1]) if containers else None,
+            )
+            for j in range(num_in_layer)
+        ]
+        containers.append(layer)
+
+    # Assemble tables
     tables = [
         Table(
             f"table-{i}",
-            container=random.choice(containers),
+            container=random.choice(containers[-1]),
             columns=[
                 f"column-{j}-{uuid.uuid4()}"
                 for j in range(columns_per_table.sample_with_floor())
             ],
+            column_mapping=None,
         )
         for i in range(num_tables)
     ]
     views = [
         View(
             f"view-{i}",
-            container=random.choice(containers),
+            container=random.choice(containers[-1]),
             columns=[
                 f"column-{j}-{uuid.uuid4()}"
                 for j in range(columns_per_table.sample_with_floor())
             ],
+            column_mapping=None,
             definition=f"{uuid.uuid4()}-{'*' * view_definition_length.sample_with_floor(10)}",
             parents=random.sample(tables, parents_per_view.sample_with_floor()),
         )
         for i in range(num_views)
     ]
 
+    for table in tables + views:
+        _generate_column_mapping(table)
+
     now = datetime.now(tz=timezone.utc)
     return SeedMetadata(
         containers=containers,
@@ -162,6 +191,18 @@ def generate_queries(
         )
 
 
+def _generate_column_mapping(table: Table) -> ColumnMapping:
+    d = {}
+    for column in table.columns:
+        d[column] = Column(
+            name=column,
+            type=random.choice(list(ColumnType)),
+            nullable=random.random() < 0.1,  # Fixed 10% chance for now
+        )
+    table.column_mapping = d
+    return d
+
+
 def _sample_list(lst: List[T], dist: NormalDistribution, floor: int = 1) -> List[T]:
     return random.sample(lst, min(dist.sample_with_floor(floor), len(lst)))
 
diff --git a/metadata-ingestion/tests/performance/data_model.py b/metadata-ingestion/tests/performance/data_model.py
index c593e69ceb9a7..9425fa827070e 100644
--- a/metadata-ingestion/tests/performance/data_model.py
+++ b/metadata-ingestion/tests/performance/data_model.py
@@ -1,10 +1,10 @@
 from dataclasses import dataclass
 from datetime import datetime
-from typing import List, Optional
+from enum import Enum
+from typing import Dict, List, Optional
 
 from typing_extensions import Literal
 
-Column = str
 StatementType = Literal[  # SELECT + values from OperationTypeClass
     "SELECT",
     "INSERT",
@@ -21,13 +21,36 @@
 @dataclass
 class Container:
     name: str
+    parent: Optional["Container"] = None
+
+
+class ColumnType(str, Enum):
+    # Can add types that take parameters in the future
+
+    INTEGER = "INTEGER"
+    FLOAT = "FLOAT"  # Double precision (64 bit)
+    STRING = "STRING"
+    BOOLEAN = "BOOLEAN"
+    DATETIME = "DATETIME"
+
+
+@dataclass
+class Column:
+    name: str
+    type: ColumnType
+    nullable: bool
+
+
+ColumnRef = str
+ColumnMapping = Dict[ColumnRef, Column]
 
 
 @dataclass
 class Table:
     name: str
     container: Container
-    columns: List[Column]
+    columns: List[ColumnRef]
+    column_mapping: Optional[ColumnMapping]
 
     def is_view(self) -> bool:
         return False
@@ -44,7 +67,7 @@ def is_view(self) -> bool:
 
 @dataclass
 class FieldAccess:
-    column: Column
+    column: ColumnRef
     table: Table
 
 
diff --git a/metadata-ingestion/tests/performance/databricks/__init__.py b/metadata-ingestion/tests/performance/databricks/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/performance/databricks/test_unity.py b/metadata-ingestion/tests/performance/databricks/test_unity.py
new file mode 100644
index 0000000000000..cc9558f0692ed
--- /dev/null
+++ b/metadata-ingestion/tests/performance/databricks/test_unity.py
@@ -0,0 +1,71 @@
+import logging
+import os
+from unittest.mock import patch
+
+import humanfriendly
+import psutil
+
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.source.unity.config import UnityCatalogSourceConfig
+from datahub.ingestion.source.unity.source import UnityCatalogSource
+from datahub.utilities.perf_timer import PerfTimer
+from tests.performance.data_generation import (
+    NormalDistribution,
+    generate_data,
+    generate_queries,
+)
+from tests.performance.databricks.unity_proxy_mock import UnityCatalogApiProxyMock
+from tests.performance.helpers import workunit_sink
+
+
+def run_test():
+    seed_metadata = generate_data(
+        num_containers=[1, 100, 5000],
+        num_tables=50000,
+        num_views=10000,
+        columns_per_table=NormalDistribution(100, 50),
+        parents_per_view=NormalDistribution(5, 5),
+        view_definition_length=NormalDistribution(1000, 300),
+    )
+    queries = generate_queries(
+        seed_metadata,
+        num_selects=100000,
+        num_operations=100000,
+        num_unique_queries=10000,
+        num_users=1000,
+    )
+    proxy_mock = UnityCatalogApiProxyMock(
+        seed_metadata, queries=queries, num_service_principals=10000
+    )
+    print("Data generated")
+
+    config = UnityCatalogSourceConfig(
+        token="", workspace_url="http://localhost:1234", include_usage_statistics=False
+    )
+    ctx = PipelineContext(run_id="test")
+    with patch(
+        "datahub.ingestion.source.unity.source.UnityCatalogApiProxy",
+        lambda *args, **kwargs: proxy_mock,
+    ):
+        source: UnityCatalogSource = UnityCatalogSource(ctx, config)
+
+    pre_mem_usage = psutil.Process(os.getpid()).memory_info().rss
+    print(f"Test data size: {humanfriendly.format_size(pre_mem_usage)}")
+
+    with PerfTimer() as timer:
+        workunits = source.get_workunits()
+        num_workunits, peak_memory_usage = workunit_sink(workunits)
+        print(f"Workunits Generated: {num_workunits}")
+        print(f"Seconds Elapsed: {timer.elapsed_seconds():.2f} seconds")
+
+    print(
+        f"Peak Memory Used: {humanfriendly.format_size(peak_memory_usage - pre_mem_usage)}"
+    )
+    print(source.report.aspects)
+
+
+if __name__ == "__main__":
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.INFO)
+    root_logger.addHandler(logging.StreamHandler())
+    run_test()
diff --git a/metadata-ingestion/tests/performance/databricks/unity_proxy_mock.py b/metadata-ingestion/tests/performance/databricks/unity_proxy_mock.py
new file mode 100644
index 0000000000000..593163e12bf0a
--- /dev/null
+++ b/metadata-ingestion/tests/performance/databricks/unity_proxy_mock.py
@@ -0,0 +1,183 @@
+import uuid
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Dict, Iterable, List
+
+from databricks.sdk.service.catalog import ColumnTypeName
+from databricks.sdk.service.sql import QueryStatementType
+
+from datahub.ingestion.source.unity.proxy_types import (
+    Catalog,
+    CatalogType,
+    Column,
+    Metastore,
+    Query,
+    Schema,
+    ServicePrincipal,
+    Table,
+    TableType,
+)
+from tests.performance import data_model
+from tests.performance.data_generation import SeedMetadata
+from tests.performance.data_model import ColumnType, StatementType
+
+
+class UnityCatalogApiProxyMock:
+    """Mimics UnityCatalogApiProxy for performance testing."""
+
+    def __init__(
+        self,
+        seed_metadata: SeedMetadata,
+        queries: Iterable[data_model.Query] = (),
+        num_service_principals: int = 0,
+    ) -> None:
+        self.seed_metadata = seed_metadata
+        self.queries = queries
+        self.num_service_principals = num_service_principals
+        self.warehouse_id = "invalid-warehouse-id"
+
+        # Cache for performance
+        self._schema_to_table: Dict[str, List[data_model.Table]] = defaultdict(list)
+        for table in seed_metadata.all_tables:
+            self._schema_to_table[table.container.name].append(table)
+
+    def check_basic_connectivity(self) -> bool:
+        return True
+
+    def assigned_metastore(self) -> Metastore:
+        container = self.seed_metadata.containers[0][0]
+        return Metastore(
+            id=container.name,
+            name=container.name,
+            global_metastore_id=container.name,
+            metastore_id=container.name,
+            comment=None,
+            owner=None,
+            cloud=None,
+            region=None,
+        )
+
+    def catalogs(self, metastore: Metastore) -> Iterable[Catalog]:
+        for container in self.seed_metadata.containers[1]:
+            if not container.parent or metastore.name != container.parent.name:
+                continue
+
+            yield Catalog(
+                id=f"{metastore.id}.{container.name}",
+                name=container.name,
+                metastore=metastore,
+                comment=None,
+                owner=None,
+                type=CatalogType.MANAGED_CATALOG,
+            )
+
+    def schemas(self, catalog: Catalog) -> Iterable[Schema]:
+        for container in self.seed_metadata.containers[2]:
+            # Assumes all catalog names are unique
+            if not container.parent or catalog.name != container.parent.name:
+                continue
+
+            yield Schema(
+                id=f"{catalog.id}.{container.name}",
+                name=container.name,
+                catalog=catalog,
+                comment=None,
+                owner=None,
+            )
+
+    def tables(self, schema: Schema) -> Iterable[Table]:
+        for table in self._schema_to_table[schema.name]:
+            columns = []
+            if table.column_mapping:
+                for i, col_name in enumerate(table.columns):
+                    column = table.column_mapping[col_name]
+                    columns.append(
+                        Column(
+                            id=column.name,
+                            name=column.name,
+                            type_name=self._convert_column_type(column.type),
+                            type_text=column.type.value,
+                            nullable=column.nullable,
+                            position=i,
+                            comment=None,
+                            type_precision=0,
+                            type_scale=0,
+                        )
+                    )
+
+            yield Table(
+                id=f"{schema.id}.{table.name}",
+                name=table.name,
+                schema=schema,
+                table_type=TableType.VIEW if table.is_view() else TableType.MANAGED,
+                columns=columns,
+                created_at=datetime.now(tz=timezone.utc),
+                comment=None,
+                owner=None,
+                storage_location=None,
+                data_source_format=None,
+                generation=None,
+                created_by="",
+                updated_at=None,
+                updated_by=None,
+                table_id="",
+                view_definition=table.definition
+                if isinstance(table, data_model.View)
+                else None,
+                properties={},
+            )
+
+    def service_principals(self) -> Iterable[ServicePrincipal]:
+        for i in range(self.num_service_principals):
+            yield ServicePrincipal(
+                id=str(i),
+                application_id=str(uuid.uuid4()),
+                display_name=f"user-{i}",
+                active=True,
+            )
+
+    def query_history(
+        self,
+        start_time: datetime,
+        end_time: datetime,
+    ) -> Iterable[Query]:
+        for i, query in enumerate(self.queries):
+            yield Query(
+                query_id=str(i),
+                query_text=query.text,
+                statement_type=self._convert_statement_type(query.type),
+                start_time=query.timestamp,
+                end_time=query.timestamp,
+                user_id=hash(query.actor),
+                user_name=query.actor,
+                executed_as_user_id=hash(query.actor),
+                executed_as_user_name=None,
+            )
+
+    def table_lineage(self, table: Table) -> None:
+        pass
+
+    def get_column_lineage(self, table: Table) -> None:
+        pass
+
+    @staticmethod
+    def _convert_column_type(t: ColumnType) -> ColumnTypeName:
+        if t == ColumnType.INTEGER:
+            return ColumnTypeName.INT
+        elif t == ColumnType.FLOAT:
+            return ColumnTypeName.DOUBLE
+        elif t == ColumnType.STRING:
+            return ColumnTypeName.STRING
+        elif t == ColumnType.BOOLEAN:
+            return ColumnTypeName.BOOLEAN
+        elif t == ColumnType.DATETIME:
+            return ColumnTypeName.TIMESTAMP
+        else:
+            raise ValueError(f"Unknown column type: {t}")
+
+    @staticmethod
+    def _convert_statement_type(t: StatementType) -> QueryStatementType:
+        if t == "CUSTOM" or t == "UNKNOWN":
+            return QueryStatementType.OTHER
+        else:
+            return QueryStatementType[t]
diff --git a/metadata-ingestion/tests/performance/helpers.py b/metadata-ingestion/tests/performance/helpers.py
new file mode 100644
index 0000000000000..eb98e53670c96
--- /dev/null
+++ b/metadata-ingestion/tests/performance/helpers.py
@@ -0,0 +1,21 @@
+import os
+from typing import Iterable, Tuple
+
+import psutil
+
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+
+
+def workunit_sink(workunits: Iterable[MetadataWorkUnit]) -> Tuple[int, int]:
+    peak_memory_usage = psutil.Process(os.getpid()).memory_info().rss
+    i: int = 0
+    for i, wu in enumerate(workunits):
+        if i % 10_000 == 0:
+            peak_memory_usage = max(
+                peak_memory_usage, psutil.Process(os.getpid()).memory_info().rss
+            )
+    peak_memory_usage = max(
+        peak_memory_usage, psutil.Process(os.getpid()).memory_info().rss
+    )
+
+    return i, peak_memory_usage
diff --git a/metadata-ingestion/tests/unit/test_bigquery_usage.py b/metadata-ingestion/tests/unit/test_bigquery_usage.py
index e06c6fb3fe7e5..1eb5d8b00e27c 100644
--- a/metadata-ingestion/tests/unit/test_bigquery_usage.py
+++ b/metadata-ingestion/tests/unit/test_bigquery_usage.py
@@ -35,7 +35,7 @@
     TimeWindowSizeClass,
 )
 from datahub.testing.compare_metadata_json import diff_metadata_json
-from tests.performance.bigquery import generate_events, ref_from_table
+from tests.performance.bigquery.bigquery_events import generate_events, ref_from_table
 from tests.performance.data_generation import generate_data, generate_queries
 from tests.performance.data_model import Container, FieldAccess, Query, Table, View
 
@@ -45,14 +45,15 @@
 ACTOR_2, ACTOR_2_URN = "b@acryl.io", "urn:li:corpuser:b"
 DATABASE_1 = Container("database_1")
 DATABASE_2 = Container("database_2")
-TABLE_1 = Table("table_1", DATABASE_1, ["id", "name", "age"])
-TABLE_2 = Table("table_2", DATABASE_1, ["id", "table_1_id", "value"])
+TABLE_1 = Table("table_1", DATABASE_1, ["id", "name", "age"], None)
+TABLE_2 = Table("table_2", DATABASE_1, ["id", "table_1_id", "value"], None)
 VIEW_1 = View(
     name="view_1",
     container=DATABASE_1,
     columns=["id", "name", "total"],
     definition="VIEW DEFINITION 1",
     parents=[TABLE_1, TABLE_2],
+    column_mapping=None,
 )
 ALL_TABLES = [TABLE_1, TABLE_2, VIEW_1]
 

From 165aa54d1e6f1a1707f79be3cce39ec06c8a1652 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= <sgomezvillamor@gmail.com>
Date: Wed, 4 Oct 2023 19:24:04 +0200
Subject: [PATCH 003/259] doc: DataHubUpgradeHistory_v1 (#8918)

---
 docs/deploy/confluent-cloud.md |  5 +++++
 docs/how/kafka-config.md       | 17 +++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/docs/deploy/confluent-cloud.md b/docs/deploy/confluent-cloud.md
index 794b55d4686bf..096fd9984f474 100644
--- a/docs/deploy/confluent-cloud.md
+++ b/docs/deploy/confluent-cloud.md
@@ -16,6 +16,11 @@ First, you'll need to create following new topics in the [Confluent Control Cent
 6. (Deprecated) **MetadataChangeEvent_v4**: Metadata change proposal messages
 7. (Deprecated) **MetadataAuditEvent_v4**: Metadata change log messages
 8. (Deprecated) **FailedMetadataChangeEvent_v4**: Failed to process #1 event
+9. **MetadataGraphEvent_v4**:
+10. **MetadataGraphEvent_v4**:
+11. **PlatformEvent_v1**
+12. **DataHubUpgradeHistory_v1**: Notifies the end of DataHub Upgrade job so dependants can act accordingly (_eg_, startup).
+    Note this topic requires special configuration: **Infinite retention**. Also, 1 partition is enough for the occasional traffic.
 
 The first five are the most important, and are explained in more depth in [MCP/MCL](../advanced/mcp-mcl.md). The final topics are
 those which are deprecated but still used under certain circumstances. It is likely that in the future they will be completely 
diff --git a/docs/how/kafka-config.md b/docs/how/kafka-config.md
index f3f81c3d07c01..2f20e8b548f83 100644
--- a/docs/how/kafka-config.md
+++ b/docs/how/kafka-config.md
@@ -52,16 +52,21 @@ Also see [Kafka Connect Security](https://docs.confluent.io/current/connect/secu
 
 By default, DataHub relies on the a set of Kafka topics to operate. By default, they have the following names:
 
-- **MetadataChangeProposal_v1**
-- **FailedMetadataChangeProposal_v1**
-- **MetadataChangeLog_Versioned_v1**
-- **MetadataChangeLog_Timeseries_v1**
-- **DataHubUsageEvent_v1**: User behavior tracking event for UI
+1. **MetadataChangeProposal_v1**
+2. **FailedMetadataChangeProposal_v1**
+3. **MetadataChangeLog_Versioned_v1**
+4. **MetadataChangeLog_Timeseries_v1**
+5. **DataHubUsageEvent_v1**: User behavior tracking event for UI
 6. (Deprecated) **MetadataChangeEvent_v4**: Metadata change proposal messages
 7. (Deprecated) **MetadataAuditEvent_v4**: Metadata change log messages
 8. (Deprecated) **FailedMetadataChangeEvent_v4**: Failed to process #1 event
+9. **MetadataGraphEvent_v4**:
+10. **MetadataGraphEvent_v4**:
+11. **PlatformEvent_v1**:
+12. **DataHubUpgradeHistory_v1**: Notifies the end of DataHub Upgrade job so dependants can act accordingly (_eg_, startup).
+    Note this topic requires special configuration: **Infinite retention**. Also, 1 partition is enough for the occasional traffic.
 
-These topics are discussed at more length in [Metadata Events](../what/mxe.md).
+How Metadata Events relate to these topics is discussed at more length in [Metadata Events](../what/mxe.md).
 
 We've included environment variables to customize the name each of these topics, for cases where an organization has naming rules for your topics.
 

From 3a9452c2072c95cbd7a4bf1270b4ef07abd1b1eb Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Thu, 5 Oct 2023 03:42:00 +0900
Subject: [PATCH 004/259] fix: fix typo on aws guide (#8944)

---
 docs/deploy/aws.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md
index 228fcb51d1a28..e0f57b4a0b0cb 100644
--- a/docs/deploy/aws.md
+++ b/docs/deploy/aws.md
@@ -100,7 +100,7 @@ eksctl create iamserviceaccount \
 Install the TargetGroupBinding custom resource definition by running the following.
 
 ```
-kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master"
+kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master"
 ```
 
 Add the helm chart repository containing the latest version of the ALB controller.

From e2afd44bfeb287e8365b99bc7677d06e4172643b Mon Sep 17 00:00:00 2001
From: ethan-cartwright <ethan.cartwright.m@gmail.com>
Date: Wed, 4 Oct 2023 16:38:58 -0400
Subject: [PATCH 005/259] feat(dbt-ingestion): add documentation link from dbt
 source to institutionalMemory (#8686)

Co-authored-by: Ethan Cartwright <ethan.cartwright@acryl.io>
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../docs/sources/dbt/dbt-cloud_recipe.yml     |  8 +--
 metadata-ingestion/docs/sources/dbt/dbt.md    |  7 ++
 .../ingestion/source/dbt/dbt_common.py        |  6 ++
 .../src/datahub/utilities/mapping.py          | 67 ++++++++++++++++++-
 metadata-ingestion/tests/unit/test_mapping.py | 41 ++++++++++++
 5 files changed, 123 insertions(+), 6 deletions(-)

diff --git a/metadata-ingestion/docs/sources/dbt/dbt-cloud_recipe.yml b/metadata-ingestion/docs/sources/dbt/dbt-cloud_recipe.yml
index 113303cfc1ad4..ef0776b189ca9 100644
--- a/metadata-ingestion/docs/sources/dbt/dbt-cloud_recipe.yml
+++ b/metadata-ingestion/docs/sources/dbt/dbt-cloud_recipe.yml
@@ -6,14 +6,14 @@ source:
     # In the URL https://cloud.getdbt.com/next/deploy/107298/projects/175705/jobs/148094,
     # 107298 is the account_id, 175705 is the project_id, and 148094 is the job_id
 
-    account_id: # set to your dbt cloud account id
-    project_id: # set to your dbt cloud project id
-    job_id: # set to your dbt cloud job id
+    account_id: "${DBT_ACCOUNT_ID}" # set to your dbt cloud account id
+    project_id: "${DBT_PROJECT_ID}" # set to your dbt cloud project id
+    job_id: "${DBT_JOB_ID}" # set to your dbt cloud job id
     run_id: # set to your dbt cloud run id. This is optional, and defaults to the latest run
 
     target_platform: postgres
 
     # Options
-    target_platform: "my_target_platform_id" # e.g. bigquery/postgres/etc.
+    target_platform: "${TARGET_PLATFORM_ID}" # e.g. bigquery/postgres/etc.
 
 # sink configs
diff --git a/metadata-ingestion/docs/sources/dbt/dbt.md b/metadata-ingestion/docs/sources/dbt/dbt.md
index bfc3ebd5bb350..43ced13c3b1f8 100644
--- a/metadata-ingestion/docs/sources/dbt/dbt.md
+++ b/metadata-ingestion/docs/sources/dbt/dbt.md
@@ -38,6 +38,12 @@ meta_mapping:
     operation: "add_terms"
     config:
       separator: ","
+  documentation_link:
+    match: "(?:https?)?\:\/\/\w*[^#]*"
+    operation: "add_doc_link"
+    config:
+      link: {{ $match }}
+      description: "Documentation Link"
 column_meta_mapping:
   terms_list:
     match: ".*"
@@ -57,6 +63,7 @@ We support the following operations:
 2. add_term - Requires `term` property in config.
 3. add_terms - Accepts an optional `separator` property in config.
 4. add_owner - Requires `owner_type` property in config which can be either user or group. Optionally accepts the `owner_category` config property which you can set to one of `['TECHNICAL_OWNER', 'BUSINESS_OWNER', 'DATA_STEWARD', 'DATAOWNER'` (defaults to `DATAOWNER`).
+5. add_doc_link - Requires `link` and `description` properties in config. Upon ingestion run, this will overwrite current links in the institutional knowledge section with this new link. The anchor text is defined here in the meta_mappings as `description`.
 
 Note:
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 782d94f39e8a5..3edeb695e9f21 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -1188,9 +1188,15 @@ def _generate_base_aspects(
         ):
             aspects.append(meta_aspects.get(Constants.ADD_TERM_OPERATION))
 
+        # add meta links aspect
+        meta_links_aspect = meta_aspects.get(Constants.ADD_DOC_LINK_OPERATION)
+        if meta_links_aspect and self.config.enable_meta_mapping:
+            aspects.append(meta_links_aspect)
+
         # add schema metadata aspect
         schema_metadata = self.get_schema_metadata(self.report, node, mce_platform)
         aspects.append(schema_metadata)
+
         return aspects
 
     def get_schema_metadata(
diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py
index 793eccfb22c7e..eb2d975ee607f 100644
--- a/metadata-ingestion/src/datahub/utilities/mapping.py
+++ b/metadata-ingestion/src/datahub/utilities/mapping.py
@@ -2,12 +2,16 @@
 import logging
 import operator
 import re
+import time
 from functools import reduce
-from typing import Any, Dict, List, Match, Optional, Union
+from typing import Any, Dict, List, Match, Optional, Union, cast
 
 from datahub.emitter import mce_builder
 from datahub.emitter.mce_builder import OwnerType
 from datahub.metadata.schema_classes import (
+    AuditStampClass,
+    InstitutionalMemoryClass,
+    InstitutionalMemoryMetadataClass,
     OwnerClass,
     OwnershipClass,
     OwnershipSourceClass,
@@ -39,6 +43,7 @@ def _insert_match_value(original_value: str, match_value: str) -> str:
 
 
 class Constants:
+    ADD_DOC_LINK_OPERATION = "add_doc_link"
     ADD_TAG_OPERATION = "add_tag"
     ADD_TERM_OPERATION = "add_term"
     ADD_TERMS_OPERATION = "add_terms"
@@ -47,6 +52,8 @@ class Constants:
     OPERATION_CONFIG = "config"
     TAG = "tag"
     TERM = "term"
+    DOC_LINK = "link"
+    DOC_DESCRIPTION = "description"
     OWNER_TYPE = "owner_type"
     OWNER_CATEGORY = "owner_category"
     MATCH = "match"
@@ -163,7 +170,6 @@ def process(self, raw_props: Dict[str, Any]) -> Dict[str, Any]:
                             )
                             operations_value_list.append(operation)  # type: ignore
                             operations_map[operation_type] = operations_value_list
-
             aspect_map = self.convert_to_aspects(operations_map)
         except Exception as e:
             logger.error(f"Error while processing operation defs over raw_props: {e}")
@@ -173,6 +179,7 @@ def convert_to_aspects(
         self, operation_map: Dict[str, Union[set, list]]
     ) -> Dict[str, Any]:
         aspect_map: Dict[str, Any] = {}
+
         if Constants.ADD_TAG_OPERATION in operation_map:
             tag_aspect = mce_builder.make_global_tag_aspect_with_tag_list(
                 sorted(operation_map[Constants.ADD_TAG_OPERATION])
@@ -195,11 +202,57 @@ def convert_to_aspects(
                 ]
             )
             aspect_map[Constants.ADD_OWNER_OPERATION] = owner_aspect
+
         if Constants.ADD_TERM_OPERATION in operation_map:
             term_aspect = mce_builder.make_glossary_terms_aspect_from_urn_list(
                 sorted(operation_map[Constants.ADD_TERM_OPERATION])
             )
             aspect_map[Constants.ADD_TERM_OPERATION] = term_aspect
+
+        if Constants.ADD_DOC_LINK_OPERATION in operation_map:
+            try:
+                if len(
+                    operation_map[Constants.ADD_DOC_LINK_OPERATION]
+                ) == 1 and isinstance(
+                    operation_map[Constants.ADD_DOC_LINK_OPERATION], list
+                ):
+                    docs_dict = cast(
+                        List[Dict], operation_map[Constants.ADD_DOC_LINK_OPERATION]
+                    )[0]
+                    if "description" not in docs_dict or "link" not in docs_dict:
+                        raise Exception(
+                            "Documentation_link meta_mapping config needs a description key and a link key"
+                        )
+
+                    now = int(time.time() * 1000)  # milliseconds since epoch
+                    institutional_memory_element = InstitutionalMemoryMetadataClass(
+                        url=docs_dict["link"],
+                        description=docs_dict["description"],
+                        createStamp=AuditStampClass(
+                            time=now, actor="urn:li:corpuser:ingestion"
+                        ),
+                    )
+
+                    # create a new institutional memory aspect
+                    institutional_memory_aspect = InstitutionalMemoryClass(
+                        elements=[institutional_memory_element]
+                    )
+
+                    aspect_map[
+                        Constants.ADD_DOC_LINK_OPERATION
+                    ] = institutional_memory_aspect
+                else:
+                    raise Exception(
+                        f"Expected 1 item of type list for the documentation_link meta_mapping config,"
+                        f" received type of {type(operation_map[Constants.ADD_DOC_LINK_OPERATION])}"
+                        f", and size of {len(operation_map[Constants.ADD_DOC_LINK_OPERATION])}."
+                    )
+
+            except Exception as e:
+                logger.error(
+                    f"Error while constructing aspect for documentation link and description : {e}"
+                )
+
         return aspect_map
 
     def get_operation_value(
@@ -248,6 +301,16 @@ def get_operation_value(
             term = operation_config[Constants.TERM]
             term = _insert_match_value(term, _get_best_match(match, "term"))
             return mce_builder.make_term_urn(term)
+        elif (
+            operation_type == Constants.ADD_DOC_LINK_OPERATION
+            and operation_config[Constants.DOC_LINK]
+            and operation_config[Constants.DOC_DESCRIPTION]
+        ):
+            link = operation_config[Constants.DOC_LINK]
+            link = _insert_match_value(link, _get_best_match(match, "link"))
+            description = operation_config[Constants.DOC_DESCRIPTION]
+            return {"link": link, "description": description}
+
         elif operation_type == Constants.ADD_TERMS_OPERATION:
             separator = operation_config.get(Constants.SEPARATOR, ",")
             captured_terms = match.group(0)
diff --git a/metadata-ingestion/tests/unit/test_mapping.py b/metadata-ingestion/tests/unit/test_mapping.py
index d69dd4a8a96b0..5c258f16535f8 100644
--- a/metadata-ingestion/tests/unit/test_mapping.py
+++ b/metadata-ingestion/tests/unit/test_mapping.py
@@ -4,6 +4,7 @@
 from datahub.metadata.schema_classes import (
     GlobalTagsClass,
     GlossaryTermsClass,
+    InstitutionalMemoryClass,
     OwnerClass,
     OwnershipClass,
     OwnershipSourceTypeClass,
@@ -233,6 +234,46 @@ def test_operation_processor_advanced_matching_tags():
     assert tag_aspect.tags[0].tag == "urn:li:tag:case_4567"
 
 
+def test_operation_processor_institutional_memory():
+    raw_props = {
+        "documentation_link": "https://test.com/documentation#ignore-this",
+    }
+    processor = OperationProcessor(
+        operation_defs={
+            "documentation_link": {
+                "match": r"(?:https?)?\:\/\/\w*[^#]*",
+                "operation": "add_doc_link",
+                "config": {"link": "{{ $match }}", "description": "test"},
+            },
+        },
+    )
+    aspect_map = processor.process(raw_props)
+    assert "add_doc_link" in aspect_map
+
+    doc_link_aspect: InstitutionalMemoryClass = aspect_map["add_doc_link"]
+
+    assert doc_link_aspect.elements[0].url == "https://test.com/documentation"
+    assert doc_link_aspect.elements[0].description == "test"
+
+
+def test_operation_processor_institutional_memory_no_description():
+    raw_props = {
+        "documentation_link": "test.com/documentation#ignore-this",
+    }
+    processor = OperationProcessor(
+        operation_defs={
+            "documentation_link": {
+                "match": r"(?:https?)?\:\/\/\w*[^#]*",
+                "operation": "add_doc_link",
+                "config": {"link": "{{ $match }}"},
+            },
+        },
+    )
+    # we require a description, so this should stay empty
+    aspect_map = processor.process(raw_props)
+    assert aspect_map == {}
+
+
 def test_operation_processor_matching_nested_props():
     raw_props = {
         "gdpr": {

From 0f8d2757352597ceaed62b93547381255dbc650e Mon Sep 17 00:00:00 2001
From: John Joyce <john@acryl.io>
Date: Wed, 4 Oct 2023 20:03:40 -0700
Subject: [PATCH 006/259] refactor(style): Improve search bar input focus +
 styling (#8955)

---
 .../src/app/search/SearchBar.tsx              | 46 +++++++++++--------
 .../src/app/shared/admin/HeaderLinks.tsx      | 28 +++++------
 .../src/conf/theme/theme_dark.config.json     |  4 +-
 .../src/conf/theme/theme_light.config.json    |  4 +-
 4 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
index fb10e1ca0026e..b4699994bc460 100644
--- a/datahub-web-react/src/app/search/SearchBar.tsx
+++ b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -6,7 +6,7 @@ import { useHistory } from 'react-router';
 import { AutoCompleteResultForEntity, EntityType, FacetFilterInput, ScenarioType } from '../../types.generated';
 import EntityRegistry from '../entity/EntityRegistry';
 import filterSearchQuery from './utils/filterSearchQuery';
-import { ANTD_GRAY, ANTD_GRAY_V2 } from '../entity/shared/constants';
+import { ANTD_GRAY, ANTD_GRAY_V2, REDESIGN_COLORS } from '../entity/shared/constants';
 import { getEntityPath } from '../entity/shared/containers/profile/utils';
 import { EXACT_SEARCH_PREFIX } from './utils/constants';
 import { useListRecommendationsQuery } from '../../graphql/recommendations.generated';
@@ -20,7 +20,6 @@ import RecommendedOption from './autoComplete/RecommendedOption';
 import SectionHeader, { EntityTypeLabel } from './autoComplete/SectionHeader';
 import { useUserContext } from '../context/useUserContext';
 import { navigateToSearchUrl } from './utils/navigateToSearchUrl';
-import { getQuickFilterDetails } from './autoComplete/quickFilters/utils';
 import ViewAllSearchItem from './ViewAllSearchItem';
 import { ViewSelect } from '../entity/view/select/ViewSelect';
 import { combineSiblingsInAutoComplete } from './utils/combineSiblingsInAutoComplete';
@@ -39,13 +38,14 @@ const StyledSearchBar = styled(Input)`
     &&& {
         border-radius: 70px;
         height: 40px;
-        font-size: 20px;
-        color: ${ANTD_GRAY[7]};
-        background-color: ${ANTD_GRAY_V2[2]};
-    }
-    > .ant-input {
         font-size: 14px;
+        color: ${ANTD_GRAY[7]};
         background-color: ${ANTD_GRAY_V2[2]};
+        border: 2px solid transparent;
+
+        &:focus-within {
+            border: 1.5px solid ${REDESIGN_COLORS.BLUE};
+        }
     }
     > .ant-input::placeholder {
         color: ${ANTD_GRAY_V2[10]};
@@ -203,23 +203,16 @@ export const SearchBar = ({
     const { quickFilters, selectedQuickFilter, setSelectedQuickFilter } = useQuickFiltersContext();
 
     const autoCompleteQueryOptions = useMemo(() => {
-        const query = suggestions.length ? effectiveQuery : '';
-        const selectedQuickFilterLabel =
-            showQuickFilters && selectedQuickFilter
-                ? getQuickFilterDetails(selectedQuickFilter, entityRegistry).label
-                : '';
-        const text = query || selectedQuickFilterLabel;
-
-        if (!text) return [];
+        if (effectiveQuery === '') return [];
 
         return [
             {
-                value: `${EXACT_SEARCH_PREFIX}${text}`,
-                label: <ViewAllSearchItem searchTarget={text} />,
+                value: `${EXACT_SEARCH_PREFIX}${effectiveQuery}`,
+                label: <ViewAllSearchItem searchTarget={effectiveQuery} />,
                 type: EXACT_AUTOCOMPLETE_OPTION_TYPE,
             },
         ];
-    }, [showQuickFilters, suggestions.length, effectiveQuery, selectedQuickFilter, entityRegistry]);
+    }, [effectiveQuery]);
 
     const autoCompleteEntityOptions = useMemo(() => {
         return suggestions.map((suggestion: AutoCompleteResultForEntity) => {
@@ -296,6 +289,22 @@ export const SearchBar = ({
         }
     }
 
+    const searchInputRef = useRef(null);
+
+    useEffect(() => {
+        const handleKeyDown = (event) => {
+            // Support command-k to select the search bar.
+            // 75 is the keyCode for 'k'
+            if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) {
+                (searchInputRef?.current as any)?.focus();
+            }
+        };
+        document.addEventListener('keydown', handleKeyDown);
+        return () => {
+            document.removeEventListener('keydown', handleKeyDown);
+        };
+    }, []);
+
     return (
         <AutoCompleteContainer style={style} ref={searchBarWrapperRef}>
             <StyledAutoComplete
@@ -399,6 +408,7 @@ export const SearchBar = ({
                             />
                         </>
                     }
+                    ref={searchInputRef}
                 />
             </StyledAutoComplete>
         </AutoCompleteContainer>
diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
index ced7d8642576b..ce1ad93565ba4 100644
--- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
+++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
@@ -93,20 +93,6 @@ export function HeaderLinks(props: Props) {
                     </Link>
                 </LinkWrapper>
             )}
-            {showIngestion && (
-                <LinkWrapper>
-                    <Link to="/ingestion">
-                        <Button id={HOME_PAGE_INGESTION_ID} type="text">
-                            <Tooltip title="Connect DataHub to your organization's data sources">
-                                <NavTitleContainer>
-                                    <ApiOutlined />
-                                    <NavTitleText>Ingestion</NavTitleText>
-                                </NavTitleContainer>
-                            </Tooltip>
-                        </Button>
-                    </Link>
-                </LinkWrapper>
-            )}
             <Dropdown
                 trigger={['click']}
                 overlay={
@@ -145,6 +131,20 @@ export function HeaderLinks(props: Props) {
                     </Button>
                 </LinkWrapper>
             </Dropdown>
+            {showIngestion && (
+                <LinkWrapper>
+                    <Link to="/ingestion">
+                        <Button id={HOME_PAGE_INGESTION_ID} type="text">
+                            <Tooltip title="Connect DataHub to your organization's data sources">
+                                <NavTitleContainer>
+                                    <ApiOutlined />
+                                    <NavTitleText>Ingestion</NavTitleText>
+                                </NavTitleContainer>
+                            </Tooltip>
+                        </Button>
+                    </Link>
+                </LinkWrapper>
+            )}
             {showSettings && (
                 <LinkWrapper style={{ marginRight: 12 }}>
                     <Link to="/settings">
diff --git a/datahub-web-react/src/conf/theme/theme_dark.config.json b/datahub-web-react/src/conf/theme/theme_dark.config.json
index 9746c3ddde5f3..54ebebd3b692b 100644
--- a/datahub-web-react/src/conf/theme/theme_dark.config.json
+++ b/datahub-web-react/src/conf/theme/theme_dark.config.json
@@ -30,7 +30,7 @@
             "homepageMessage": "Find data you can count(*) on"
         },
         "search": {
-            "searchbarMessage": "Search Datasets, People, & more..."
+            "searchbarMessage": "Search Tables, Dashboards, People, & more..."
         },
         "menu": {
             "items": [
@@ -52,4 +52,4 @@
             ]
         }
     }
-}
+}
\ No newline at end of file
diff --git a/datahub-web-react/src/conf/theme/theme_light.config.json b/datahub-web-react/src/conf/theme/theme_light.config.json
index 906c04e38a1ba..6b9ef3eac52b0 100644
--- a/datahub-web-react/src/conf/theme/theme_light.config.json
+++ b/datahub-web-react/src/conf/theme/theme_light.config.json
@@ -33,7 +33,7 @@
             "homepageMessage": "Find data you can count on"
         },
         "search": {
-            "searchbarMessage": "Search Datasets, People, & more..."
+            "searchbarMessage": "Search Tables, Dashboards, People, & more..."
         },
         "menu": {
             "items": [
@@ -60,4 +60,4 @@
             ]
         }
     }
-}
+}
\ No newline at end of file

From 817c371fbf8f8287480a2150925e9526a28f1f6e Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Wed, 4 Oct 2023 23:11:06 -0400
Subject: [PATCH 007/259] feat: data contracts models + CLI (#8923)

Co-authored-by: Shirshanka Das <shirshanka@apache.org>
Co-authored-by: John Joyce <john@acryl.io>
---
 .../linkedin/datahub/graphql/TestUtils.java   |   3 +
 .../test/resources/test-entity-registry.yaml  |   8 +
 .../pet_of_the_week.dhub.dc.yaml              |  21 ++
 .../api/entities/datacontract/__init__.py     |   0
 .../datacontract/data_quality_assertion.py    | 107 +++++++++
 .../api/entities/datacontract/datacontract.py | 213 ++++++++++++++++++
 .../datacontract/freshness_assertion.py       |  86 +++++++
 .../entities/datacontract/schema_assertion.py |  81 +++++++
 .../datahub/cli/specific/datacontract_cli.py  |  80 +++++++
 .../src/datahub/cli/specific/file_loader.py   |  26 +--
 .../src/datahub/emitter/mce_builder.py        |  24 +-
 .../src/datahub/emitter/mcp_builder.py        |  27 +--
 metadata-ingestion/src/datahub/entrypoints.py |   2 +
 .../src/datahub/ingestion/api/closeable.py    |   8 +-
 .../ingestion/source/dbt/dbt_common.py        |  28 ++-
 .../integrations/great_expectations/action.py |  19 +-
 .../tests/unit/test_mcp_builder.py            |   3 +-
 .../linkedin/assertion/AssertionAction.pdl    |  22 ++
 .../linkedin/assertion/AssertionActions.pdl   |  18 ++
 .../com/linkedin/assertion/AssertionInfo.pdl  |  49 +++-
 .../linkedin/assertion/AssertionResult.pdl    |  18 +-
 .../assertion/AssertionResultError.pdl        |  45 ++++
 .../linkedin/assertion/AssertionRunEvent.pdl  |  57 +++--
 .../linkedin/assertion/AssertionSource.pdl    |  27 +++
 .../assertion/AssertionStdAggregation.pdl     |  10 +-
 .../assertion/AssertionValueChangeType.pdl    |  16 ++
 .../com/linkedin/assertion/AuditLogSpec.pdl   |  18 ++
 .../assertion/DatasetAssertionInfo.pdl        |  19 +-
 .../assertion/FixedIntervalSchedule.pdl       |  10 +
 .../assertion/FreshnessAssertionInfo.pdl      |  53 +++++
 .../assertion/FreshnessAssertionSchedule.pdl  |  66 ++++++
 .../assertion/FreshnessCronSchedule.pdl       |  25 ++
 .../linkedin/assertion/FreshnessFieldKind.pdl |  17 ++
 .../linkedin/assertion/FreshnessFieldSpec.pdl |  14 ++
 .../IncrementingSegmentFieldTransformer.pdl   |  60 +++++
 .../IncrementingSegmentRowCountChange.pdl     |  33 +++
 .../IncrementingSegmentRowCountTotal.pdl      |  27 +++
 .../assertion/IncrementingSegmentSpec.pdl     |  33 +++
 .../com/linkedin/assertion/RowCountChange.pdl |  27 +++
 .../com/linkedin/assertion/RowCountTotal.pdl  |  22 ++
 .../assertion/SchemaAssertionInfo.pdl         |  29 +++
 .../assertion/VolumeAssertionInfo.pdl         |  82 +++++++
 .../datacontract/DataContractProperties.pdl   |  59 +++++
 .../datacontract/DataContractStatus.pdl       |  27 +++
 .../datacontract/DataQualityContract.pdl      |  16 ++
 .../datacontract/FreshnessContract.pdl        |  13 ++
 .../linkedin/datacontract/SchemaContract.pdl  |  13 ++
 .../com/linkedin/dataset/DatasetFilter.pdl    |  30 +++
 .../linkedin/metadata/key/DataContractKey.pdl |  14 ++
 .../com/linkedin/schema/SchemaFieldSpec.pdl   |  21 ++
 .../src/main/resources/entity-registry.yml    |   9 +
 51 files changed, 1641 insertions(+), 94 deletions(-)
 create mode 100644 metadata-ingestion/examples/data_contract/pet_of_the_week.dhub.dc.yaml
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/__init__.py
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py
 create mode 100644 metadata-ingestion/src/datahub/cli/specific/datacontract_cli.py
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionAction.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionActions.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionValueChangeType.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AuditLogSpec.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/FixedIntervalSchedule.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionInfo.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessCronSchedule.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldKind.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentFieldTransformer.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountChange.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountTotal.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentSpec.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountChange.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountTotal.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractProperties.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractStatus.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/datacontract/FreshnessContract.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetFilter.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataContractKey.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/schema/SchemaFieldSpec.pdl

diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java
index 272a93fa1989c..606123cac926d 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java
@@ -8,6 +8,7 @@
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.transactions.AspectsBatchImpl;
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
@@ -21,6 +22,8 @@
 public class TestUtils {
 
   public static EntityService getMockEntityService() {
+    PathSpecBasedSchemaAnnotationVisitor.class.getClassLoader()
+        .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false);
     EntityRegistry registry = new ConfigEntityRegistry(TestUtils.class.getResourceAsStream("/test-entity-registry.yaml"));
     EntityService mockEntityService = Mockito.mock(EntityService.class);
     Mockito.when(mockEntityService.getEntityRegistry()).thenReturn(registry);
diff --git a/datahub-graphql-core/src/test/resources/test-entity-registry.yaml b/datahub-graphql-core/src/test/resources/test-entity-registry.yaml
index d694ae53ac42f..efd75a7fb07f5 100644
--- a/datahub-graphql-core/src/test/resources/test-entity-registry.yaml
+++ b/datahub-graphql-core/src/test/resources/test-entity-registry.yaml
@@ -181,6 +181,7 @@ entities:
     - assertionInfo
     - dataPlatformInstance
     - assertionRunEvent
+    - assertionActions
     - status
 - name: dataHubRetention
   category: internal
@@ -292,4 +293,11 @@ entities:
   aspects:
     - ownershipTypeInfo
     - status
+- name: dataContract
+  category: core
+  keyAspect: dataContractKey
+  aspects:
+    - dataContractProperties
+    - dataContractStatus
+    - status
 events:
diff --git a/metadata-ingestion/examples/data_contract/pet_of_the_week.dhub.dc.yaml b/metadata-ingestion/examples/data_contract/pet_of_the_week.dhub.dc.yaml
new file mode 100644
index 0000000000000..c73904403f678
--- /dev/null
+++ b/metadata-ingestion/examples/data_contract/pet_of_the_week.dhub.dc.yaml
@@ -0,0 +1,21 @@
+# id: pet_details_dc # Optional: This is the unique identifier for the data contract
+display_name: Data Contract for SampleHiveDataset
+entity: urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)
+freshness:
+  time: 0700
+  granularity: DAILY
+schema:
+  properties:
+    field_foo:
+      type: string
+      native_type: VARCHAR(100)
+    field_bar:
+      type: boolean
+  required:
+    - field_bar
+data_quality:
+  - type: column_range
+    config:
+      column: field_foo
+      min: 0
+      max: 100
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/__init__.py b/metadata-ingestion/src/datahub/api/entities/datacontract/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py
new file mode 100644
index 0000000000000..a665e95e93c43
--- /dev/null
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py
@@ -0,0 +1,107 @@
+from typing import List, Optional, Union
+
+import pydantic
+from typing_extensions import Literal
+
+import datahub.emitter.mce_builder as builder
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import (
+    AssertionInfoClass,
+    AssertionStdAggregationClass,
+    AssertionStdOperatorClass,
+    AssertionStdParameterClass,
+    AssertionStdParametersClass,
+    AssertionStdParameterTypeClass,
+    AssertionTypeClass,
+    DatasetAssertionInfoClass,
+    DatasetAssertionScopeClass,
+)
+
+
+class IdConfigMixin(ConfigModel):
+    id_raw: Optional[str] = pydantic.Field(
+        default=None,
+        alias="id",
+        description="The id of the assertion. If not provided, one will be generated using the type.",
+    )
+
+    def generate_default_id(self) -> str:
+        raise NotImplementedError
+
+
+class CustomSQLAssertion(IdConfigMixin, ConfigModel):
+    type: Literal["custom_sql"]
+
+    sql: str
+
+    def generate_dataset_assertion_info(
+        self, entity_urn: str
+    ) -> DatasetAssertionInfoClass:
+        return DatasetAssertionInfoClass(
+            dataset=entity_urn,
+            scope=DatasetAssertionScopeClass.UNKNOWN,
+            fields=[],
+            operator=AssertionStdOperatorClass._NATIVE_,
+            aggregation=AssertionStdAggregationClass._NATIVE_,
+            logic=self.sql,
+        )
+
+
+class ColumnUniqueAssertion(IdConfigMixin, ConfigModel):
+    type: Literal["unique"]
+
+    # TODO: support multiple columns?
+    column: str
+
+    def generate_default_id(self) -> str:
+        return f"{self.type}-{self.column}"
+
+    def generate_dataset_assertion_info(
+        self, entity_urn: str
+    ) -> DatasetAssertionInfoClass:
+        return DatasetAssertionInfoClass(
+            dataset=entity_urn,
+            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+            fields=[builder.make_schema_field_urn(entity_urn, self.column)],
+            operator=AssertionStdOperatorClass.EQUAL_TO,
+            aggregation=AssertionStdAggregationClass.UNIQUE_PROPOTION,  # purposely using the misspelled version to work with gql
+            parameters=AssertionStdParametersClass(
+                value=AssertionStdParameterClass(
+                    value="1", type=AssertionStdParameterTypeClass.NUMBER
+                )
+            ),
+        )
+
+
+class DataQualityAssertion(ConfigModel):
+    __root__: Union[
+        CustomSQLAssertion,
+        ColumnUniqueAssertion,
+    ] = pydantic.Field(discriminator="type")
+
+    @property
+    def id(self) -> str:
+        if self.__root__.id_raw:
+            return self.__root__.id_raw
+        try:
+            return self.__root__.generate_default_id()
+        except NotImplementedError:
+            return self.__root__.type
+
+    def generate_mcp(
+        self, assertion_urn: str, entity_urn: str
+    ) -> List[MetadataChangeProposalWrapper]:
+        dataset_assertion_info = self.__root__.generate_dataset_assertion_info(
+            entity_urn
+        )
+
+        return [
+            MetadataChangeProposalWrapper(
+                entityUrn=assertion_urn,
+                aspect=AssertionInfoClass(
+                    type=AssertionTypeClass.DATASET,
+                    datasetAssertion=dataset_assertion_info,
+                ),
+            )
+        ]
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py b/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
new file mode 100644
index 0000000000000..2df446623a9d6
--- /dev/null
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
@@ -0,0 +1,213 @@
+import collections
+from typing import Iterable, List, Optional, Tuple
+
+import pydantic
+from ruamel.yaml import YAML
+from typing_extensions import Literal
+
+import datahub.emitter.mce_builder as builder
+from datahub.api.entities.datacontract.data_quality_assertion import (
+    DataQualityAssertion,
+)
+from datahub.api.entities.datacontract.freshness_assertion import FreshnessAssertion
+from datahub.api.entities.datacontract.schema_assertion import SchemaAssertion
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mce_builder import datahub_guid, make_assertion_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import (
+    DataContractPropertiesClass,
+    DataContractStateClass,
+    DataContractStatusClass,
+    DataQualityContractClass,
+    FreshnessContractClass,
+    SchemaContractClass,
+    StatusClass,
+)
+from datahub.utilities.urns.urn import guess_entity_type
+
+
+class DataContract(ConfigModel):
+    """A yml representation of a Data Contract.
+
+    This model is used as a simpler, Python-native representation of a DataHub data contract.
+    It can be easily parsed from a YAML file, and can be easily converted into series of MCPs
+    that can be emitted to DataHub.
+    """
+
+    version: Literal[1]
+
+    id: Optional[str] = pydantic.Field(
+        default=None,
+        alias="urn",
+        description="The data contract urn. If not provided, one will be generated.",
+    )
+    entity: str = pydantic.Field(
+        description="The entity urn that the Data Contract is associated with"
+    )
+    # TODO: add support for properties
+    # properties: Optional[Dict[str, str]] = None
+
+    schema_field: Optional[SchemaAssertion] = pydantic.Field(
+        default=None, alias="schema"
+    )
+
+    freshness: Optional[FreshnessAssertion] = pydantic.Field(default=None)
+
+    # TODO: Add a validator to ensure that ids are unique
+    data_quality: Optional[List[DataQualityAssertion]] = None
+
+    _original_yaml_dict: Optional[dict] = None
+
+    @pydantic.validator("data_quality")
+    def validate_data_quality(
+        cls, data_quality: Optional[List[DataQualityAssertion]]
+    ) -> Optional[List[DataQualityAssertion]]:
+        if data_quality:
+            # Raise an error if there are duplicate ids.
+            id_counts = collections.Counter(dq_check.id for dq_check in data_quality)
+            duplicates = [id for id, count in id_counts.items() if count > 1]
+
+            if duplicates:
+                raise ValueError(
+                    f"Got multiple data quality tests with the same type or ID: {duplicates}. Set a unique ID for each data quality test."
+                )
+
+        return data_quality
+
+    @property
+    def urn(self) -> str:
+        if self.id:
+            assert guess_entity_type(self.id) == "dataContract"
+            return self.id
+
+        # Data contract urns are stable
+        guid_obj = {"entity": self.entity}
+        urn = f"urn:li:dataContract:{datahub_guid(guid_obj)}"
+        return urn
+
+    def _generate_freshness_assertion(
+        self, freshness: FreshnessAssertion
+    ) -> Tuple[str, List[MetadataChangeProposalWrapper]]:
+        guid_dict = {
+            "contract": self.urn,
+            "entity": self.entity,
+            "freshness": freshness.id,
+        }
+        assertion_urn = builder.make_assertion_urn(builder.datahub_guid(guid_dict))
+
+        return (
+            assertion_urn,
+            freshness.generate_mcp(assertion_urn, self.entity),
+        )
+
+    def _generate_schema_assertion(
+        self, schema_metadata: SchemaAssertion
+    ) -> Tuple[str, List[MetadataChangeProposalWrapper]]:
+        # ingredients for guid -> the contract id, the fact that this is a schema assertion and the entity on which the assertion is made
+        guid_dict = {
+            "contract": self.urn,
+            "entity": self.entity,
+            "schema": schema_metadata.id,
+        }
+        assertion_urn = make_assertion_urn(datahub_guid(guid_dict))
+
+        return (
+            assertion_urn,
+            schema_metadata.generate_mcp(assertion_urn, self.entity),
+        )
+
+    def _generate_data_quality_assertion(
+        self, data_quality: DataQualityAssertion
+    ) -> Tuple[str, List[MetadataChangeProposalWrapper]]:
+        guid_dict = {
+            "contract": self.urn,
+            "entity": self.entity,
+            "data_quality": data_quality.id,
+        }
+        assertion_urn = make_assertion_urn(datahub_guid(guid_dict))
+
+        return (
+            assertion_urn,
+            data_quality.generate_mcp(assertion_urn, self.entity),
+        )
+
+    def _generate_dq_assertions(
+        self, data_quality_spec: List[DataQualityAssertion]
+    ) -> Tuple[List[str], List[MetadataChangeProposalWrapper]]:
+        assertion_urns = []
+        assertion_mcps = []
+
+        for dq_check in data_quality_spec:
+            assertion_urn, assertion_mcp = self._generate_data_quality_assertion(
+                dq_check
+            )
+
+            assertion_urns.append(assertion_urn)
+            assertion_mcps.extend(assertion_mcp)
+
+        return (assertion_urns, assertion_mcps)
+
+    def generate_mcp(
+        self,
+    ) -> Iterable[MetadataChangeProposalWrapper]:
+        schema_assertion_urn = None
+        if self.schema_field is not None:
+            (
+                schema_assertion_urn,
+                schema_assertion_mcps,
+            ) = self._generate_schema_assertion(self.schema_field)
+            yield from schema_assertion_mcps
+
+        freshness_assertion_urn = None
+        if self.freshness:
+            (
+                freshness_assertion_urn,
+                sla_assertion_mcps,
+            ) = self._generate_freshness_assertion(self.freshness)
+            yield from sla_assertion_mcps
+
+        dq_assertions, dq_assertion_mcps = self._generate_dq_assertions(
+            self.data_quality or []
+        )
+        yield from dq_assertion_mcps
+
+        # Now that we've generated the assertions, we can generate
+        # the actual data contract.
+        yield from MetadataChangeProposalWrapper.construct_many(
+            entityUrn=self.urn,
+            aspects=[
+                DataContractPropertiesClass(
+                    entity=self.entity,
+                    schema=[SchemaContractClass(assertion=schema_assertion_urn)]
+                    if schema_assertion_urn
+                    else None,
+                    freshness=[
+                        FreshnessContractClass(assertion=freshness_assertion_urn)
+                    ]
+                    if freshness_assertion_urn
+                    else None,
+                    dataQuality=[
+                        DataQualityContractClass(assertion=dq_assertion_urn)
+                        for dq_assertion_urn in dq_assertions
+                    ],
+                ),
+                # Also emit status.
+                StatusClass(removed=False),
+                # Emit the contract state as PENDING.
+                DataContractStatusClass(state=DataContractStateClass.PENDING)
+                if True
+                else None,
+            ],
+        )
+
+    @classmethod
+    def from_yaml(
+        cls,
+        file: str,
+    ) -> "DataContract":
+        with open(file) as fp:
+            yaml = YAML(typ="rt")  # default, if not specfied, is 'rt' (round-trip)
+            orig_dictionary = yaml.load(fp)
+            parsed_data_contract = DataContract.parse_obj(orig_dictionary)
+            parsed_data_contract._original_yaml_dict = orig_dictionary
+            return parsed_data_contract
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py
new file mode 100644
index 0000000000000..ee8fa1181e614
--- /dev/null
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py
@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import List, Union
+
+import pydantic
+from typing_extensions import Literal
+
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import (
+    AssertionInfoClass,
+    AssertionTypeClass,
+    CalendarIntervalClass,
+    FixedIntervalScheduleClass,
+    FreshnessAssertionInfoClass,
+    FreshnessAssertionScheduleClass,
+    FreshnessAssertionScheduleTypeClass,
+    FreshnessAssertionTypeClass,
+    FreshnessCronScheduleClass,
+)
+
+
+class CronFreshnessAssertion(ConfigModel):
+    type: Literal["cron"]
+
+    cron: str = pydantic.Field(
+        description="The cron expression to use. See https://crontab.guru/ for help."
+    )
+    timezone: str = pydantic.Field(
+        "UTC",
+        description="The timezone to use for the cron schedule. Defaults to UTC.",
+    )
+
+
+class FixedIntervalFreshnessAssertion(ConfigModel):
+    type: Literal["interval"]
+
+    interval: timedelta
+
+
+class FreshnessAssertion(ConfigModel):
+    __root__: Union[
+        CronFreshnessAssertion, FixedIntervalFreshnessAssertion
+    ] = pydantic.Field(discriminator="type")
+
+    @property
+    def id(self):
+        return self.__root__.type
+
+    def generate_mcp(
+        self, assertion_urn: str, entity_urn: str
+    ) -> List[MetadataChangeProposalWrapper]:
+        freshness = self.__root__
+
+        if isinstance(freshness, CronFreshnessAssertion):
+            schedule = FreshnessAssertionScheduleClass(
+                type=FreshnessAssertionScheduleTypeClass.CRON,
+                cron=FreshnessCronScheduleClass(
+                    cron=freshness.cron,
+                    timezone=freshness.timezone,
+                ),
+            )
+        elif isinstance(freshness, FixedIntervalFreshnessAssertion):
+            schedule = FreshnessAssertionScheduleClass(
+                type=FreshnessAssertionScheduleTypeClass.FIXED_INTERVAL,
+                fixedInterval=FixedIntervalScheduleClass(
+                    unit=CalendarIntervalClass.SECOND,
+                    multiple=int(freshness.interval.total_seconds()),
+                ),
+            )
+        else:
+            raise ValueError(f"Unknown freshness type {freshness}")
+
+        assertionInfo = AssertionInfoClass(
+            type=AssertionTypeClass.FRESHNESS,
+            freshnessAssertion=FreshnessAssertionInfoClass(
+                entity=entity_urn,
+                type=FreshnessAssertionTypeClass.DATASET_CHANGE,
+                schedule=schedule,
+            ),
+        )
+
+        return [
+            MetadataChangeProposalWrapper(entityUrn=assertion_urn, aspect=assertionInfo)
+        ]
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py
new file mode 100644
index 0000000000000..b5b592e01f58f
--- /dev/null
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py
@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import json
+from typing import List, Union
+
+import pydantic
+from typing_extensions import Literal
+
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.extractor.json_schema_util import get_schema_metadata
+from datahub.metadata.schema_classes import (
+    AssertionInfoClass,
+    AssertionTypeClass,
+    SchemaAssertionInfoClass,
+    SchemaFieldClass,
+    SchemalessClass,
+    SchemaMetadataClass,
+)
+
+
+class JsonSchemaContract(ConfigModel):
+    type: Literal["json-schema"]
+
+    json_schema: dict = pydantic.Field(alias="json-schema")
+
+    _schema_metadata: SchemaMetadataClass
+
+    def _init_private_attributes(self) -> None:
+        super()._init_private_attributes()
+        self._schema_metadata = get_schema_metadata(
+            platform="urn:li:dataPlatform:datahub",
+            name="",
+            json_schema=self.json_schema,
+            raw_schema_string=json.dumps(self.json_schema),
+        )
+
+
+class FieldListSchemaContract(ConfigModel, arbitrary_types_allowed=True):
+    type: Literal["field-list"]
+
+    fields: List[SchemaFieldClass]
+
+    _schema_metadata: SchemaMetadataClass
+
+    def _init_private_attributes(self) -> None:
+        super()._init_private_attributes()
+        self._schema_metadata = SchemaMetadataClass(
+            schemaName="",
+            platform="urn:li:dataPlatform:datahub",
+            version=0,
+            hash="",
+            platformSchema=SchemalessClass(),
+            fields=self.fields,
+        )
+
+
+class SchemaAssertion(ConfigModel):
+    __root__: Union[JsonSchemaContract, FieldListSchemaContract] = pydantic.Field(
+        discriminator="type"
+    )
+
+    @property
+    def id(self):
+        return self.__root__.type
+
+    def generate_mcp(
+        self, assertion_urn: str, entity_urn: str
+    ) -> List[MetadataChangeProposalWrapper]:
+        schema_metadata = self.__root__._schema_metadata
+
+        assertionInfo = AssertionInfoClass(
+            type=AssertionTypeClass.DATA_SCHEMA,
+            schemaAssertion=SchemaAssertionInfoClass(
+                entity=entity_urn, schema=schema_metadata
+            ),
+        )
+
+        return [
+            MetadataChangeProposalWrapper(entityUrn=assertion_urn, aspect=assertionInfo)
+        ]
diff --git a/metadata-ingestion/src/datahub/cli/specific/datacontract_cli.py b/metadata-ingestion/src/datahub/cli/specific/datacontract_cli.py
new file mode 100644
index 0000000000000..3745943c8c96a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/cli/specific/datacontract_cli.py
@@ -0,0 +1,80 @@
+import logging
+from typing import Optional
+
+import click
+from click_default_group import DefaultGroup
+
+from datahub.api.entities.datacontract.datacontract import DataContract
+from datahub.ingestion.graph.client import get_default_graph
+from datahub.telemetry import telemetry
+from datahub.upgrade import upgrade
+
+logger = logging.getLogger(__name__)
+
+
+@click.group(cls=DefaultGroup, default="upsert")
+def datacontract() -> None:
+    """A group of commands to interact with the DataContract entity in DataHub."""
+    pass
+
+
+@datacontract.command()
+@click.option("-f", "--file", required=True, type=click.Path(exists=True))
+@upgrade.check_upgrade
+@telemetry.with_telemetry()
+def upsert(file: str) -> None:
+    """Upsert (create or update) a Data Contract in DataHub."""
+
+    data_contract: DataContract = DataContract.from_yaml(file)
+    urn = data_contract.urn
+
+    with get_default_graph() as graph:
+        if not graph.exists(data_contract.entity):
+            raise ValueError(
+                f"Cannot define a data contract for non-existent entity {data_contract.entity}"
+            )
+
+        try:
+            for mcp in data_contract.generate_mcp():
+                graph.emit(mcp)
+            click.secho(f"Update succeeded for urn {urn}.", fg="green")
+        except Exception as e:
+            logger.exception(e)
+            click.secho(
+                f"Update failed for {urn}: {e}",
+                fg="red",
+            )
+
+
+@datacontract.command()
+@click.option(
+    "--urn", required=False, type=str, help="The urn for the data contract to delete"
+)
+@click.option(
+    "-f",
+    "--file",
+    required=False,
+    type=click.Path(exists=True),
+    help="The file containing the data contract definition",
+)
+@click.option("--hard/--soft", required=False, is_flag=True, default=False)
+@upgrade.check_upgrade
+@telemetry.with_telemetry()
+def delete(urn: Optional[str], file: Optional[str], hard: bool) -> None:
+    """Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
+
+    if not urn:
+        if not file:
+            raise click.UsageError(
+                "Must provide either an urn or a file to delete a data contract"
+            )
+
+        data_contract = DataContract.from_yaml(file)
+        urn = data_contract.urn
+
+    with get_default_graph() as graph:
+        if not graph.exists(urn):
+            raise ValueError(f"Data Contract {urn} does not exist")
+
+        graph.delete_entity(urn, hard=hard)
+        click.secho(f"Data Contract {urn} deleted")
diff --git a/metadata-ingestion/src/datahub/cli/specific/file_loader.py b/metadata-ingestion/src/datahub/cli/specific/file_loader.py
index 54f12e024d294..a9787343fdb91 100644
--- a/metadata-ingestion/src/datahub/cli/specific/file_loader.py
+++ b/metadata-ingestion/src/datahub/cli/specific/file_loader.py
@@ -1,9 +1,7 @@
-import io
 from pathlib import Path
 from typing import Union
 
-from datahub.configuration.common import ConfigurationError
-from datahub.configuration.yaml import YamlConfigurationMechanism
+from datahub.configuration.config_loader import load_config_file
 
 
 def load_file(config_file: Path) -> Union[dict, list]:
@@ -17,19 +15,11 @@ def load_file(config_file: Path) -> Union[dict, list]:
     evolve to becoming a standard function that all the specific. cli variants will use
     to load up the models from external files
     """
-    if not isinstance(config_file, Path):
-        config_file = Path(config_file)
-    if not config_file.is_file():
-        raise ConfigurationError(f"Cannot open config file {config_file}")
 
-    if config_file.suffix in {".yaml", ".yml"}:
-        config_mech: YamlConfigurationMechanism = YamlConfigurationMechanism()
-    else:
-        raise ConfigurationError(
-            f"Only .yaml and .yml are supported. Cannot process file type {config_file.suffix}"
-        )
-
-    raw_config_file = config_file.read_text()
-    config_fp = io.StringIO(raw_config_file)
-    raw_config = config_mech.load_config(config_fp)
-    return raw_config
+    res = load_config_file(
+        config_file,
+        squirrel_original_config=False,
+        resolve_env_vars=False,
+        allow_stdin=False,
+    )
+    return res
diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py
index 0928818c7005c..64c9ec1bb5704 100644
--- a/metadata-ingestion/src/datahub/emitter/mce_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py
@@ -1,11 +1,11 @@
 """Convenience functions for creating MCEs"""
+import hashlib
 import json
 import logging
 import os
 import re
 import time
 from enum import Enum
-from hashlib import md5
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -21,7 +21,6 @@
 import typing_inspect
 
 from datahub.configuration.source_common import DEFAULT_ENV as DEFAULT_ENV_CONFIGURATION
-from datahub.emitter.serialization_helper import pre_json_transform
 from datahub.metadata.schema_classes import (
     AssertionKeyClass,
     AuditStampClass,
@@ -159,11 +158,24 @@ def container_urn_to_key(guid: str) -> Optional[ContainerKeyClass]:
     return None
 
 
+class _DatahubKeyJSONEncoder(json.JSONEncoder):
+    # overload method default
+    def default(self, obj: Any) -> Any:
+        if hasattr(obj, "guid"):
+            return obj.guid()
+        # Call the default method for other types
+        return json.JSONEncoder.default(self, obj)
+
+
 def datahub_guid(obj: dict) -> str:
-    obj_str = json.dumps(
-        pre_json_transform(obj), separators=(",", ":"), sort_keys=True
-    ).encode("utf-8")
-    return md5(obj_str).hexdigest()
+    json_key = json.dumps(
+        obj,
+        separators=(",", ":"),
+        sort_keys=True,
+        cls=_DatahubKeyJSONEncoder,
+    )
+    md5_hash = hashlib.md5(json_key.encode("utf-8"))
+    return str(md5_hash.hexdigest())
 
 
 def make_assertion_urn(assertion_id: str) -> str:
diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
index 7419577b367aa..06f689dfd317b 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -1,11 +1,10 @@
-import hashlib
-import json
-from typing import Any, Dict, Iterable, List, Optional, TypeVar
+from typing import Dict, Iterable, List, Optional, TypeVar
 
 from pydantic.fields import Field
 from pydantic.main import BaseModel
 
 from datahub.emitter.mce_builder import (
+    datahub_guid,
     make_container_urn,
     make_data_platform_urn,
     make_dataplatform_instance_urn,
@@ -33,24 +32,13 @@
 )
 
 
-def _stable_guid_from_dict(d: dict) -> str:
-    json_key = json.dumps(
-        d,
-        separators=(",", ":"),
-        sort_keys=True,
-        cls=DatahubKeyJSONEncoder,
-    )
-    md5_hash = hashlib.md5(json_key.encode("utf-8"))
-    return str(md5_hash.hexdigest())
-
-
 class DatahubKey(BaseModel):
     def guid_dict(self) -> Dict[str, str]:
         return self.dict(by_alias=True, exclude_none=True)
 
     def guid(self) -> str:
         bag = self.guid_dict()
-        return _stable_guid_from_dict(bag)
+        return datahub_guid(bag)
 
 
 class ContainerKey(DatahubKey):
@@ -137,15 +125,6 @@ def as_urn(self) -> str:
         )
 
 
-class DatahubKeyJSONEncoder(json.JSONEncoder):
-    # overload method default
-    def default(self, obj: Any) -> Any:
-        if hasattr(obj, "guid"):
-            return obj.guid()
-        # Call the default method for other types
-        return json.JSONEncoder.default(self, obj)
-
-
 KeyType = TypeVar("KeyType", bound=ContainerKey)
 
 
diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py
index 84615fd9a6148..5bfab3b841fa3 100644
--- a/metadata-ingestion/src/datahub/entrypoints.py
+++ b/metadata-ingestion/src/datahub/entrypoints.py
@@ -21,6 +21,7 @@
 from datahub.cli.ingest_cli import ingest
 from datahub.cli.migrate import migrate
 from datahub.cli.put_cli import put
+from datahub.cli.specific.datacontract_cli import datacontract
 from datahub.cli.specific.dataproduct_cli import dataproduct
 from datahub.cli.specific.group_cli import group
 from datahub.cli.specific.user_cli import user
@@ -158,6 +159,7 @@ def init() -> None:
 datahub.add_command(user)
 datahub.add_command(group)
 datahub.add_command(dataproduct)
+datahub.add_command(datacontract)
 
 try:
     from datahub.cli.lite_cli import lite
diff --git a/metadata-ingestion/src/datahub/ingestion/api/closeable.py b/metadata-ingestion/src/datahub/ingestion/api/closeable.py
index 523174b9978b3..80a5008ed6368 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/closeable.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/closeable.py
@@ -1,7 +1,9 @@
 from abc import abstractmethod
 from contextlib import AbstractContextManager
 from types import TracebackType
-from typing import Optional, Type
+from typing import Optional, Type, TypeVar
+
+_Self = TypeVar("_Self", bound="Closeable")
 
 
 class Closeable(AbstractContextManager):
@@ -9,6 +11,10 @@ class Closeable(AbstractContextManager):
     def close(self) -> None:
         pass
 
+    def __enter__(self: _Self) -> _Self:
+        # This method is mainly required for type checking.
+        return self
+
     def __exit__(
         self,
         exc_type: Optional[Type[BaseException]],
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 3edeb695e9f21..f9b71892975b4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -701,18 +701,22 @@ def create_test_entity_mcps(
             assertion_urn = mce_builder.make_assertion_urn(
                 mce_builder.datahub_guid(
                     {
-                        "platform": DBT_PLATFORM,
-                        "name": node.dbt_name,
-                        "instance": self.config.platform_instance,
-                        **(
-                            # Ideally we'd include the env unconditionally. However, we started out
-                            # not including env in the guid, so we need to maintain backwards compatibility
-                            # with existing PROD assertions.
-                            {"env": self.config.env}
-                            if self.config.env != mce_builder.DEFAULT_ENV
-                            and self.config.include_env_in_assertion_guid
-                            else {}
-                        ),
+                        k: v
+                        for k, v in {
+                            "platform": DBT_PLATFORM,
+                            "name": node.dbt_name,
+                            "instance": self.config.platform_instance,
+                            **(
+                                # Ideally we'd include the env unconditionally. However, we started out
+                                # not including env in the guid, so we need to maintain backwards compatibility
+                                # with existing PROD assertions.
+                                {"env": self.config.env}
+                                if self.config.env != mce_builder.DEFAULT_ENV
+                                and self.config.include_env_in_assertion_guid
+                                else {}
+                            ),
+                        }.items()
+                        if v is not None
                     }
                 )
             )
diff --git a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py
index f116550328819..8b393a8f6f1c6 100644
--- a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py
+++ b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py
@@ -35,6 +35,7 @@
 from datahub.cli.cli_utils import get_boolean_env_variable
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
+from datahub.emitter.serialization_helper import pre_json_transform
 from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
     get_platform_from_sqlalchemy_uri,
 )
@@ -253,13 +254,15 @@ def get_assertions_with_results(
             # possibly for each validation run
             assertionUrn = builder.make_assertion_urn(
                 builder.datahub_guid(
-                    {
-                        "platform": GE_PLATFORM_NAME,
-                        "nativeType": expectation_type,
-                        "nativeParameters": kwargs,
-                        "dataset": assertion_datasets[0],
-                        "fields": assertion_fields,
-                    }
+                    pre_json_transform(
+                        {
+                            "platform": GE_PLATFORM_NAME,
+                            "nativeType": expectation_type,
+                            "nativeParameters": kwargs,
+                            "dataset": assertion_datasets[0],
+                            "fields": assertion_fields,
+                        }
+                    )
                 )
             )
             logger.debug(
@@ -638,7 +641,7 @@ def get_dataset_partitions(self, batch_identifier, data_asset):
                 ].batch_request.runtime_parameters["query"]
                 partitionSpec = PartitionSpecClass(
                     type=PartitionTypeClass.QUERY,
-                    partition=f"Query_{builder.datahub_guid(query)}",
+                    partition=f"Query_{builder.datahub_guid(pre_json_transform(query))}",
                 )
 
                 batchSpec = BatchSpec(
diff --git a/metadata-ingestion/tests/unit/test_mcp_builder.py b/metadata-ingestion/tests/unit/test_mcp_builder.py
index 23f2bddc2084e..561b782ef9e46 100644
--- a/metadata-ingestion/tests/unit/test_mcp_builder.py
+++ b/metadata-ingestion/tests/unit/test_mcp_builder.py
@@ -1,5 +1,4 @@
 import datahub.emitter.mcp_builder as builder
-from datahub.emitter.mce_builder import datahub_guid
 
 
 def test_guid_generator():
@@ -80,7 +79,7 @@ def test_guid_generators():
     key = builder.SchemaKey(
         database="test", schema="Test", platform="mysql", instance="TestInstance"
     )
-    guid_datahub = datahub_guid(key.dict(by_alias=True))
+    guid_datahub = key.guid()
 
     guid = key.guid()
     assert guid == guid_datahub
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionAction.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionAction.pdl
new file mode 100644
index 0000000000000..df6620b66bfd8
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionAction.pdl
@@ -0,0 +1,22 @@
+namespace com.linkedin.assertion
+
+/**
+ * The Actions about an Assertion.
+ * In the future, we'll likely extend this model to support additional
+ * parameters or options related to the assertion actions.
+ */
+record AssertionAction {
+    /**
+     * The type of the Action
+     */
+    type: enum AssertionActionType {
+      /**
+      * Raise an incident.
+      */
+      RAISE_INCIDENT
+      /**
+      * Resolve open incidents related to the assertion.
+      */
+      RESOLVE_INCIDENT
+    }
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionActions.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionActions.pdl
new file mode 100644
index 0000000000000..61846c1ba9c12
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionActions.pdl
@@ -0,0 +1,18 @@
+namespace com.linkedin.assertion
+
+/**
+ * The Actions about an Assertion
+ */
+@Aspect = {
+  "name": "assertionActions"
+}
+record AssertionActions {
+    /**
+    * Actions to be executed on successful assertion run.
+    */
+    onSuccess: array[AssertionAction] = []
+    /**
+    * Actions to be executed on failed assertion run.
+    */
+    onFailure: array[AssertionAction] = []
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl
index 77ee147a781e2..ae2a58028057b 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl
@@ -13,13 +13,58 @@ record AssertionInfo includes CustomProperties, ExternalReference {
     /**
     * Type of assertion. Assertion types can evolve to span Datasets, Flows (Pipelines), Models, Features etc.
     */
+    @Searchable = { }
     type: enum AssertionType {
-      // A single-dataset assertion. When this is the value, the datasetAssertion field will be populated.
+      /**
+       * A single-dataset assertion. When this is the value, the datasetAssertion field will be populated.
+       */
       DATASET
+
+      /**
+       * A freshness assertion, or an assertion which indicates when a particular operation should occur
+       * to an asset.
+       */
+      FRESHNESS
+
+      /**
+       * A volume assertion, or an assertion which indicates how much data should be available for a
+       * particular asset.
+       */
+      VOLUME
+
+      /**
+       * A schema or structural assertion.
+       *
+       * Would have named this SCHEMA but the codegen for PDL does not allow this (reserved word).
+       */
+      DATA_SCHEMA
     }
 
     /**
-    * Dataset Assertion information when type is DATASET
+    * A Dataset Assertion definition. This field is populated when the type is DATASET.
     */
     datasetAssertion: optional DatasetAssertionInfo
+
+    /**
+    * An Freshness Assertion definition. This field is populated when the type is FRESHNESS.
+    */
+    freshnessAssertion: optional FreshnessAssertionInfo
+
+    /**
+    * An Volume Assertion definition. This field is populated when the type is VOLUME.
+    */
+    volumeAssertion: optional VolumeAssertionInfo
+
+    /**
+    * An schema Assertion definition. This field is populated when the type is DATASET_SCHEMA
+    */
+    schemaAssertion: optional SchemaAssertionInfo
+
+    /**
+    * The source or origin of the Assertion definition.
+    *
+    * If the source type of the Assertion is EXTERNAL, it is expected to have a corresponding dataPlatformInstance aspect detailing
+    * the platform where it was ingested from.
+    */
+    source: optional AssertionSource
 }
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl
index decbfc08263de..ded84e1969153 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl
@@ -5,10 +5,15 @@ namespace com.linkedin.assertion
  */
 record AssertionResult {
   /**
-  *  The final result, e.g. either SUCCESS or FAILURE.
+  *  The final result, e.g. either SUCCESS, FAILURE, or ERROR.
   */
   @TimeseriesField = {}
+  @Searchable = {}
   type: enum AssertionResultType {
+    /**
+    *  The Assertion has not yet been fully evaluated
+    */
+    INIT
     /**
     *  The Assertion Succeeded
     */
@@ -17,6 +22,10 @@ record AssertionResult {
     *  The Assertion Failed
     */
     FAILURE
+    /**
+    *  The Assertion encountered an Error
+    */
+    ERROR
   }
 
   /**
@@ -45,8 +54,13 @@ record AssertionResult {
   nativeResults: optional map[string, string]
 
   /**
-   * URL where full results are available
+   * External URL where full results are available. Only present when assertion source is not native.
    */
   externalUrl: optional string
 
+  /**
+   *  The error object if AssertionResultType is an Error
+   */
+  error: optional AssertionResultError
+
 }
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl
new file mode 100644
index 0000000000000..e768fe8521942
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl
@@ -0,0 +1,45 @@
+namespace com.linkedin.assertion
+
+/**
+ *  An error encountered when evaluating an AssertionResult
+ */
+record AssertionResultError {
+  /**
+  *  The type of error encountered
+  */
+  type: enum AssertionResultErrorType {
+    /**
+    *  Source is unreachable
+    */
+    SOURCE_CONNECTION_ERROR
+    /**
+    *  Source query failed to execute
+    */
+    SOURCE_QUERY_FAILED
+    /**
+    *  Insufficient data to evaluate the assertion
+    */
+    INSUFFICIENT_DATA
+    /**
+    *  Invalid parameters were detected
+    */
+    INVALID_PARAMETERS
+    /**
+    *  Event type not supported by the specified source
+    */
+    INVALID_SOURCE_TYPE
+    /**
+    *  Unsupported platform
+    */
+    UNSUPPORTED_PLATFORM
+    /**
+    *  Unknown error
+    */
+    UNKNOWN_ERROR
+  }
+
+  /**
+  *  Additional metadata depending on the type of error
+  */
+  properties: optional map[string, string]
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl
index 9e75f96fafd06..14f1204232740 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl
@@ -1,6 +1,7 @@
 namespace com.linkedin.assertion
 
-import com.linkedin.timeseries.TimeseriesAspectBase
+import com.linkedin.timeseries.PartitionSpec
+import com.linkedin.timeseries.TimeWindowSize
 import com.linkedin.common.ExternalReference
 import com.linkedin.common.Urn
 
@@ -12,36 +13,31 @@ import com.linkedin.common.Urn
   "name": "assertionRunEvent",
   "type": "timeseries",
 }
-record AssertionRunEvent includes TimeseriesAspectBase {
+record AssertionRunEvent {
+
+  /**
+   * The event timestamp field as epoch at UTC in milli seconds.
+   */
+  @Searchable = {
+    "fieldName": "lastCompletedTime",
+    "fieldType": "DATETIME"
+  }
+  timestampMillis: long
 
   /**
   *  Native (platform-specific) identifier for this run
   */
-  //Multiple assertions could occur in same evaluator run
   runId: string
 
-  /*
-  * Urn of assertion which is evaluated
-  */
-  @TimeseriesField = {}
-  assertionUrn: Urn
-
   /*
   * Urn of entity on which the assertion is applicable
   */
-  //example - dataset urn, if dataset is being asserted
   @TimeseriesField = {}
   asserteeUrn: Urn
-  
-  /**
-  * Specification of the batch which this run is evaluating
-  */
-  batchSpec: optional BatchSpec
 
   /**
   * The status of the assertion run as per this timeseries event.
   */
-  // Currently just supports COMPLETE, but should evolve to support other statuses like STARTED, RUNNING, etc.
   @TimeseriesField = {}
   status: enum AssertionRunStatus {
     /**
@@ -59,4 +55,33 @@ record AssertionRunEvent includes TimeseriesAspectBase {
    * Runtime parameters of evaluation
    */
   runtimeContext: optional map[string, string]
+
+  /**
+  * Specification of the batch which this run is evaluating
+  */
+  batchSpec: optional BatchSpec
+
+  /*
+  * Urn of assertion which is evaluated
+  */
+  @TimeseriesField = {}
+  assertionUrn: Urn
+
+  /**
+   * Granularity of the event if applicable
+   */
+  eventGranularity: optional TimeWindowSize
+
+  /**
+   * The optional partition specification.
+   */
+  partitionSpec: optional PartitionSpec = {
+    "type":"FULL_TABLE",
+    "partition":"FULL_TABLE_SNAPSHOT"
+  }
+
+  /**
+   * The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value.
+   */
+  messageId: optional string
 }
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl
new file mode 100644
index 0000000000000..d8892c0c71c6f
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl
@@ -0,0 +1,27 @@
+namespace com.linkedin.assertion
+
+/**
+ * The source of an assertion
+ */
+record AssertionSource {
+    /**
+     * The type of the Assertion Source
+     */
+    @Searchable = {
+       "fieldName": "sourceType"
+    }
+    type:  enum AssertionSourceType {
+      /**
+       * The assertion was defined natively on DataHub by a user.
+       */
+      NATIVE
+      /**
+       * The assertion was defined and managed externally of DataHub.
+       */
+      EXTERNAL
+      /**
+       * The assertion was inferred, e.g. from offline AI / ML models.
+       */
+      INFERRED
+    }
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdAggregation.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdAggregation.pdl
index b79b96f9379b0..968944165a1c8 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdAggregation.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdAggregation.pdl
@@ -4,6 +4,7 @@ namespace com.linkedin.assertion
  * The function that is applied to the aggregation input (schema, rows, column values) before evaluating an operator.
  */
 enum AssertionStdAggregation {
+
   /**
   * Assertion is applied on number of rows.
   */
@@ -20,7 +21,7 @@ enum AssertionStdAggregation {
   COLUMN_COUNT
 
   /**
-  * Assertion is applied on individual column value.
+  * Assertion is applied on individual column value. (No aggregation)
   */
   IDENTITY
 
@@ -42,6 +43,13 @@ enum AssertionStdAggregation {
   /**
   * Assertion is applied on proportion of distinct values in column
   */
+  UNIQUE_PROPORTION
+
+  /**
+  * Assertion is applied on proportion of distinct values in column
+  *
+  * Deprecated! Use UNIQUE_PROPORTION instead.
+  */
   UNIQUE_PROPOTION
 
   /**
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionValueChangeType.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionValueChangeType.pdl
new file mode 100644
index 0000000000000..5a1ff4fa73ffb
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionValueChangeType.pdl
@@ -0,0 +1,16 @@
+namespace com.linkedin.assertion
+
+/**
+* An enum to represent a type of change in an assertion value, metric, or measurement.
+*/
+enum AssertionValueChangeType {
+  /**
+   * A change that is defined in absolute terms.
+   */
+   ABSOLUTE
+   /**
+   * A change that is defined in relative terms using percentage change
+   * from the original value.
+   */
+   PERCENTAGE
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AuditLogSpec.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AuditLogSpec.pdl
new file mode 100644
index 0000000000000..4d5bf261cbf89
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AuditLogSpec.pdl
@@ -0,0 +1,18 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.schema.SchemaFieldDataType
+
+/**
+* Information about the Audit Log operation to use in evaluating an assertion.
+**/
+record AuditLogSpec {
+  /**
+  * The list of operation types that should be monitored. If not provided, a default set will be used.
+  */
+  operationTypes: optional array [string]
+
+  /**
+  * Optional: The user name associated with the operation.
+  */
+  userName: optional string
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/DatasetAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/DatasetAssertionInfo.pdl
index c411c7ff8a572..2a8bf28f1ff11 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/DatasetAssertionInfo.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/DatasetAssertionInfo.pdl
@@ -18,9 +18,10 @@ record DatasetAssertionInfo {
     /**
     * Scope of the Assertion. What part of the dataset does this assertion apply to?
     **/
+    @Searchable = {}
     scope: enum DatasetAssertionScope {
       /**
-      * This assertion applies to dataset columns
+      * This assertion applies to dataset column(s)
       */
       DATASET_COLUMN
 
@@ -29,6 +30,11 @@ record DatasetAssertionInfo {
       */
       DATASET_ROWS
 
+      /**
+      * This assertion applies to the storage size of the dataset
+      */
+      DATASET_STORAGE_SIZE
+
       /**
       * This assertion applies to the schema of the dataset
       */
@@ -41,7 +47,9 @@ record DatasetAssertionInfo {
     }
 
     /**
-    * One or more dataset schema fields that are targeted by this assertion
+    * One or more dataset schema fields that are targeted by this assertion.
+    *
+    * This field is expected to be provided if the assertion scope is DATASET_COLUMN.
     */
     @Relationship = {
       "/*": {
@@ -49,11 +57,18 @@ record DatasetAssertionInfo {
         "entityTypes": [ "schemaField" ]
       }
     }
+    @Searchable = {
+      "/*": {
+        "fieldType": "URN"
+      }
+    }
     fields: optional array[Urn]
 
     /**
     * Standardized assertion operator
+    * This field is left blank if there is no selected aggregation or metric for a particular column.
     */
+    @Searchable = {}
     aggregation: optional AssertionStdAggregation
 
     /**
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FixedIntervalSchedule.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FixedIntervalSchedule.pdl
new file mode 100644
index 0000000000000..c08c33ffb92d3
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FixedIntervalSchedule.pdl
@@ -0,0 +1,10 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.common.Urn
+import com.linkedin.timeseries.TimeWindowSize
+
+/**
+* Attributes defining a relative fixed interval SLA schedule.
+*/
+record FixedIntervalSchedule includes TimeWindowSize {
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionInfo.pdl
new file mode 100644
index 0000000000000..4445a11ff40a7
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionInfo.pdl
@@ -0,0 +1,53 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.common.Urn
+import com.linkedin.dataset.DatasetFilter
+
+/**
+* Attributes defining a Freshness Assertion.
+**/
+record FreshnessAssertionInfo {
+    /**
+     * The type of the freshness assertion being monitored.
+     */
+    @Searchable = {}
+    type: enum FreshnessAssertionType {
+      /**
+       * An Freshness based on Operations performed on a particular Dataset (insert, update, delete, etc) and sourced from an audit log, as
+       * opposed to based on the highest watermark in a timestamp column (e.g. a query). Only valid when entity is of type "dataset".
+       */
+       DATASET_CHANGE
+       /**
+       * An Freshness based on a successful execution of a Data Job.
+       */
+       DATA_JOB_RUN
+    }
+
+    /**
+    * The entity targeted by this Freshness check.
+    */
+    @Searchable = {
+      "fieldType": "URN"
+    }
+    @Relationship = {
+      "name": "Asserts",
+      "entityTypes": [ "dataset", "dataJob" ]
+    }
+    entity: Urn
+
+    /**
+    * Produce FAILURE Assertion Result if the asset is not updated on the cadence and within the time range described by the schedule.
+    */
+    @Searchable = {
+      "/type": {
+        "fieldName": "scheduleType"
+      }
+    }
+    schedule: FreshnessAssertionSchedule
+
+    /**
+     * A definition of the specific filters that should be applied, when performing monitoring.
+     * If not provided, there is no filter, and the full table is under consideration.
+     */
+    filter: optional DatasetFilter
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl
new file mode 100644
index 0000000000000..a87342ad4f5ed
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl
@@ -0,0 +1,66 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.common.Urn
+
+/**
+* Attributes defining a single Freshness schedule.
+*/
+record FreshnessAssertionSchedule {
+
+  /**
+   * The type of a Freshness Assertion Schedule.
+   *
+   * Once we support data-time-relative schedules (e.g. schedules relative to time partitions),
+   * we will add those schedule types here.
+   */
+  type: enum FreshnessAssertionScheduleType {
+    /**
+     * An highly configurable recurring schedule which describes the times of events described
+     * by a CRON schedule, with the evaluation schedule assuming to be matching the cron schedule.
+     *
+     * In a CRON schedule type, we compute the look-back window to be the time between the last scheduled event
+     * and the current event (evaluation time). This means that the evaluation schedule must match exactly
+     * the schedule defined inside the cron schedule.
+     *
+     * For example, a CRON schedule defined as "0 8 * * *" would represent a schedule of "every day by 8am". Assuming
+     * that the assertion evaluation schedule is defined to match this, the freshness assertion would be evaluated in the following way:
+     *
+     *     1. Compute the "last scheduled occurrence" of the event using the CRON schedule. For example, yesterday at 8am.
+     *     2. Compute the bounds of a time window between the "last scheduled occurrence" (yesterday at 8am) until the "current occurrence" (today at 8am)
+     *     3. Verify that the target event has occurred within the CRON-interval window.
+     *     4. If the target event has occurred within the time window, then assertion passes.
+     *     5. If the target event has not occurred within the time window, then the assertion fails.
+     *
+     */
+    CRON
+    /**
+     * A fixed interval which is used to compute a look-back window for use when evaluating the assertion relative
+     * to the Evaluation Time of the Assertion.
+     *
+     * To compute the valid look-back window, we subtract the fixed interval from the evaluation time. Then, we verify
+     * that the target event has occurred within that window.
+     *
+     * For example, a fixed interval of "24h" would represent a schedule of "in the last 24 hours".
+     * The 24 hour interval is relative to the evaluation time of the assertion. For example if we schedule the assertion
+     * to be evaluated each hour, we'd compute the result as follows:
+     *
+     *     1. Subtract the fixed interval from the current time (Evaluation time) to compute the bounds of a fixed look-back window.
+     *     2. Verify that the target event has occurred within the CRON-interval window.
+     *     3. If the target event has occurred within the time window, then assertion passes.
+     *     4. If the target event has not occurred within the time window, then the assertion fails.
+     *
+     */
+    FIXED_INTERVAL
+  }
+
+  /**
+   * A cron schedule. This field is required when type is CRON.
+   */
+  cron: optional FreshnessCronSchedule
+
+  /**
+   * A fixed interval schedule. This field is required when type is FIXED_INTERVAL.
+   */
+  fixedInterval: optional FixedIntervalSchedule
+
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessCronSchedule.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessCronSchedule.pdl
new file mode 100644
index 0000000000000..d48900690c51d
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessCronSchedule.pdl
@@ -0,0 +1,25 @@
+namespace com.linkedin.assertion
+
+/**
+* Attributes defining a CRON-formatted schedule used for defining a freshness assertion.
+*/
+record FreshnessCronSchedule {
+  /**
+   * A cron-formatted execution interval, as a cron string, e.g. 1 * * * *
+   */
+  cron: string
+
+  /**
+   * Timezone in which the cron interval applies, e.g. America/Los Angeles
+   */
+  timezone: string
+
+  /**
+   * An optional offset in milliseconds to SUBTRACT from the timestamp generated by the cron schedule
+   * to generate the lower bounds of the "freshness window", or the window of time in which an event must have occurred in order for the Freshness check
+   * to be considering passing.
+   *
+   * If left empty, the start of the SLA window will be the _end_ of the previously evaluated Freshness window.
+   */
+   windowStartOffsetMs: optional long
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldKind.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldKind.pdl
new file mode 100644
index 0000000000000..7b25589e500da
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldKind.pdl
@@ -0,0 +1,17 @@
+namespace com.linkedin.assertion
+
+enum FreshnessFieldKind {
+  /**
+  * Determine that a change has occurred by inspecting an last modified field which
+  * represents the last time at which a row was changed.
+  */
+  LAST_MODIFIED,
+  /**
+   * Determine that a change has occurred by inspecting a field which should be tracked as the
+   * "high watermark" for the table. This should be an ascending number or date field.
+   *
+   * If rows with this column have not been added since the previous check
+   * then the Freshness Assertion will fail.
+   */
+  HIGH_WATERMARK
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl
new file mode 100644
index 0000000000000..04acd1c71352d
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl
@@ -0,0 +1,14 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.schema.SchemaFieldSpec
+
+
+/**
+* Lightweight spec used for referencing a particular schema field.
+**/
+record FreshnessFieldSpec includes SchemaFieldSpec {
+  /**
+   * The type of the field being used to verify the Freshness Assertion.
+   */
+  kind: optional FreshnessFieldKind
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentFieldTransformer.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentFieldTransformer.pdl
new file mode 100644
index 0000000000000..d1d3e7b23b666
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentFieldTransformer.pdl
@@ -0,0 +1,60 @@
+namespace com.linkedin.assertion
+
+/**
+* The definition of the transformer function  that should be applied to a given field / column value in a dataset
+* in order to determine the segment or bucket that it belongs to, which in turn is used to evaluate
+* volume assertions.
+*/
+record IncrementingSegmentFieldTransformer {
+  /**
+  * A 'standard' transformer type. Note that not all source systems will support all operators.
+  */
+  type: enum IncrementingSegmentFieldTransformerType {
+    /**
+    * Rounds a timestamp (in seconds) down to the start of the month.
+    */
+    TIMESTAMP_MS_TO_MINUTE
+
+    /**
+    * Rounds a timestamp (in milliseconds) down to the nearest hour.
+    */
+    TIMESTAMP_MS_TO_HOUR
+
+    /**
+    * Rounds a timestamp (in milliseconds) down to the start of the day.
+    */
+    TIMESTAMP_MS_TO_DATE
+
+    /**
+    * Rounds a timestamp (in milliseconds) down to the start of the month
+    */
+    TIMESTAMP_MS_TO_MONTH
+
+    /**
+    * Rounds a timestamp (in milliseconds) down to the start of the year
+    */
+    TIMESTAMP_MS_TO_YEAR
+
+    /**
+    * Rounds a numeric value down to the nearest integer.
+    */
+    FLOOR
+
+    /**
+    * Rounds a numeric value up to the nearest integer.
+    */
+    CEILING
+
+    /**
+    * A backdoor to provide a native operator type specific to a given source system like
+    * Snowflake, Redshift, BQ, etc.
+    */
+    NATIVE
+  }
+
+  /**
+  * The 'native' transformer type, useful as a back door if a custom operator is required.
+  * This field is required if the type is NATIVE.
+  */
+  nativeType: optional string
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountChange.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountChange.pdl
new file mode 100644
index 0000000000000..7c4c73f2ea887
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountChange.pdl
@@ -0,0 +1,33 @@
+namespace com.linkedin.assertion
+
+
+/**
+* Attributes defining an INCREMENTING_SEGMENT_ROW_COUNT_CHANGE volume assertion.
+*/
+record IncrementingSegmentRowCountChange {
+  /**
+   * A specification of how the 'segment' can be derived using a column and an optional transformer function.
+   */
+  segment: IncrementingSegmentSpec
+
+  /**
+  * The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage.
+  */
+  type: AssertionValueChangeType
+
+  /**
+  * The operator you'd like to apply to the row count value
+  *
+  * Note that only numeric operators are valid inputs:
+  * GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO,
+  * BETWEEN.
+  */
+  operator: AssertionStdOperator
+
+  /**
+  * The parameters you'd like to provide as input to the operator.
+  *
+  * Note that only numeric parameter types are valid inputs: NUMBER.
+  */
+  parameters: AssertionStdParameters
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountTotal.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountTotal.pdl
new file mode 100644
index 0000000000000..6b035107aae09
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentRowCountTotal.pdl
@@ -0,0 +1,27 @@
+namespace com.linkedin.assertion
+
+/**
+* Attributes defining an INCREMENTING_SEGMENT_ROW_COUNT_TOTAL volume assertion.
+*/
+record IncrementingSegmentRowCountTotal {
+  /**
+   * A specification of how the 'segment' can be derived using a column and an optional transformer function.
+   */
+  segment: IncrementingSegmentSpec
+
+  /**
+  * The operator you'd like to apply.
+  *
+  * Note that only numeric operators are valid inputs:
+  * GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO,
+  * BETWEEN.
+  */
+  operator: AssertionStdOperator
+
+  /**
+  * The parameters you'd like to provide as input to the operator.
+  *
+  * Note that only numeric parameter types are valid inputs: NUMBER.
+  */
+  parameters: AssertionStdParameters
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentSpec.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentSpec.pdl
new file mode 100644
index 0000000000000..eddd0c3da3df7
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/IncrementingSegmentSpec.pdl
@@ -0,0 +1,33 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.schema.SchemaFieldSpec
+
+/**
+* Core attributes required to identify an incrementing segment in a table. This type is mainly useful
+* for tables that constantly increase with new rows being added on a particular cadence (e.g. fact or event tables)
+*
+* An incrementing segment represents a logical chunk of data which is INSERTED
+* into a dataset on a regular interval, along with the presence of a constantly-incrementing column
+* value such as an event time, date partition, or last modified column.
+*
+* An incrementing segment is principally identified by 2 key attributes combined:
+*
+*  1. A field or column that represents the incrementing value. New rows that are inserted will be identified using this column.
+*     Note that the value of this column may not by itself represent the "bucket" or the "segment" in which the row falls.
+*
+*  2. [Optional] An transformer function that may be applied to the selected column value in order
+*     to obtain the final "segment identifier" or "bucket identifier". Rows that have the same value after applying the transformation
+*     will be grouped into the same segment, using which the final value (e.g. row count) will be determined.
+*/
+record IncrementingSegmentSpec {
+  /**
+  * The field to use to generate segments. It must be constantly incrementing as new rows are inserted.
+  */
+  field: SchemaFieldSpec
+
+  /**
+  * Optional transformer function to apply to the field in order to obtain the final segment or bucket identifier.
+  * If not provided, then no operator will be applied to the field. (identity function)
+  */
+  transformer: optional IncrementingSegmentFieldTransformer
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountChange.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountChange.pdl
new file mode 100644
index 0000000000000..85a915066f584
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountChange.pdl
@@ -0,0 +1,27 @@
+namespace com.linkedin.assertion
+
+/**
+* Attributes defining a ROW_COUNT_CHANGE volume assertion.
+*/
+record RowCountChange {
+  /**
+  * The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage.
+  */
+  type: AssertionValueChangeType
+
+  /**
+  * The operator you'd like to apply.
+  *
+  * Note that only numeric operators are valid inputs:
+  * GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO,
+  * BETWEEN.
+  */
+  operator: AssertionStdOperator
+
+  /**
+  * The parameters you'd like to provide as input to the operator.
+  *
+  * Note that only numeric parameter types are valid inputs: NUMBER.
+  */
+  parameters: AssertionStdParameters
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountTotal.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountTotal.pdl
new file mode 100644
index 0000000000000..f691f15f62e04
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/RowCountTotal.pdl
@@ -0,0 +1,22 @@
+namespace com.linkedin.assertion
+
+/**
+* Attributes defining a ROW_COUNT_TOTAL volume assertion.
+*/
+record RowCountTotal {
+  /**
+  * The operator you'd like to apply.
+  *
+  * Note that only numeric operators are valid inputs:
+  * GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO,
+  * BETWEEN.
+  */
+  operator: AssertionStdOperator
+
+  /**
+  * The parameters you'd like to provide as input to the operator.
+  *
+  * Note that only numeric parameter types are valid inputs: NUMBER.
+  */
+  parameters: AssertionStdParameters
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl
new file mode 100644
index 0000000000000..fd246e0c7cfc4
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl
@@ -0,0 +1,29 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.common.Urn
+import com.linkedin.schema.SchemaMetadata
+
+/**
+* Attributes that are applicable to schema assertions
+**/
+record SchemaAssertionInfo {
+    /**
+    * The entity targeted by the assertion
+    */
+    @Searchable = {
+      "fieldType": "URN"
+    }
+    @Relationship = {
+      "name": "Asserts",
+      "entityTypes": [ "dataset", "dataJob" ]
+    }
+    entity: Urn
+
+    /**
+     * A definition of the expected structure for the asset
+     *
+     * Note that many of the fields of this model, especially those related to metadata (tags, terms)
+     * will go unused in this context.
+     */
+     schema: SchemaMetadata
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl
new file mode 100644
index 0000000000000..327b76f95762e
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl
@@ -0,0 +1,82 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.common.Urn
+import com.linkedin.dataset.DatasetFilter
+
+/**
+* Attributes defining a dataset Volume Assertion
+*/
+record VolumeAssertionInfo {
+    /**
+     * The type of the freshness assertion being monitored.
+     */
+    @Searchable = {}
+    type: enum VolumeAssertionType {
+      /**
+       * A volume assertion that is evaluated against the total row count of a dataset.
+       */
+       ROW_COUNT_TOTAL
+       /**
+       * A volume assertion that is evaluated against an incremental row count of a dataset,
+       * or a row count change.
+       */
+       ROW_COUNT_CHANGE
+       /**
+        * A volume assertion that checks the latest "segment" in a table based on an incrementing
+        * column to check whether it's row count falls into a particular range.
+        *
+        * This can be used to monitor the row count of an incrementing date-partition column segment.
+        */
+       INCREMENTING_SEGMENT_ROW_COUNT_TOTAL
+       /**
+        * A volume assertion that compares the row counts in neighboring "segments" or "partitions"
+        * of an incrementing column.
+        * This can be used to track changes between subsequent date partition
+        * in a table, for example.
+        */
+       INCREMENTING_SEGMENT_ROW_COUNT_CHANGE
+    }
+
+    /**
+    * The entity targeted by this Volume check.
+    */
+    @Searchable = {
+      "fieldType": "URN"
+    }
+    @Relationship = {
+      "name": "Asserts",
+      "entityTypes": [ "dataset" ]
+    }
+    entity: Urn
+
+    /**
+    * Produce FAILURE Assertion Result if the row count of the asset does not meet specific requirements.
+    * Required if type is 'ROW_COUNT_TOTAL'
+    */
+    rowCountTotal: optional RowCountTotal
+
+    /**
+    * Produce FAILURE Assertion Result if the delta row count of the asset does not meet specific requirements
+    * within a given period of time.
+    * Required if type is 'ROW_COUNT_CHANGE'
+    */
+    rowCountChange: optional RowCountChange
+
+    /**
+    * Produce FAILURE Assertion Result if the asset's latest incrementing segment row count total
+    * does not meet specific requirements. Required if type is 'INCREMENTING_SEGMENT_ROW_COUNT_TOTAL'
+    */
+    incrementingSegmentRowCountTotal: optional IncrementingSegmentRowCountTotal
+
+    /**
+    * Produce FAILURE Assertion Result if the asset's incrementing segment row count delta
+    * does not meet specific requirements. Required if type is 'INCREMENTING_SEGMENT_ROW_COUNT_CHANGE'
+    */
+    incrementingSegmentRowCountChange: optional IncrementingSegmentRowCountChange
+
+    /**
+     * A definition of the specific filters that should be applied, when performing monitoring.
+     * If not provided, there is no filter, and the full table is under consideration.
+     */
+    filter: optional DatasetFilter
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractProperties.pdl
new file mode 100644
index 0000000000000..a623f585df30c
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractProperties.pdl
@@ -0,0 +1,59 @@
+namespace com.linkedin.datacontract
+
+import com.linkedin.common.Urn
+
+/**
+ * Information about a data contract
+ */
+@Aspect = {
+  "name": "dataContractProperties"
+}
+record DataContractProperties {
+  /**
+   * The entity that this contract is associated with. Currently, we only support Dataset contracts, but
+   * in the future we may also support Data Product level contracts.
+   */
+  @Relationship = {
+    "name": "ContractFor",
+    "entityTypes": [ "dataset" ]
+  }
+  entity: Urn
+
+  /**
+   * An optional set of schema contracts. If this is a dataset contract, there will only be one.
+   */
+   @Relationship = {
+     "/*/assertion": {
+        "name": "IncludesSchemaAssertion",
+        "entityTypes": [ "assertion" ]
+     }
+   }
+  schema: optional array[SchemaContract]
+
+  /**
+   * An optional set of FRESHNESS contracts. If this is a dataset contract, there will only be one.
+   */
+   @Relationship = {
+     "/*/assertion": {
+        "name": "IncludesFreshnessAssertion",
+        "entityTypes": [ "assertion" ]
+     }
+   }
+  freshness: optional array[FreshnessContract]
+
+  /**
+   * An optional set of Data Quality contracts, e.g. table and column level contract constraints.
+   */
+   @Relationship = {
+     "/*/assertion": {
+        "name": "IncludesDataQualityAssertion",
+        "entityTypes": [ "assertion" ]
+     }
+   }
+  dataQuality: optional array[DataQualityContract]
+
+  /**
+   * YAML-formatted contract definition
+   */
+  rawContract: optional string
+}
diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractStatus.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractStatus.pdl
new file mode 100644
index 0000000000000..d61fb191ae53d
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataContractStatus.pdl
@@ -0,0 +1,27 @@
+namespace com.linkedin.datacontract
+
+import com.linkedin.common.Urn
+import com.linkedin.common.CustomProperties
+
+/**
+ * Information about the status of a data contract
+ */
+@Aspect = {
+  "name": "dataContractStatus"
+}
+record DataContractStatus includes CustomProperties {
+  /**
+   * The latest state of the data contract
+   */
+   @Searchable = {}
+   state: enum DataContractState {
+    /**
+    * The data contract is active.
+    */
+    ACTIVE
+    /**
+    * The data contract is pending implementation.
+    */
+    PENDING
+   }
+}
diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl
new file mode 100644
index 0000000000000..273d2c2a56f95
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl
@@ -0,0 +1,16 @@
+namespace com.linkedin.datacontract
+
+import com.linkedin.common.Urn
+
+
+/**
+ * A data quality contract pertaining to a physical data asset
+ * Data Quality contracts are used to make assertions about data quality metrics for a physical data asset
+ */
+record DataQualityContract {
+  /**
+   * The assertion representing the Data Quality contract.
+   * E.g. a table or column-level assertion.
+   */
+  assertion: Urn
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/FreshnessContract.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/FreshnessContract.pdl
new file mode 100644
index 0000000000000..8cfa66846d505
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/FreshnessContract.pdl
@@ -0,0 +1,13 @@
+namespace com.linkedin.datacontract
+
+import com.linkedin.common.Urn
+
+/**
+ * A contract pertaining to the operational SLAs of a physical data asset
+ */
+record FreshnessContract {
+  /**
+   * The assertion representing the SLA contract.
+   */
+  assertion: Urn
+}
diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl
new file mode 100644
index 0000000000000..6c11e0da5b128
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl
@@ -0,0 +1,13 @@
+namespace com.linkedin.datacontract
+
+import com.linkedin.common.Urn
+
+/**
+ * Expectations for a logical schema
+ */
+record SchemaContract {
+  /**
+   * The assertion representing the schema contract.
+   */
+  assertion: Urn
+}
diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetFilter.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetFilter.pdl
new file mode 100644
index 0000000000000..6823398f79f3d
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetFilter.pdl
@@ -0,0 +1,30 @@
+namespace com.linkedin.dataset
+
+/**
+ * A definition of filters that should be used when
+ * querying an external Dataset or Table.
+ *
+ * Note that this models should NOT be used for working with
+ * search / filter on DataHub Platform itself.
+ */
+record DatasetFilter {
+  /**
+   * How the partition will be represented in this model.
+   *
+   * In the future, we'll likely add support for more structured
+   * predicates.
+   */
+  type: enum DatasetFilterType {
+    /**
+     * The partition is represented as a an opaque, raw SQL
+     * clause.
+     */
+    SQL
+  }
+
+  /**
+   * The raw where clause string which will be used for monitoring.
+   * Required if the type is SQL.
+   */
+  sql: optional string
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataContractKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataContractKey.pdl
new file mode 100644
index 0000000000000..f1d4a709cd6bf
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataContractKey.pdl
@@ -0,0 +1,14 @@
+namespace com.linkedin.metadata.key
+
+/**
+ * Key for a Data Contract
+ */
+@Aspect = {
+  "name": "dataContractKey"
+}
+record DataContractKey {
+  /**
+  * Unique id for the contract
+  */
+  id: string
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaFieldSpec.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaFieldSpec.pdl
new file mode 100644
index 0000000000000..e875ff7a84403
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaFieldSpec.pdl
@@ -0,0 +1,21 @@
+namespace com.linkedin.schema
+
+/**
+* Lightweight spec used for referencing a particular schema field.
+**/
+record SchemaFieldSpec {
+  /**
+  * The field path
+  */
+  path: string
+
+  /**
+  * The DataHub standard schema field type.
+  */
+  type: string
+
+  /**
+  * The native field type
+  */
+  nativeType: string
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml
index 56fc5f6568eb7..11d0f74305d7b 100644
--- a/metadata-models/src/main/resources/entity-registry.yml
+++ b/metadata-models/src/main/resources/entity-registry.yml
@@ -262,6 +262,7 @@ entities:
       - assertionInfo
       - dataPlatformInstance
       - assertionRunEvent
+      - assertionActions
       - status
   - name: dataHubRetention
     category: internal
@@ -457,4 +458,12 @@ entities:
     aspects:
       - ownershipTypeInfo
       - status
+  - name: dataContract
+    category: core
+    keyAspect: dataContractKey
+    aspects:
+      - dataContractProperties
+      - dataContractStatus
+      - status
+
 events:

From 2bc685d3b98f879d1c3051a8484a78489359d910 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Thu, 5 Oct 2023 09:31:32 +0530
Subject: [PATCH 008/259] ci: tweak ci to decrease wait time of devs (#8945)

---
 .github/workflows/build-and-test.yml               | 14 ++++++++++----
 .github/workflows/metadata-ingestion.yml           |  7 ++++---
 .../integration/powerbi/test_admin_only_api.py     |  3 +++
 .../tests/integration/powerbi/test_m_parser.py     |  2 +-
 .../tests/integration/powerbi/test_powerbi.py      |  2 +-
 .../tests/integration/snowflake/test_snowflake.py  |  4 ++--
 .../integration/tableau/test_tableau_ingest.py     |  2 +-
 .../tests/integration/trino/test_trino.py          |  5 ++---
 8 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 3f409878b191f..96b9bb2a14933 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -27,8 +27,8 @@ jobs:
         command:
           [
             # metadata-ingestion and airflow-plugin each have dedicated build jobs
-            "./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel",
-            "./gradlew :datahub-frontend:build :datahub-web-react:build --parallel",
+            "except_metadata_ingestion",
+            "frontend"
           ]
         timezone:
           [
@@ -53,9 +53,15 @@ jobs:
         with:
           python-version: "3.10"
           cache: pip
-      - name: Gradle build (and test)
+      - name: Gradle build (and test) for metadata ingestion
+        # we only need the timezone runs for frontend tests
+        if: ${{  matrix.command == 'except_metadata_ingestion' && matrix.timezone == 'America/New_York' }}
         run: |
-          ${{ matrix.command }}
+          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel
+      - name: Gradle build (and test) for frontend
+        if: ${{  matrix.command == 'frontend' }}
+        run: |
+          ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
           NODE_OPTIONS: "--max-old-space-size=3072"
       - uses: actions/upload-artifact@v3
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
index 8d56a0adf5bd5..dea4603868f8e 100644
--- a/.github/workflows/metadata-ingestion.yml
+++ b/.github/workflows/metadata-ingestion.yml
@@ -34,7 +34,6 @@ jobs:
         python-version: ["3.7", "3.10"]
         command:
           [
-            "lint",
             "testQuick",
             "testIntegrationBatch0",
             "testIntegrationBatch1",
@@ -54,6 +53,9 @@ jobs:
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package
         run: ./gradlew :metadata-ingestion:installPackageOnly
+      - name: Run lint alongwith testQuick
+        if: ${{  matrix.command == 'testQuick' }}
+        run: ./gradlew :metadata-ingestion:lint
       - name: Run metadata-ingestion tests
         run: ./gradlew :metadata-ingestion:${{ matrix.command }}
       - name: Debug info
@@ -65,7 +67,6 @@ jobs:
           docker image ls
           docker system df
       - uses: actions/upload-artifact@v3
-        if: ${{ always() && matrix.command != 'lint' }}
         with:
           name: Test Results (metadata ingestion ${{ matrix.python-version }})
           path: |
@@ -73,7 +74,7 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
       - name: Upload coverage to Codecov
-        if: ${{ always() && matrix.python-version == '3.10' && matrix.command != 'lint' }}
+        if: ${{ always() && matrix.python-version == '3.10' }}
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
index f95fd81681a9a..6f45dcf97f1dd 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
@@ -3,11 +3,14 @@
 from typing import Any, Dict
 from unittest import mock
 
+import pytest
 from freezegun import freeze_time
 
 from datahub.ingestion.run.pipeline import Pipeline
 from tests.test_helpers import mce_helpers
 
+pytestmark = pytest.mark.integration_batch_2
+
 FROZEN_TIME = "2022-02-03 07:00:00"
 
 
diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
index 2e9c02ef759a5..e3cc6c8101650 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
@@ -19,7 +19,7 @@
 from datahub.ingestion.source.powerbi.m_query.resolver import DataPlatformTable, Lineage
 from datahub.utilities.sqlglot_lineage import ColumnLineageInfo, DownstreamColumnRef
 
-pytestmark = pytest.mark.slow
+pytestmark = pytest.mark.integration_batch_2
 
 M_QUERIES = [
     'let\n    Source = Snowflake.Databases("bu10758.ap-unknown-2.fakecomputing.com","PBI_TEST_WAREHOUSE_PROD",[Role="PBI_TEST_MEMBER"]),\n    PBI_TEST_Database = Source{[Name="PBI_TEST",Kind="Database"]}[Data],\n    TEST_Schema = PBI_TEST_Database{[Name="TEST",Kind="Schema"]}[Data],\n    TESTTABLE_Table = TEST_Schema{[Name="TESTTABLE",Kind="Table"]}[Data]\nin\n    TESTTABLE_Table',
diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
index b0695e3ea9954..7232d2a38da1d 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
@@ -21,7 +21,7 @@
 )
 from tests.test_helpers import mce_helpers
 
-pytestmark = pytest.mark.slow
+pytestmark = pytest.mark.integration_batch_2
 FROZEN_TIME = "2022-02-03 07:00:00"
 
 
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
index dec50aefd19f0..2c77ace8b53e5 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
@@ -30,6 +30,8 @@
 from tests.integration.snowflake.common import FROZEN_TIME, default_query_results
 from tests.test_helpers import mce_helpers
 
+pytestmark = pytest.mark.integration_batch_2
+
 
 def random_email():
     return (
@@ -55,7 +57,6 @@ def random_cloud_region():
     )
 
 
-@pytest.mark.integration
 def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/snowflake"
 
@@ -183,7 +184,6 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
 
 
 @freeze_time(FROZEN_TIME)
-@pytest.mark.integration
 def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/snowflake"
 
diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
index 53b8519a886d3..c31867f5aa904 100644
--- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
+++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
@@ -757,7 +757,7 @@ def test_tableau_no_verify():
 
 
 @freeze_time(FROZEN_TIME)
-@pytest.mark.slow
+@pytest.mark.integration_batch_2
 def test_tableau_signout_timeout(pytestconfig, tmp_path, mock_datahub_graph):
     enable_logging()
     output_file_name: str = "tableau_signout_timeout_mces.json"
diff --git a/metadata-ingestion/tests/integration/trino/test_trino.py b/metadata-ingestion/tests/integration/trino/test_trino.py
index 22e5f6f91a06e..177c273c0d242 100644
--- a/metadata-ingestion/tests/integration/trino/test_trino.py
+++ b/metadata-ingestion/tests/integration/trino/test_trino.py
@@ -13,6 +13,8 @@
 from tests.test_helpers import fs_helpers, mce_helpers
 from tests.test_helpers.docker_helpers import wait_for_port
 
+pytestmark = pytest.mark.integration_batch_1
+
 FROZEN_TIME = "2021-09-23 12:00:00"
 
 data_platform = "trino"
@@ -51,7 +53,6 @@ def loaded_trino(trino_runner):
 
 
 @freeze_time(FROZEN_TIME)
-@pytest.mark.integration
 @pytest.mark.xfail
 def test_trino_ingest(
     loaded_trino, test_resources_dir, pytestconfig, tmp_path, mock_time
@@ -111,7 +112,6 @@ def test_trino_ingest(
 
 
 @freeze_time(FROZEN_TIME)
-@pytest.mark.integration
 def test_trino_hive_ingest(
     loaded_trino, test_resources_dir, pytestconfig, tmp_path, mock_time
 ):
@@ -167,7 +167,6 @@ def test_trino_hive_ingest(
 
 
 @freeze_time(FROZEN_TIME)
-@pytest.mark.integration
 def test_trino_instance_ingest(
     loaded_trino, test_resources_dir, pytestconfig, tmp_path, mock_time
 ):

From 2fcced6db9d30228c421d0773c8249c889cd0d9f Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Thu, 5 Oct 2023 09:31:57 +0530
Subject: [PATCH 009/259] docs(ingest): add permissions required for athena
 ingestion (#8948)

---
 .../docs/sources/athena/athena_pre.md         | 72 +++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 metadata-ingestion/docs/sources/athena/athena_pre.md

diff --git a/metadata-ingestion/docs/sources/athena/athena_pre.md b/metadata-ingestion/docs/sources/athena/athena_pre.md
new file mode 100644
index 0000000000000..a56457d3f84fc
--- /dev/null
+++ b/metadata-ingestion/docs/sources/athena/athena_pre.md
@@ -0,0 +1,72 @@
+### Prerequisities
+
+In order to execute this source, you will need to create a policy with below permissions and attach it to the the aws role or credentials used in ingestion recipe.
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "VisualEditor0",
+      "Effect": "Allow",
+      "Action": [
+        "athena:GetTableMetadata", 
+        "athena:StartQueryExecution", 
+        "athena:GetQueryResults", 
+        "athena:GetDatabase", 
+        "athena:ListDataCatalogs",
+        "athena:GetDataCatalog", 
+        "athena:ListQueryExecutions", 
+        "athena:GetWorkGroup", 
+        "athena:StopQueryExecution", 
+        "athena:GetQueryResultsStream", 
+        "athena:ListDatabases", 
+        "athena:GetQueryExecution", 
+        "athena:ListTableMetadata", 
+        "athena:BatchGetQueryExecution", 
+        "glue:GetTables", 
+        "glue:GetDatabases", 
+        "glue:GetTable",
+        "glue:GetDatabase",
+        "glue:SearchTables",
+        "glue:GetTableVersions",
+        "glue:GetTableVersion",
+        "glue:GetPartition", 
+        "glue:GetPartitions", 
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:GetBucketLocation",
+      ],
+      "Resource": [
+        "arn:aws:athena:${region-id}:${account-id}:datacatalog/*",
+        "arn:aws:athena:${region-id}:${account-id}:workgroup/*",
+        "arn:aws:glue:${region-id}:${account-id}:tableVersion/*/*/*",
+        "arn:aws:glue:${region-id}:${account-id}:table/*/*", 
+        "arn:aws:glue:${region-id}:${account-id}:catalog", 
+        "arn:aws:glue:${region-id}:${account-id}:database/*", 
+        "arn:aws:s3:::${datasets-bucket}",
+        "arn:aws:s3:::${datasets-bucket}/*"
+      ]
+    },
+    {
+      "Sid": "VisualEditor1",
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:ListBucketMultipartUploads",
+        "s3:AbortMultipartUpload",
+        "s3:ListBucket",
+        "s3:GetBucketLocation",
+        "s3:ListMultipartUploadParts"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${athena-query-result-bucket}/*",
+        "arn:aws:s3:::${athena-query-result-bucket}"
+      ]
+    },
+  ]
+}
+```
+
+Replace `${var}` with appropriate values as per your athena setup.
\ No newline at end of file

From 6310e51eb09711e98d86625578127349c5144c66 Mon Sep 17 00:00:00 2001
From: Jinlin Yang <86577891+jinlintt@users.noreply.github.com>
Date: Wed, 4 Oct 2023 21:03:31 -0700
Subject: [PATCH 010/259] feat(ingestion/dynamodb): implement pagination for
 list_tables (#8910)

---
 .../app/ingest/source/builder/sources.json    |  4 +-
 .../docs/sources/dynamodb/dynamodb_post.md    | 13 ++-
 .../docs/sources/dynamodb/dynamodb_pre.md     |  6 +-
 .../docs/sources/dynamodb/dynamodb_recipe.yml | 16 ++--
 .../ingestion/source/dynamodb/dynamodb.py     | 85 +++++++++++--------
 5 files changed, 65 insertions(+), 59 deletions(-)

diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
index 1bd5b6f1f768b..b18384909c33f 100644
--- a/datahub-web-react/src/app/ingest/source/builder/sources.json
+++ b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -130,7 +130,7 @@
         "name": "dynamodb",
         "displayName": "DynamoDB",
         "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
-        "recipe": "source:\n    type: dynamodb\n    config:\n        platform_instance: \"AWS_ACCOUNT_ID\"\n        aws_access_key_id : '${AWS_ACCESS_KEY_ID}'\n        aws_secret_access_key : '${AWS_SECRET_ACCESS_KEY}'\n        # User could use the below option to provide a list of primary keys of a table in dynamodb format,\n        # those items from given primary keys will be included when we scan the table.\n        # For each table we can retrieve up to 16 MB of data, which can contain as many as 100 items.\n        # We'll enforce the the primary keys list size not to exceed 100\n        # The total items we'll try to retrieve in these two scenarios:\n        # 1. If user don't specify include_table_item: we'll retrieve up to 100 items\n        # 2. If user specifies include_table_item: we'll retrieve up to 100 items plus user specified items in\n        # the table, with a total not more than 200 items\n        # include_table_item:\n        #   table_name:\n        #     [\n        #       {\n        #         'partition_key_name': { 'attribute_type': 'attribute_value' },\n        #         'sort_key_name': { 'attribute_type': 'attribute_value' },\n        #       },\n        #     ]"
+        "recipe": "source:\n    type: dynamodb\n    config:\n        platform_instance: \"AWS_ACCOUNT_ID\"\n        aws_access_key_id : '${AWS_ACCESS_KEY_ID}'\n        aws_secret_access_key : '${AWS_SECRET_ACCESS_KEY}'\n        # If there are items that have most representative fields of the table, users could use the\n        # `include_table_item` option to provide a list of primary keys of the table in dynamodb format.\n        # For each `region.table`, the list of primary keys can be at most 100.\n        # We include these items in addition to the first 100 items in the table when we scan it.\n        # include_table_item:\n        #   region.table_name:\n        #     [\n        #       {\n        #         'partition_key_name': { 'attribute_type': 'attribute_value' },\n        #         'sort_key_name': { 'attribute_type': 'attribute_value' },\n        #       },\n        #     ]"
     },
     {
         "urn": "urn:li:dataPlatform:glue",
@@ -223,4 +223,4 @@
         "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
         "recipe": "source:\n  type: <source-type>\n  config:\n    # Source-type specifics config\n    <source-configs>"
     }
-]
\ No newline at end of file
+]
diff --git a/metadata-ingestion/docs/sources/dynamodb/dynamodb_post.md b/metadata-ingestion/docs/sources/dynamodb/dynamodb_post.md
index 7f9a0324c7bc6..a1c0a6e2d4d21 100644
--- a/metadata-ingestion/docs/sources/dynamodb/dynamodb_post.md
+++ b/metadata-ingestion/docs/sources/dynamodb/dynamodb_post.md
@@ -1,21 +1,18 @@
-## Limitations
-
-For each region, the list table operation returns maximum number 100 tables, we need to further improve it by implementing pagination for listing tables
-
 ## Advanced Configurations
 
 ### Using `include_table_item` config
 
-If there are items that have most representative fields of the table, user could use the `include_table_item` option to provide a list of primary keys of a table in dynamodb format, those items from given primary keys will be included when we scan the table.
+If there are items that have most representative fields of the table, users could use the `include_table_item` option to provide a list of primary keys of the table in dynamodb format. We include these items in addition to the first 100 items in the table when we scan it.
 
-Take [AWS DynamoDB Developer Guide Example tables and data](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/AppendixSampleTables.html) as an example, if user has a table `Reply` with composite primary key `Id` and `ReplyDateTime`, user can use `include_table_item` to include 2 items as following:
+Take [AWS DynamoDB Developer Guide Example tables and data](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/AppendixSampleTables.html) as an example, if a account has a table `Reply` in the `us-west-2` region with composite primary key `Id` and `ReplyDateTime`, users can use `include_table_item` to include 2 items as following:
 
 Example:
 
 ```yml
-# put the table name and composite key in DynamoDB format
+# The table name should be in the format of region.table_name
+# The primary keys should be in the DynamoDB format
 include_table_item:
-  Reply:
+  us-west-2.Reply:
     [
       {
         "ReplyDateTime": { "S": "2015-09-22T19:58:22.947Z" },
diff --git a/metadata-ingestion/docs/sources/dynamodb/dynamodb_pre.md b/metadata-ingestion/docs/sources/dynamodb/dynamodb_pre.md
index a48e8d5be04aa..598d0ecdb3786 100644
--- a/metadata-ingestion/docs/sources/dynamodb/dynamodb_pre.md
+++ b/metadata-ingestion/docs/sources/dynamodb/dynamodb_pre.md
@@ -1,8 +1,8 @@
 ### Prerequisities
 
-In order to execute this source, you will need to create access key and secret keys that have DynamoDB read access. You can create these policies and attach to your account or can ask your account admin to attach these policies to your account.
+In order to execute this source, you need to attach the `AmazonDynamoDBReadOnlyAccess` policy to a user in your AWS account. Then create an API access key and secret for the user.
 
-For access key permissions, you can create a policy with permissions below and attach to your account, you can find more details in [Managing access keys for IAM users](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
+For a user to be able to create API access key, it needs the following access key permissions. Your AWS account admin can create a policy with these permissions and attach to the user, you can find more details in [Managing access keys for IAM users](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
 
 ```json
 {
@@ -22,5 +22,3 @@ For access key permissions, you can create a policy with permissions below and a
   ]
 }
 ```
-
-For DynamoDB read access, you can simply attach AWS managed policy `AmazonDynamoDBReadOnlyAccess` to your account, you can find more details in [Attaching a policy to an IAM user group](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_groups_manage_attach-policy.html)
diff --git a/metadata-ingestion/docs/sources/dynamodb/dynamodb_recipe.yml b/metadata-ingestion/docs/sources/dynamodb/dynamodb_recipe.yml
index bd41637907b5c..4f4edc9a7d496 100644
--- a/metadata-ingestion/docs/sources/dynamodb/dynamodb_recipe.yml
+++ b/metadata-ingestion/docs/sources/dynamodb/dynamodb_recipe.yml
@@ -4,16 +4,14 @@ source:
     platform_instance: "AWS_ACCOUNT_ID"
     aws_access_key_id: "${AWS_ACCESS_KEY_ID}"
     aws_secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
-    # User could use the below option to provide a list of primary keys of a table in dynamodb format,
-    # those items from given primary keys will be included when we scan the table.
-    # For each table we can retrieve up to 16 MB of data, which can contain as many as 100 items.
-    # We'll enforce the the primary keys list size not to exceed 100
-    # The total items we'll try to retrieve in these two scenarios:
-    # 1. If user don't specify include_table_item: we'll retrieve up to 100 items
-    # 2. If user specifies include_table_item: we'll retrieve up to 100 items plus user specified items in
-    # the table, with a total not more than 200 items
+    #
+    # If there are items that have most representative fields of the table, users could use the
+    # `include_table_item` option to provide a list of primary keys of the table in dynamodb format.
+    # For each `region.table`, the list of primary keys can be at most 100.
+    # We include these items in addition to the first 100 items in the table when we scan it.
+    #
     # include_table_item:
-    #   table_name:
+    #   region.table_name:
     #     [
     #       {
     #         "partition_key_name": { "attribute_type": "attribute_value" },
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
index 6b7c118373673..d7f3dfb9279fb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
@@ -1,5 +1,5 @@
 import logging
-from dataclasses import field
+from dataclasses import dataclass, field
 from typing import Any, Counter, Dict, Iterable, List, Optional, Type, Union
 
 import boto3
@@ -79,12 +79,13 @@ class DynamoDBConfig(DatasetSourceConfigMixin, StatefulIngestionConfigBase):
 
     table_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
-        description="regex patterns for tables to filter in ingestion.",
+        description="Regex patterns for tables to filter in ingestion. The table name format is 'region.table'",
     )
     # Custom Stateful Ingestion settings
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
 
 
+@dataclass
 class DynamoDBSourceReport(StaleEntityRemovalSourceReport):
     filtered: List[str] = field(default_factory=list)
 
@@ -175,39 +176,30 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
         # traverse databases in sorted order so output is consistent
         for region in dynamodb_regions:
-            try:
-                # create a new dynamodb client for each region,
-                # it seems for one client we could only list the table of one specific region,
-                # the list_tables() method don't take any config that related to region
-                # TODO: list table returns maximum number 100, need to implement pagination here
-                dynamodb_client = boto3.client(
-                    "dynamodb",
-                    region_name=region,
-                    aws_access_key_id=self.config.aws_access_key_id
-                    if self.config.aws_access_key_id
-                    else None,
-                    aws_secret_access_key=self.config.aws_secret_access_key.get_secret_value()
-                    if self.config.aws_secret_access_key
-                    else None,
-                )
-                table_names: List[str] = dynamodb_client.list_tables()["TableNames"]
-            except Exception as ex:
-                # TODO: If regions is config input then this would be self.report.report_warning,
-                # we can create dynamodb client to take aws region or regions as user input
-                logger.info(f"exception happen in region {region}, skipping: {ex}")
-                continue
-            for table_name in sorted(table_names):
-                if not self.config.table_pattern.allowed(table_name):
+            logger.info(f"Processing region {region}")
+            # create a new dynamodb client for each region,
+            # it seems for one client we could only list the table of one specific region,
+            # the list_tables() method don't take any config that related to region
+            dynamodb_client = boto3.client(
+                "dynamodb",
+                region_name=region,
+                aws_access_key_id=self.config.aws_access_key_id,
+                aws_secret_access_key=self.config.aws_secret_access_key.get_secret_value(),
+            )
+
+            for table_name in self._list_tables(dynamodb_client):
+                dataset_name = f"{region}.{table_name}"
+                if not self.config.table_pattern.allowed(dataset_name):
+                    logger.debug(f"skipping table: {dataset_name}")
+                    self.report.report_dropped(dataset_name)
                     continue
+
+                logger.debug(f"Processing table: {dataset_name}")
                 table_info = dynamodb_client.describe_table(TableName=table_name)[
                     "Table"
                 ]
                 account_id = table_info["TableArn"].split(":")[4]
-                if not self.config.table_pattern.allowed(table_name):
-                    self.report.report_dropped(table_name)
-                    continue
                 platform_instance = self.config.platform_instance or account_id
-                dataset_name = f"{region}.{table_name}"
                 dataset_urn = make_dataset_urn_with_platform_instance(
                     platform=self.platform,
                     platform_instance=platform_instance,
@@ -222,7 +214,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 )
                 primary_key_dict = self.extract_primary_key_from_key_schema(table_info)
                 table_schema = self.construct_schema_from_dynamodb(
-                    dynamodb_client, table_name
+                    dynamodb_client, region, table_name
                 )
                 schema_metadata = self.construct_schema_metadata(
                     table_name,
@@ -254,9 +246,25 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     aspect=platform_instance_aspect,
                 ).as_workunit()
 
+    def _list_tables(
+        self,
+        dynamodb_client: BaseClient,
+    ) -> Iterable[str]:
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/paginator/ListTables.html
+        try:
+            for page in dynamodb_client.get_paginator("list_tables").paginate():
+                table_names = page.get("TableNames")
+                if table_names:
+                    yield from table_names
+        except Exception as ex:
+            # TODO: If regions is config input then this would be self.report.report_warning,
+            # we can create dynamodb client to take aws region or regions as user input
+            logger.info(f"Exception happened while listing tables, skipping: {ex}")
+
     def construct_schema_from_dynamodb(
         self,
         dynamodb_client: BaseClient,
+        region: str,
         table_name: str,
     ) -> Dict[str, SchemaDescription]:
         """
@@ -275,7 +283,7 @@ def construct_schema_from_dynamodb(
         The MaxItems is the total number of items to return, and PageSize is the size of each page, we are assigning same value
         to these two config. If MaxItems is more than PageSize then we expect MaxItems / PageSize pages in response_iterator will return
         """
-        self.include_table_item_to_schema(dynamodb_client, table_name, schema)
+        self.include_table_item_to_schema(dynamodb_client, region, table_name, schema)
         response_iterator = paginator.paginate(
             TableName=table_name,
             PaginationConfig={
@@ -294,33 +302,38 @@ def construct_schema_from_dynamodb(
     def include_table_item_to_schema(
         self,
         dynamodb_client: Any,
+        region: str,
         table_name: str,
         schema: Dict[str, SchemaDescription],
     ) -> None:
         """
-        It will look up in the config include_table_item dict to see if the current table name exists as key,
+        It will look up in the config include_table_item dict to see if "region.table_name" exists as key,
         if it exists then get the items by primary key from the table and put it to schema
         """
         if self.config.include_table_item is None:
             return
-        if table_name not in self.config.include_table_item.keys():
+        dataset_name = f"{region}.{table_name}"
+        if dataset_name not in self.config.include_table_item.keys():
             return
-        primary_key_list = self.config.include_table_item.get(table_name)
+        primary_key_list = self.config.include_table_item.get(dataset_name)
         assert isinstance(primary_key_list, List)
         if len(primary_key_list) > MAX_PRIMARY_KEYS_SIZE:
             logger.info(
-                f"the provided primary keys list size exceeded the max size for table {table_name}, we'll only process the first {MAX_PRIMARY_KEYS_SIZE} items"
+                f"the provided primary keys list size exceeded the max size for table {dataset_name}, we'll only process the first {MAX_PRIMARY_KEYS_SIZE} items"
             )
             primary_key_list = primary_key_list[0:MAX_PRIMARY_KEYS_SIZE]
         items = []
         response = dynamodb_client.batch_get_item(
             RequestItems={table_name: {"Keys": primary_key_list}}
-        ).get("Responses", None)
+        ).get("Responses")
         if response is None:
             logger.error(
                 f"failed to retrieve item from table {table_name} by the given key {primary_key_list}"
             )
             return
+        logger.debug(
+            f"successfully retrieved {len(primary_key_list)} items based on supplied primary key list"
+        )
         items = response.get(table_name)
 
         self.construct_schema_from_items(items, schema)

From c9309ff1579e31c79d2d8e764a89f7c5e3ff483c Mon Sep 17 00:00:00 2001
From: Shirshanka Das <shirshanka@apache.org>
Date: Thu, 5 Oct 2023 09:07:12 -0700
Subject: [PATCH 011/259] feat(ci): enable ci to run on PR-s targeting all
 branches (#8933)

---
 .github/workflows/airflow-plugin.yml        |  2 +-
 .github/workflows/build-and-test.yml        | 11 +++--------
 .github/workflows/check-datahub-jars.yml    |  9 ++-------
 .github/workflows/close-stale-issues.yml    |  4 +++-
 .github/workflows/code-checks.yml           | 13 ++++---------
 .github/workflows/docker-postgres-setup.yml |  3 +--
 .github/workflows/docker-unified.yml        |  7 +++----
 .github/workflows/documentation.yml         |  2 +-
 .github/workflows/lint-actions.yml          |  4 +++-
 .github/workflows/metadata-ingestion.yml    |  2 +-
 .github/workflows/metadata-io.yml           |  2 +-
 .github/workflows/spark-smoke-test.yml      |  2 +-
 12 files changed, 24 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
index a250bddcc16d1..54042d104d906 100644
--- a/.github/workflows/airflow-plugin.yml
+++ b/.github/workflows/airflow-plugin.yml
@@ -10,7 +10,7 @@ on:
       - "metadata-models/**"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
       - ".github/**"
       - "metadata-ingestion-modules/airflow-plugin/**"
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 96b9bb2a14933..25f3957e8f086 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -8,7 +8,7 @@ on:
       - "**.md"
   pull_request:
     branches:
-      - master
+      - "**"
     paths-ignore:
       - "docs/**"
       - "**.md"
@@ -24,17 +24,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        command:
-          [
+        command: [
             # metadata-ingestion and airflow-plugin each have dedicated build jobs
             "except_metadata_ingestion",
             "frontend"
           ]
-        timezone:
-          [
-            "UTC",
-            "America/New_York",
-          ]
+        timezone: ["UTC", "America/New_York"]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
index 841a9ed5f9bc7..9a17a70e7f8d4 100644
--- a/.github/workflows/check-datahub-jars.yml
+++ b/.github/workflows/check-datahub-jars.yml
@@ -10,7 +10,7 @@ on:
       - "**.md"
   pull_request:
     branches:
-      - master
+      - "**"
     paths-ignore:
       - "docker/**"
       - "docs/**"
@@ -28,12 +28,7 @@ jobs:
       max-parallel: 1
       fail-fast: false
       matrix:
-        command:
-          [
-            "datahub-client",
-            "datahub-protobuf",
-            "spark-lineage"
-          ]
+        command: ["datahub-client", "datahub-protobuf", "spark-lineage"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
index a7809087702ac..98e3041f28804 100644
--- a/.github/workflows/close-stale-issues.yml
+++ b/.github/workflows/close-stale-issues.yml
@@ -18,7 +18,9 @@ jobs:
           days-before-issue-stale: 30
           days-before-issue-close: 30
           stale-issue-label: "stale"
-          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity. If you believe this is still an issue on the latest DataHub release please leave a comment with the version that you tested it with. If this is a question/discussion please head to https://slack.datahubproject.io. For feature requests please use https://feature-requests.datahubproject.io"
+          stale-issue-message:
+            "This issue is stale because it has been open for 30 days with no activity. If you believe this is still an issue on the latest DataHub release please leave a comment with the version that you tested it with. If this is a question/discussion please head to https://slack.datahubproject.io.\
+            \ For feature requests please use https://feature-requests.datahubproject.io"
           close-issue-message: "This issue was closed because it has been inactive for 30 days since being marked as stale."
           days-before-pr-stale: -1
           days-before-pr-close: -1
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 6ce19a5b4616e..e12971b8a6208 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -10,7 +10,7 @@ on:
       - ".github/workflows/code-checks.yml"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
       - "metadata-io/**"
       - "datahub-web-react/**"
@@ -21,17 +21,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-
 jobs:
   code_check:
     strategy:
       fail-fast: false
       matrix:
-        command:
-          [
-            "check_event_type.py",
-            "check_policies.py"
-          ]
+        command: ["check_event_type.py", "check_policies.py"]
     name: run code checks
     runs-on: ubuntu-latest
     steps:
@@ -43,5 +38,5 @@ jobs:
         with:
           python-version: "3.10"
       - name: run check ${{ matrix.command }}
-        run: |
-          python .github/scripts/${{ matrix.command }}
\ No newline at end of file
+        run: |-
+          python .github/scripts/${{ matrix.command }}
diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
index a5d421d4b7ff5..fda4349f90bf7 100644
--- a/.github/workflows/docker-postgres-setup.yml
+++ b/.github/workflows/docker-postgres-setup.yml
@@ -8,7 +8,7 @@ on:
       - ".github/workflows/docker-postgres-setup.yml"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
       - "docker/postgres-setup/**"
       - ".github/workflows/docker-postgres-setup.yml"
@@ -61,4 +61,3 @@ jobs:
           context: .
           file: ./docker/postgres-setup/Dockerfile
           platforms: linux/amd64,linux/arm64
-
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index 2aae6bf51529d..8666a5e2e2171 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -8,7 +8,7 @@ on:
       - "**.md"
   pull_request:
     branches:
-      - master
+      - "**"
     paths-ignore:
       - "docs/**"
       - "**.md"
@@ -545,7 +545,6 @@ jobs:
         id: tag
         run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT
 
-
   datahub_ingestion_slim_build:
     name: Build and Push DataHub Ingestion Docker Images
     runs-on: ubuntu-latest
@@ -809,8 +808,8 @@ jobs:
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }}
-          ACTIONS_EXTRA_PACKAGES: 'acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5'
-          ACTIONS_CONFIG: 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
+          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
+          ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |
           ./smoke-test/run-quickstart.sh
       - name: sleep 60s
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
index 68432a4feb13d..ebe2990f3a3cd 100644
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@@ -3,7 +3,7 @@ name: documentation
 on:
   pull_request:
     branches:
-      - master
+      - "**"
   push:
     branches:
       - master
diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml
index b285e46da4857..6f34bf292bf51 100644
--- a/.github/workflows/lint-actions.yml
+++ b/.github/workflows/lint-actions.yml
@@ -2,8 +2,10 @@ name: Lint actions
 on:
   pull_request:
     paths:
-      - '.github/workflows/**'
+      - ".github/workflows/**"
 
+    branches:
+      - "**"
 jobs:
   actionlint:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
index dea4603868f8e..699ca330ce0ac 100644
--- a/.github/workflows/metadata-ingestion.yml
+++ b/.github/workflows/metadata-ingestion.yml
@@ -9,7 +9,7 @@ on:
       - "metadata-models/**"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
       - ".github/**"
       - "metadata-ingestion/**"
diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
index e37ddd0ce4e86..48f230ce14c8d 100644
--- a/.github/workflows/metadata-io.yml
+++ b/.github/workflows/metadata-io.yml
@@ -10,7 +10,7 @@ on:
       - "metadata-io/**"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
       - "**/*.gradle"
       - "li-utils/**"
diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
index b2482602e7548..541b2019b93ef 100644
--- a/.github/workflows/spark-smoke-test.yml
+++ b/.github/workflows/spark-smoke-test.yml
@@ -12,7 +12,7 @@ on:
       - ".github/workflows/spark-smoke-test.yml"
   pull_request:
     branches:
-      - master
+      - "**"
     paths:
       - "metadata_models/**"
       - "metadata-integration/java/datahub-client/**"

From 3cede10ab30e22dcad286bd42bcd154732e40942 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 5 Oct 2023 13:29:47 -0400
Subject: [PATCH 012/259] feat(ingest/dbt): support `use_compiled_code` and
 `test_warnings_are_errors` (#8956)

---
 .../datahub/configuration/source_common.py    |  2 +-
 ...ation.py => validate_field_deprecation.py} | 14 +++++--
 .../ingestion/source/dbt/dbt_common.py        | 41 ++++++++++++++-----
 .../src/datahub/ingestion/source/file.py      |  2 +-
 .../ingestion/source/powerbi/config.py        |  2 +-
 .../ingestion/source/redshift/config.py       |  2 +-
 .../src/datahub/ingestion/source/s3/config.py |  2 +-
 .../ingestion/source/sql/clickhouse.py        |  2 +-
 .../ingestion/source/sql/sql_config.py        |  2 +-
 .../src/datahub/ingestion/source/tableau.py   |  2 +-
 .../tests/unit/test_pydantic_validators.py    |  2 +-
 11 files changed, 51 insertions(+), 22 deletions(-)
 rename metadata-ingestion/src/datahub/configuration/{pydantic_field_deprecation.py => validate_field_deprecation.py} (74%)

diff --git a/metadata-ingestion/src/datahub/configuration/source_common.py b/metadata-ingestion/src/datahub/configuration/source_common.py
index 37b93f3e598e1..a9f891ddb7b1e 100644
--- a/metadata-ingestion/src/datahub/configuration/source_common.py
+++ b/metadata-ingestion/src/datahub/configuration/source_common.py
@@ -4,7 +4,7 @@
 from pydantic.fields import Field
 
 from datahub.configuration.common import ConfigModel, ConfigurationError
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.metadata.schema_classes import FabricTypeClass
 
 DEFAULT_ENV = FabricTypeClass.PROD
diff --git a/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py b/metadata-ingestion/src/datahub/configuration/validate_field_deprecation.py
similarity index 74%
rename from metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py
rename to metadata-ingestion/src/datahub/configuration/validate_field_deprecation.py
index ed82acb594ed7..6134c4dab4817 100644
--- a/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py
+++ b/metadata-ingestion/src/datahub/configuration/validate_field_deprecation.py
@@ -1,20 +1,28 @@
 import warnings
-from typing import Optional, Type
+from typing import Any, Optional, Type
 
 import pydantic
 
 from datahub.configuration.common import ConfigurationWarning
 from datahub.utilities.global_warning_util import add_global_warning
 
+_unset = object()
 
-def pydantic_field_deprecated(field: str, message: Optional[str] = None) -> classmethod:
+
+def pydantic_field_deprecated(
+    field: str,
+    warn_if_value_is_not: Any = _unset,
+    message: Optional[str] = None,
+) -> classmethod:
     if message:
         output = message
     else:
         output = f"{field} is deprecated and will be removed in a future release. Please remove it from your config."
 
     def _validate_deprecated(cls: Type, values: dict) -> dict:
-        if field in values:
+        if field in values and (
+            warn_if_value_is_not is _unset or values[field] != warn_if_value_is_not
+        ):
             add_global_warning(output)
             warnings.warn(output, ConfigurationWarning, stacklevel=2)
         return values
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index f9b71892975b4..0f5c08eb6ac54 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -18,8 +18,8 @@
     ConfigurationError,
     LineageConfig,
 )
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
@@ -214,7 +214,9 @@ class DBTCommonConfig(
         default=False,
         description="Use model identifier instead of model name if defined (if not, default to model name).",
     )
-    _deprecate_use_identifiers = pydantic_field_deprecated("use_identifiers")
+    _deprecate_use_identifiers = pydantic_field_deprecated(
+        "use_identifiers", warn_if_value_is_not=False
+    )
 
     entities_enabled: DBTEntitiesEnabled = Field(
         DBTEntitiesEnabled(),
@@ -278,6 +280,14 @@ class DBTCommonConfig(
         description="When enabled, converts column URNs to lowercase to ensure cross-platform compatibility. "
         "If `target_platform` is Snowflake, the default is True.",
     )
+    use_compiled_code: bool = Field(
+        default=False,
+        description="When enabled, uses the compiled dbt code instead of the raw dbt node definition.",
+    )
+    test_warnings_are_errors: bool = Field(
+        default=False,
+        description="When enabled, dbt test warnings will be treated as failures.",
+    )
 
     @validator("target_platform")
     def validate_target_platform_value(cls, target_platform: str) -> str:
@@ -811,7 +821,7 @@ def _make_assertion_from_test(
                         mce_builder.make_schema_field_urn(upstream_urn, column_name)
                     ],
                     nativeType=node.name,
-                    logic=node.compiled_code if node.compiled_code else node.raw_code,
+                    logic=node.compiled_code or node.raw_code,
                     aggregation=AssertionStdAggregationClass._NATIVE_,
                     nativeParameters=string_map(kw_args),
                 ),
@@ -825,7 +835,7 @@ def _make_assertion_from_test(
                     dataset=upstream_urn,
                     scope=DatasetAssertionScopeClass.DATASET_ROWS,
                     operator=AssertionStdOperatorClass._NATIVE_,
-                    logic=node.compiled_code if node.compiled_code else node.raw_code,
+                    logic=node.compiled_code or node.raw_code,
                     nativeType=node.name,
                     aggregation=AssertionStdAggregationClass._NATIVE_,
                     nativeParameters=string_map(kw_args),
@@ -856,6 +866,10 @@ def _make_assertion_result_from_test(
             result=AssertionResultClass(
                 type=AssertionResultTypeClass.SUCCESS
                 if test_result.status == "pass"
+                or (
+                    not self.config.test_warnings_are_errors
+                    and test_result.status == "warn"
+                )
                 else AssertionResultTypeClass.FAILURE,
                 nativeResults=test_result.native_results,
             ),
@@ -1007,8 +1021,8 @@ def create_platform_mces(
                     aspects.append(upstream_lineage_class)
 
                 # add view properties aspect
-                if node.raw_code and node.language == "sql":
-                    view_prop_aspect = self._create_view_properties_aspect(node)
+                view_prop_aspect = self._create_view_properties_aspect(node)
+                if view_prop_aspect:
                     aspects.append(view_prop_aspect)
 
                 # emit subtype mcp
@@ -1133,14 +1147,21 @@ def _create_dataset_properties_aspect(
     def get_external_url(self, node: DBTNode) -> Optional[str]:
         pass
 
-    def _create_view_properties_aspect(self, node: DBTNode) -> ViewPropertiesClass:
+    def _create_view_properties_aspect(
+        self, node: DBTNode
+    ) -> Optional[ViewPropertiesClass]:
+        view_logic = (
+            node.compiled_code if self.config.use_compiled_code else node.raw_code
+        )
+
+        if node.language != "sql" or not view_logic:
+            return None
+
         materialized = node.materialization in {"table", "incremental", "snapshot"}
-        # this function is only called when raw sql is present. assert is added to satisfy lint checks
-        assert node.raw_code is not None
         view_properties = ViewPropertiesClass(
             materialized=materialized,
             viewLanguage="SQL",
-            viewLogic=node.raw_code,
+            viewLogic=view_logic,
         )
         return view_properties
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/file.py b/metadata-ingestion/src/datahub/ingestion/source/file.py
index de61fa8481c58..590aa59f7b5b6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/file.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/file.py
@@ -16,7 +16,7 @@
 from pydantic.fields import Field
 
 from datahub.configuration.common import ConfigEnum, ConfigModel, ConfigurationError
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
index a8c7e48f3785c..96729f4c60c6c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
@@ -9,8 +9,8 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.source.common.subtypes import BIAssetSubTypes
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalSourceReport,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
index 93850607e551e..804a14b0fe1cf 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -7,8 +7,8 @@
 
 from datahub.configuration import ConfigModel
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import DatasetLineageProviderConfigBase
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.source.data_lake_common.path_spec import PathSpec
 from datahub.ingestion.source.sql.postgres import BasePostgresConfig
 from datahub.ingestion.source.state.stateful_ingestion_base import (
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
index f1dd622efb746..9b5296f0b9dd5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
@@ -5,8 +5,8 @@
 from pydantic.fields import Field
 
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.data_lake_common.config import PathSpecsConfigMixin
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
index 1626f86b92545..8873038079bad 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
@@ -19,9 +19,9 @@
 from sqlalchemy.types import BOOLEAN, DATE, DATETIME, INTEGER
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import DatasetLineageProviderConfigBase
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.decorators import (
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 8f1e04b915f3b..677d32c8bac08 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -7,8 +7,8 @@
 from pydantic import Field
 
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulStaleMetadataRemovalConfig,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
index 6214cba342622..e347cd26d245a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
@@ -37,11 +37,11 @@
     ConfigModel,
     ConfigurationError,
 )
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.source_common import (
     DatasetLineageProviderConfigBase,
     DatasetSourceConfigMixin,
 )
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import (
     ContainerKey,
diff --git a/metadata-ingestion/tests/unit/test_pydantic_validators.py b/metadata-ingestion/tests/unit/test_pydantic_validators.py
index 07d86043a35bf..3e9ec6cbaf357 100644
--- a/metadata-ingestion/tests/unit/test_pydantic_validators.py
+++ b/metadata-ingestion/tests/unit/test_pydantic_validators.py
@@ -4,7 +4,7 @@
 from pydantic import ValidationError
 
 from datahub.configuration.common import ConfigModel
-from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated
+from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.utilities.global_warning_util import get_global_warnings

From debac3cf5c31b471a5a82da8d18fb8303cc8b9d0 Mon Sep 17 00:00:00 2001
From: Patrick Franco Braz <patrickfbraz@poli.ufrj.br>
Date: Thu, 5 Oct 2023 17:47:10 -0300
Subject: [PATCH 013/259] refactor(boot): increases wait timeout for servlets
 initialization (#8947)

Co-authored-by: RyanHolstien <RyanHolstien@users.noreply.github.com>
---
 .../configuration/src/main/resources/application.yml     | 3 +++
 .../metadata/boot/OnBootApplicationListener.java         | 9 +++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 4be31b2b6bb15..4dfd96ac75c6c 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -276,6 +276,9 @@ bootstrap:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
   backfillBrowsePathsV2:
     enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
+  servlets:
+    waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize
+
 
 systemUpdate:
   initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000}
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java
index 980cafaceae27..032b934a7ba87 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java
@@ -15,15 +15,18 @@
 import org.apache.http.impl.client.HttpClients;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.context.event.EventListener;
 import org.springframework.stereotype.Component;
 import org.springframework.web.context.WebApplicationContext;
+import org.springframework.context.annotation.Configuration;
 
 
 /**
  * Responsible for coordinating starting steps that happen before the application starts up.
  */
+@Configuration
 @Slf4j
 @Component
 public class OnBootApplicationListener {
@@ -44,6 +47,8 @@ public class OnBootApplicationListener {
   @Qualifier("configurationProvider")
   private ConfigurationProvider provider;
 
+  @Value("${bootstrap.servlets.waitTimeout}")
+  private int _servletsWaitTimeout;
 
   @EventListener(ContextRefreshedEvent.class)
   public void onApplicationEvent(@Nonnull ContextRefreshedEvent event) {
@@ -62,7 +67,7 @@ public void onApplicationEvent(@Nonnull ContextRefreshedEvent event) {
   public Runnable isSchemaRegistryAPIServletReady() {
     return () -> {
         final HttpGet request = new HttpGet(provider.getKafka().getSchemaRegistry().getUrl());
-        int timeouts = 30;
+        int timeouts = _servletsWaitTimeout;
         boolean openAPIServeletReady = false;
         while (!openAPIServeletReady && timeouts > 0) {
           try {
@@ -79,7 +84,7 @@ public Runnable isSchemaRegistryAPIServletReady() {
           timeouts--;
         }
         if (!openAPIServeletReady) {
-          log.error("Failed to bootstrap DataHub, OpenAPI servlet was not ready after 30 seconds");
+          log.error("Failed to bootstrap DataHub, OpenAPI servlet was not ready after {} seconds", timeouts);
           System.exit(1);
         } else {
         _bootstrapManager.start();

From 26bc039b967d3a62a7079522b702e97ed8ad8d27 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Thu, 5 Oct 2023 23:23:15 -0400
Subject: [PATCH 014/259] fix(ingest/unity): Remove metastore from ingestion
 and urns; standardize platform instance; add notebook filter (#8943)

---
 docs/how/updating-datahub.md                  |  5 +
 .../src/datahub/emitter/mcp_builder.py        | 10 +-
 .../datahub/ingestion/source/unity/config.py  | 45 ++++++++-
 .../datahub/ingestion/source/unity/proxy.py   | 16 +--
 .../ingestion/source/unity/proxy_types.py     | 19 ++--
 .../datahub/ingestion/source/unity/source.py  | 99 ++++++++++++-------
 6 files changed, 145 insertions(+), 49 deletions(-)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 4df8d435cf1c4..5d0ad5eaf8f7e 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -9,6 +9,11 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 - #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
 - #8853 - The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7. See the docs for more details.
 - #8853 - Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include `pip install 'acryl-datahub-airflow-plugin[plugin-v2]'`. To continue using the v1 plugin, set the `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN` environment variable to `true`.
+- #8943 The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
+This is currently enabled by default to preserve compatibility, but will be disabled by default and then removed in the future.
+If stateful ingestion is enabled, simply setting `include_metastore: false` will perform all required cleanup.
+Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
+`datahub delete --platform databricks --soft` and then reingesting with `include_metastore: false`.
 
 ### Potential Downtime
 
diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
index 06f689dfd317b..65e0c0d6ba60d 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -94,7 +94,15 @@ class MetastoreKey(ContainerKey):
     metastore: str
 
 
-class CatalogKey(MetastoreKey):
+class CatalogKeyWithMetastore(MetastoreKey):
+    catalog: str
+
+
+class UnitySchemaKeyWithMetastore(CatalogKeyWithMetastore):
+    unity_schema: str
+
+
+class CatalogKey(ContainerKey):
     catalog: str
 
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index a49c789a82f27..f259fa260f653 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -1,3 +1,4 @@
+import logging
 import os
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Optional
@@ -21,6 +22,9 @@
     OperationConfig,
     is_profiling_enabled,
 )
+from datahub.utilities.global_warning_util import add_global_warning
+
+logger = logging.getLogger(__name__)
 
 
 class UnityCatalogProfilerConfig(ConfigModel):
@@ -97,9 +101,25 @@ class UnityCatalogSourceConfig(
         description="Name of the workspace. Default to deployment name present in workspace_url",
     )
 
+    include_metastore: bool = pydantic.Field(
+        default=True,
+        description=(
+            "Whether to ingest the workspace's metastore as a container and include it in all urns."
+            " Changing this will affect the urns of all entities in the workspace."
+            " This will be disabled by default in the future,"
+            " so it is recommended to set this to `False` for new ingestions."
+            " If you have an existing unity catalog ingestion, you'll want to avoid duplicates by soft deleting existing data."
+            " If stateful ingestion is enabled, running with `include_metastore: false` should be sufficient."
+            " Otherwise, we recommend deleting via the cli: `datahub delete --platform databricks` and re-ingesting with `include_metastore: false`."
+        ),
+    )
+
     ingest_data_platform_instance_aspect: Optional[bool] = pydantic.Field(
         default=False,
-        description="Option to enable/disable ingestion of the data platform instance aspect. The default data platform instance id for a dataset is workspace_name",
+        description=(
+            "Option to enable/disable ingestion of the data platform instance aspect."
+            " The default data platform instance id for a dataset is workspace_name"
+        ),
     )
 
     _only_ingest_assigned_metastore_removed = pydantic_removed_field(
@@ -122,6 +142,16 @@ class UnityCatalogSourceConfig(
         default=AllowDenyPattern.allow_all(),
         description="Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in `catalog.schema.table` format. e.g. to match all tables starting with customer in Customer catalog and public schema, use the regex `Customer\\.public\\.customer.*`.",
     )
+
+    notebook_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description=(
+            "Regex patterns for notebooks to filter in ingestion, based on notebook *path*."
+            " Specify regex to match the entire notebook path in `/<dir>/.../<name>` format."
+            " e.g. to match all notebooks in the root Shared directory, use the regex `/Shared/.*`."
+        ),
+    )
+
     domain: Dict[str, AllowDenyPattern] = Field(
         default=dict(),
         description='Attach domains to catalogs, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like "Marketing".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.',
@@ -182,3 +212,16 @@ def workspace_url_should_start_with_http_scheme(cls, workspace_url: str) -> str:
                 "Workspace URL must start with http scheme. e.g. https://my-workspace.cloud.databricks.com"
             )
         return workspace_url
+
+    @pydantic.validator("include_metastore")
+    def include_metastore_warning(cls, v: bool) -> bool:
+        if v:
+            msg = (
+                "`include_metastore` is enabled."
+                " This is not recommended and will be disabled by default in the future, which is a breaking change."
+                " All databricks urns will change if you re-ingest with this disabled."
+                " We recommend soft deleting all databricks data and re-ingesting with `include_metastore` set to `False`."
+            )
+            logger.warning(msg)
+            add_global_warning(msg)
+        return v
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
index 2401f1c3d163c..529d9e7b563a5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
@@ -97,14 +97,13 @@ def __init__(
         self.report = report
 
     def check_basic_connectivity(self) -> bool:
-        self._workspace_client.metastores.summary()
-        return True
+        return bool(self._workspace_client.catalogs.list())
 
     def assigned_metastore(self) -> Metastore:
         response = self._workspace_client.metastores.summary()
         return self._create_metastore(response)
 
-    def catalogs(self, metastore: Metastore) -> Iterable[Catalog]:
+    def catalogs(self, metastore: Optional[Metastore]) -> Iterable[Catalog]:
         response = self._workspace_client.catalogs.list()
         if not response:
             logger.info("Catalogs not found")
@@ -247,7 +246,7 @@ def table_lineage(
             for item in response.get("upstreams") or []:
                 if "tableInfo" in item:
                     table_ref = TableReference.create_from_lineage(
-                        item["tableInfo"], table.schema.catalog.metastore.id
+                        item["tableInfo"], table.schema.catalog.metastore
                     )
                     if table_ref:
                         table.upstreams[table_ref] = {}
@@ -276,7 +275,7 @@ def get_column_lineage(self, table: Table, include_entity_lineage: bool) -> None
                     )
                     for item in response.get("upstream_cols", []):
                         table_ref = TableReference.create_from_lineage(
-                            item, table.schema.catalog.metastore.id
+                            item, table.schema.catalog.metastore
                         )
                         if table_ref:
                             table.upstreams.setdefault(table_ref, {}).setdefault(
@@ -305,10 +304,13 @@ def _create_metastore(
             comment=None,
         )
 
-    def _create_catalog(self, metastore: Metastore, obj: CatalogInfo) -> Catalog:
+    def _create_catalog(
+        self, metastore: Optional[Metastore], obj: CatalogInfo
+    ) -> Catalog:
+        catalog_name = self._escape_sequence(obj.name)
         return Catalog(
             name=obj.name,
-            id=f"{metastore.id}.{self._escape_sequence(obj.name)}",
+            id=f"{metastore.id}.{catalog_name}" if metastore else catalog_name,
             metastore=metastore,
             comment=obj.comment,
             owner=obj.owner,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
index 54ac2e90d7c7e..18ac2475b51e0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
@@ -92,7 +92,7 @@ class Metastore(CommonProperty):
 
 @dataclass
 class Catalog(CommonProperty):
-    metastore: Metastore
+    metastore: Optional[Metastore]
     owner: Optional[str]
     type: CatalogType
 
@@ -130,7 +130,7 @@ class ServicePrincipal:
 
 @dataclass(frozen=True, order=True)
 class TableReference:
-    metastore: str
+    metastore: Optional[str]
     catalog: str
     schema: str
     table: str
@@ -138,17 +138,21 @@ class TableReference:
     @classmethod
     def create(cls, table: "Table") -> "TableReference":
         return cls(
-            table.schema.catalog.metastore.id,
+            table.schema.catalog.metastore.id
+            if table.schema.catalog.metastore
+            else None,
             table.schema.catalog.name,
             table.schema.name,
             table.name,
         )
 
     @classmethod
-    def create_from_lineage(cls, d: dict, metastore: str) -> Optional["TableReference"]:
+    def create_from_lineage(
+        cls, d: dict, metastore: Optional[Metastore]
+    ) -> Optional["TableReference"]:
         try:
             return cls(
-                metastore,
+                metastore.id if metastore else None,
                 d["catalog_name"],
                 d["schema_name"],
                 d.get("table_name", d["name"]),  # column vs table query output
@@ -158,7 +162,10 @@ def create_from_lineage(cls, d: dict, metastore: str) -> Optional["TableReferenc
             return None
 
     def __str__(self) -> str:
-        return f"{self.metastore}.{self.catalog}.{self.schema}.{self.table}"
+        if self.metastore:
+            return f"{self.metastore}.{self.catalog}.{self.schema}.{self.table}"
+        else:
+            return self.qualified_table_name
 
     @property
     def qualified_table_name(self) -> str:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
index f2da1aece9fd4..4f7866aee7681 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
@@ -16,10 +16,12 @@
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import (
     CatalogKey,
+    CatalogKeyWithMetastore,
     ContainerKey,
     MetastoreKey,
     NotebookKey,
     UnitySchemaKey,
+    UnitySchemaKeyWithMetastore,
     add_dataset_to_container,
     gen_containers,
 )
@@ -127,7 +129,7 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
     config: UnityCatalogSourceConfig
     unity_catalog_api_proxy: UnityCatalogApiProxy
     platform: str = "databricks"
-    platform_instance_name: str
+    platform_instance_name: Optional[str]
 
     def get_report(self) -> UnityCatalogReport:
         return self.report
@@ -146,11 +148,13 @@ def __init__(self, ctx: PipelineContext, config: UnityCatalogSourceConfig):
         self.external_url_base = urljoin(self.config.workspace_url, "/explore/data")
 
         # Determine the platform_instance_name
-        self.platform_instance_name = (
-            config.workspace_name
-            if config.workspace_name is not None
-            else config.workspace_url.split("//")[1].split(".")[0]
-        )
+        self.platform_instance_name = self.config.platform_instance
+        if self.config.include_metastore:
+            self.platform_instance_name = (
+                config.workspace_name
+                if config.workspace_name is not None
+                else config.workspace_url.split("//")[1].split(".")[0]
+            )
 
         if self.config.domain:
             self.domain_registry = DomainRegistry(
@@ -247,10 +251,14 @@ def build_service_principal_map(self) -> None:
 
     def process_notebooks(self) -> Iterable[MetadataWorkUnit]:
         for notebook in self.unity_catalog_api_proxy.workspace_notebooks():
+            if not self.config.notebook_pattern.allowed(notebook.path):
+                self.report.notebooks.dropped(notebook.path)
+                continue
+
             self.notebooks[str(notebook.id)] = notebook
-            yield from self._gen_notebook_aspects(notebook)
+            yield from self._gen_notebook_workunits(notebook)
 
-    def _gen_notebook_aspects(self, notebook: Notebook) -> Iterable[MetadataWorkUnit]:
+    def _gen_notebook_workunits(self, notebook: Notebook) -> Iterable[MetadataWorkUnit]:
         mcps = MetadataChangeProposalWrapper.construct_many(
             entityUrn=self.gen_notebook_urn(notebook),
             aspects=[
@@ -270,7 +278,7 @@ def _gen_notebook_aspects(self, notebook: Notebook) -> Iterable[MetadataWorkUnit
                 ),
                 SubTypesClass(typeNames=[DatasetSubTypes.NOTEBOOK]),
                 BrowsePathsClass(paths=notebook.path.split("/")),
-                # TODO: Add DPI aspect
+                self._create_data_platform_instance_aspect(),
             ],
         )
         for mcp in mcps:
@@ -296,13 +304,17 @@ def _gen_notebook_lineage(self, notebook: Notebook) -> Optional[MetadataWorkUnit
         ).as_workunit()
 
     def process_metastores(self) -> Iterable[MetadataWorkUnit]:
-        metastore = self.unity_catalog_api_proxy.assigned_metastore()
-        yield from self.gen_metastore_containers(metastore)
+        metastore: Optional[Metastore] = None
+        if self.config.include_metastore:
+            metastore = self.unity_catalog_api_proxy.assigned_metastore()
+            yield from self.gen_metastore_containers(metastore)
         yield from self.process_catalogs(metastore)
+        if metastore and self.config.include_metastore:
+            self.report.metastores.processed(metastore.id)
 
-        self.report.metastores.processed(metastore.id)
-
-    def process_catalogs(self, metastore: Metastore) -> Iterable[MetadataWorkUnit]:
+    def process_catalogs(
+        self, metastore: Optional[Metastore]
+    ) -> Iterable[MetadataWorkUnit]:
         for catalog in self.unity_catalog_api_proxy.catalogs(metastore=metastore):
             if not self.config.catalog_pattern.allowed(catalog.id):
                 self.report.catalogs.dropped(catalog.id)
@@ -353,7 +365,7 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn
         operation = self._create_table_operation_aspect(table)
         domain = self._get_domain_aspect(dataset_name=table.ref.qualified_table_name)
         ownership = self._create_table_ownership_aspect(table)
-        data_platform_instance = self._create_data_platform_instance_aspect(table)
+        data_platform_instance = self._create_data_platform_instance_aspect()
 
         if self.config.include_column_lineage:
             self.unity_catalog_api_proxy.get_column_lineage(
@@ -503,27 +515,37 @@ def gen_metastore_containers(
     def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]:
         domain_urn = self._gen_domain_urn(catalog.name)
 
-        metastore_container_key = self.gen_metastore_key(catalog.metastore)
         catalog_container_key = self.gen_catalog_key(catalog)
         yield from gen_containers(
             container_key=catalog_container_key,
             name=catalog.name,
             sub_types=[DatasetContainerSubTypes.CATALOG],
             domain_urn=domain_urn,
-            parent_container_key=metastore_container_key,
+            parent_container_key=self.gen_metastore_key(catalog.metastore)
+            if self.config.include_metastore and catalog.metastore
+            else None,
             description=catalog.comment,
             owner_urn=self.get_owner_urn(catalog.owner),
             external_url=f"{self.external_url_base}/{catalog.name}",
         )
 
     def gen_schema_key(self, schema: Schema) -> ContainerKey:
-        return UnitySchemaKey(
-            unity_schema=schema.name,
-            platform=self.platform,
-            instance=self.config.platform_instance,
-            catalog=schema.catalog.name,
-            metastore=schema.catalog.metastore.name,
-        )
+        if self.config.include_metastore:
+            assert schema.catalog.metastore
+            return UnitySchemaKeyWithMetastore(
+                unity_schema=schema.name,
+                platform=self.platform,
+                instance=self.config.platform_instance,
+                catalog=schema.catalog.name,
+                metastore=schema.catalog.metastore.name,
+            )
+        else:
+            return UnitySchemaKey(
+                unity_schema=schema.name,
+                platform=self.platform,
+                instance=self.config.platform_instance,
+                catalog=schema.catalog.name,
+            )
 
     def gen_metastore_key(self, metastore: Metastore) -> MetastoreKey:
         return MetastoreKey(
@@ -532,13 +554,21 @@ def gen_metastore_key(self, metastore: Metastore) -> MetastoreKey:
             instance=self.config.platform_instance,
         )
 
-    def gen_catalog_key(self, catalog: Catalog) -> CatalogKey:
-        return CatalogKey(
-            catalog=catalog.name,
-            metastore=catalog.metastore.name,
-            platform=self.platform,
-            instance=self.config.platform_instance,
-        )
+    def gen_catalog_key(self, catalog: Catalog) -> ContainerKey:
+        if self.config.include_metastore:
+            assert catalog.metastore
+            return CatalogKeyWithMetastore(
+                catalog=catalog.name,
+                metastore=catalog.metastore.name,
+                platform=self.platform,
+                instance=self.config.platform_instance,
+            )
+        else:
+            return CatalogKey(
+                catalog=catalog.name,
+                platform=self.platform,
+                instance=self.config.platform_instance,
+            )
 
     def _gen_domain_urn(self, dataset_name: str) -> Optional[str]:
         domain_urn: Optional[str] = None
@@ -643,15 +673,16 @@ def _create_table_ownership_aspect(self, table: Table) -> Optional[OwnershipClas
         return None
 
     def _create_data_platform_instance_aspect(
-        self, table: Table
+        self,
     ) -> Optional[DataPlatformInstanceClass]:
-        # Only ingest the DPI aspect if the flag is true
         if self.config.ingest_data_platform_instance_aspect:
             return DataPlatformInstanceClass(
                 platform=make_data_platform_urn(self.platform),
                 instance=make_dataplatform_instance_urn(
                     self.platform, self.platform_instance_name
-                ),
+                )
+                if self.platform_instance_name
+                else None,
             )
         return None
 

From ea87febd2bdf0aebf603532be9448e6435f1fea9 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Fri, 6 Oct 2023 14:36:32 +0900
Subject: [PATCH 015/259] fix: add retry for fetch_url (#8958)

---
 docs-website/download_historical_versions.py | 34 ++++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/docs-website/download_historical_versions.py b/docs-website/download_historical_versions.py
index 83157edc1972c..53ee9cf1e63ef 100644
--- a/docs-website/download_historical_versions.py
+++ b/docs-website/download_historical_versions.py
@@ -1,6 +1,7 @@
 import json
 import os
 import tarfile
+import time
 import urllib.request
 
 repo_url = "https://api.github.com/repos/datahub-project/static-assets"
@@ -16,17 +17,30 @@ def download_file(url, destination):
                 f.write(chunk)
 
 
-def fetch_urls(repo_url: str, folder_path: str, file_format: str):
+def fetch_urls(
+    repo_url: str, folder_path: str, file_format: str, max_retries=3, retry_delay=5
+):
     api_url = f"{repo_url}/contents/{folder_path}"
-    response = urllib.request.urlopen(api_url)
-    data = response.read().decode("utf-8")
-    urls = [
-        file["download_url"]
-        for file in json.loads(data)
-        if file["name"].endswith(file_format)
-    ]
-    print(urls)
-    return urls
+    for attempt in range(max_retries + 1):
+        try:
+            response = urllib.request.urlopen(api_url)
+            if response.status == 403 or (500 <= response.status < 600):
+                raise Exception(f"HTTP Error {response.status}: {response.reason}")
+            data = response.read().decode("utf-8")
+            urls = [
+                file["download_url"]
+                for file in json.loads(data)
+                if file["name"].endswith(file_format)
+            ]
+            print(urls)
+            return urls
+        except Exception as e:
+            if attempt < max_retries:
+                print(f"Attempt {attempt + 1}/{max_retries}: {e}")
+                time.sleep(retry_delay)
+            else:
+                print(f"Max retries reached. Unable to fetch data.")
+                raise
 
 
 def extract_tar_file(destination_path):

From c80da8f949aea340af73c992ff6d2bd129eb55fe Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Fri, 6 Oct 2023 10:06:36 -0400
Subject: [PATCH 016/259] feat(ingest/unity): Use ThreadPoolExecutor for CLL
 (#8952)

---
 .../datahub/ingestion/source/unity/config.py  | 11 +++++
 .../datahub/ingestion/source/unity/proxy.py   | 46 ++++++++-----------
 .../datahub/ingestion/source/unity/report.py  |  2 +
 .../datahub/ingestion/source/unity/source.py  | 33 +++++++++----
 4 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index f259fa260f653..51390873712d3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -181,6 +181,17 @@ class UnityCatalogSourceConfig(
         description="Option to enable/disable lineage generation. Currently we have to call a rest call per column to get column level lineage due to the Databrick api which can slow down ingestion. ",
     )
 
+    column_lineage_column_limit: int = pydantic.Field(
+        default=300,
+        description="Limit the number of columns to get column level lineage. ",
+    )
+
+    lineage_max_workers: int = pydantic.Field(
+        default=5 * (os.cpu_count() or 4),
+        description="Number of worker threads to use for column lineage thread pool executor. Set to 1 to disable.",
+        hidden_from_docs=True,
+    )
+
     include_usage_statistics: bool = Field(
         default=True,
         description="Generate usage statistics.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
index 529d9e7b563a5..9bcdb200f180e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
@@ -233,9 +233,7 @@ def list_lineages_by_column(self, table_name: str, column_name: str) -> dict:
             body={"table_name": table_name, "column_name": column_name},
         )
 
-    def table_lineage(
-        self, table: Table, include_entity_lineage: bool
-    ) -> Optional[dict]:
+    def table_lineage(self, table: Table, include_entity_lineage: bool) -> None:
         # Lineage endpoint doesn't exists on 2.1 version
         try:
             response: dict = self.list_lineages_by_table(
@@ -256,34 +254,30 @@ def table_lineage(
             for item in response.get("downstreams") or []:
                 for notebook in item.get("notebookInfos") or []:
                     table.downstream_notebooks.add(notebook["notebook_id"])
-
-            return response
         except Exception as e:
-            logger.error(f"Error getting lineage: {e}")
-            return None
+            logger.warning(
+                f"Error getting lineage on table {table.ref}: {e}", exc_info=True
+            )
 
-    def get_column_lineage(self, table: Table, include_entity_lineage: bool) -> None:
+    def get_column_lineage(self, table: Table, column_name: str) -> None:
         try:
-            table_lineage = self.table_lineage(
-                table, include_entity_lineage=include_entity_lineage
+            response: dict = self.list_lineages_by_column(
+                table_name=table.ref.qualified_table_name,
+                column_name=column_name,
             )
-            if table_lineage:
-                for column in table.columns:
-                    response: dict = self.list_lineages_by_column(
-                        table_name=table.ref.qualified_table_name,
-                        column_name=column.name,
-                    )
-                    for item in response.get("upstream_cols", []):
-                        table_ref = TableReference.create_from_lineage(
-                            item, table.schema.catalog.metastore
-                        )
-                        if table_ref:
-                            table.upstreams.setdefault(table_ref, {}).setdefault(
-                                column.name, []
-                            ).append(item["name"])
-
+            for item in response.get("upstream_cols") or []:
+                table_ref = TableReference.create_from_lineage(
+                    item, table.schema.catalog.metastore
+                )
+                if table_ref:
+                    table.upstreams.setdefault(table_ref, {}).setdefault(
+                        column_name, []
+                    ).append(item["name"])
         except Exception as e:
-            logger.error(f"Error getting lineage: {e}")
+            logger.warning(
+                f"Error getting column lineage on table {table.ref}, column {column_name}: {e}",
+                exc_info=True,
+            )
 
     @staticmethod
     def _escape_sequence(value: str) -> str:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
index 808172a136bb3..fa61571fa92cb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
@@ -18,6 +18,8 @@ class UnityCatalogReport(IngestionStageReport, StaleEntityRemovalSourceReport):
     table_profiles: EntityFilterReport = EntityFilterReport.field(type="table profile")
     notebooks: EntityFilterReport = EntityFilterReport.field(type="notebook")
 
+    num_column_lineage_skipped_column_count: int = 0
+
     num_queries: int = 0
     num_queries_dropped_parse_failure: int = 0
     num_queries_missing_table: int = 0  # Can be due to pattern filter
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
index 4f7866aee7681..27c1f341aa84d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
@@ -1,6 +1,7 @@
 import logging
 import re
 import time
+from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
 from typing import Dict, Iterable, List, Optional, Set, Union
 from urllib.parse import urljoin
@@ -367,15 +368,7 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn
         ownership = self._create_table_ownership_aspect(table)
         data_platform_instance = self._create_data_platform_instance_aspect()
 
-        if self.config.include_column_lineage:
-            self.unity_catalog_api_proxy.get_column_lineage(
-                table, include_entity_lineage=self.config.include_notebooks
-            )
-        elif self.config.include_table_lineage:
-            self.unity_catalog_api_proxy.table_lineage(
-                table, include_entity_lineage=self.config.include_notebooks
-            )
-        lineage = self._generate_lineage_aspect(dataset_urn, table)
+        lineage = self.ingest_lineage(table)
 
         if self.config.include_notebooks:
             for notebook_id in table.downstream_notebooks:
@@ -401,6 +394,28 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn
             )
         ]
 
+    def ingest_lineage(self, table: Table) -> Optional[UpstreamLineageClass]:
+        if self.config.include_table_lineage:
+            self.unity_catalog_api_proxy.table_lineage(
+                table, include_entity_lineage=self.config.include_notebooks
+            )
+
+        if self.config.include_column_lineage and table.upstreams:
+            if len(table.columns) > self.config.column_lineage_column_limit:
+                self.report.num_column_lineage_skipped_column_count += 1
+
+            with ThreadPoolExecutor(
+                max_workers=self.config.lineage_max_workers
+            ) as executor:
+                for column in table.columns[: self.config.column_lineage_column_limit]:
+                    executor.submit(
+                        self.unity_catalog_api_proxy.get_column_lineage,
+                        table,
+                        column.name,
+                    )
+
+        return self._generate_lineage_aspect(self.gen_dataset_urn(table.ref), table)
+
     def _generate_lineage_aspect(
         self, dataset_urn: str, table: Table
     ) -> Optional[UpstreamLineageClass]:

From 8e7f286e71b36a07b4fedc0de1807354064a4fa5 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Fri, 6 Oct 2023 20:12:39 +0530
Subject: [PATCH 017/259] feat(ingest/snowflake): support profiling with
 sampling (#8902)

Co-authored-by: Andrew Sikowitz <andrew.sikowitz@acryl.io>
---
 .../ingestion/source/bigquery_v2/profiler.py  | 127 ++++++----------
 .../ingestion/source/ge_data_profiler.py      |  32 +++--
 .../ingestion/source/ge_profiling_config.py   |   4 +-
 .../ingestion/source/redshift/profile.py      |  93 ++----------
 .../source/snowflake/snowflake_profiler.py    | 135 +++++-------------
 .../source/sql/sql_generic_profiler.py        | 105 +++++++++++++-
 6 files changed, 209 insertions(+), 287 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
index b3e88459917b3..8ae17600e0eea 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
@@ -1,12 +1,9 @@
-import dataclasses
 import logging
 from datetime import datetime
 from typing import Dict, Iterable, List, Optional, Tuple, cast
 
 from dateutil.relativedelta import relativedelta
 
-from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
 from datahub.ingestion.source.bigquery_v2.bigquery_config import BigQueryV2Config
@@ -15,7 +12,7 @@
     RANGE_PARTITION_NAME,
     BigqueryTable,
 )
-from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest
+from datahub.ingestion.source.sql.sql_generic import BaseTable
 from datahub.ingestion.source.sql.sql_generic_profiler import (
     GenericProfiler,
     TableProfilerRequest,
@@ -25,12 +22,6 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclasses.dataclass
-class BigqueryProfilerRequest(GEProfilerRequest):
-    table: BigqueryTable
-    profile_table_level_only: bool = False
-
-
 class BigqueryProfiler(GenericProfiler):
     config: BigQueryV2Config
     report: BigQueryV2Report
@@ -183,84 +174,54 @@ def get_workunits(
                     )
 
                 # Emit the profile work unit
-                profile_request = self.get_bigquery_profile_request(
-                    project=project_id, dataset=dataset, table=table
-                )
+                profile_request = self.get_profile_request(table, dataset, project_id)
                 if profile_request is not None:
+                    self.report.report_entity_profiled(profile_request.pretty_name)
                     profile_requests.append(profile_request)
 
         if len(profile_requests) == 0:
             return
-        yield from self.generate_wu_from_profile_requests(profile_requests)
-
-    def generate_wu_from_profile_requests(
-        self, profile_requests: List[BigqueryProfilerRequest]
-    ) -> Iterable[MetadataWorkUnit]:
-        table_profile_requests = cast(List[TableProfilerRequest], profile_requests)
-        for request, profile in self.generate_profiles(
-            table_profile_requests,
+        yield from self.generate_profile_workunits(
+            profile_requests,
             self.config.profiling.max_workers,
             platform=self.platform,
             profiler_args=self.get_profile_args(),
-        ):
-            if request is None or profile is None:
-                continue
-
-            request = cast(BigqueryProfilerRequest, request)
-            profile.sizeInBytes = request.table.size_in_bytes
-            # If table is partitioned we profile only one partition (if nothing set then the last one)
-            # but for table level we can use the rows_count from the table metadata
-            # This way even though column statistics only reflects one partition data but the rows count
-            # shows the proper count.
-            if profile.partitionSpec and profile.partitionSpec.partition:
-                profile.rowCount = request.table.rows_count
-
-            dataset_name = request.pretty_name
-            dataset_urn = make_dataset_urn_with_platform_instance(
-                self.platform,
-                dataset_name,
-                self.config.platform_instance,
-                self.config.env,
-            )
-            # We don't add to the profiler state if we only do table level profiling as it always happens
-            if self.state_handler and not request.profile_table_level_only:
-                self.state_handler.add_to_state(
-                    dataset_urn, int(datetime.now().timestamp() * 1000)
-                )
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn, aspect=profile
-            ).as_workunit()
+        )
 
-    def get_bigquery_profile_request(
-        self, project: str, dataset: str, table: BigqueryTable
-    ) -> Optional[BigqueryProfilerRequest]:
-        skip_profiling = False
-        profile_table_level_only = self.config.profiling.profile_table_level_only
-        dataset_name = BigqueryTableIdentifier(
-            project_id=project, dataset=dataset, table=table.name
+    def get_dataset_name(self, table_name: str, schema_name: str, db_name: str) -> str:
+        return BigqueryTableIdentifier(
+            project_id=db_name, dataset=schema_name, table=table_name
         ).get_table_name()
-        if not self.is_dataset_eligible_for_profiling(
-            dataset_name, table.last_altered, table.size_in_bytes, table.rows_count
-        ):
-            profile_table_level_only = True
-            self.report.num_tables_not_eligible_profiling[f"{project}.{dataset}"] += 1
 
-        if not table.column_count:
-            skip_profiling = True
+    def get_batch_kwargs(
+        self, table: BaseTable, schema_name: str, db_name: str
+    ) -> dict:
+        return dict(
+            schema=db_name,  # <project>
+            table=f"{schema_name}.{table.name}",  # <dataset>.<table>
+        )
 
-        if skip_profiling:
-            if self.config.profiling.report_dropped_profiles:
-                self.report.report_dropped(f"profile of {dataset_name}")
+    def get_profile_request(
+        self, table: BaseTable, schema_name: str, db_name: str
+    ) -> Optional[TableProfilerRequest]:
+        profile_request = super().get_profile_request(table, schema_name, db_name)
+
+        if not profile_request:
             return None
 
+        # Below code handles profiling changes required for partitioned or sharded tables
+        # 1. Skip profile if partition profiling is disabled.
+        # 2. Else update `profile_request.batch_kwargs` with partition and custom_sql
+
+        bq_table = cast(BigqueryTable, table)
         (partition, custom_sql) = self.generate_partition_profiler_query(
-            project, dataset, table, self.config.profiling.partition_datetime
+            db_name, schema_name, bq_table, self.config.profiling.partition_datetime
         )
-        if partition is None and table.partition_info:
+
+        if partition is None and bq_table.partition_info:
             self.report.report_warning(
                 "profile skipped as partitioned table is empty or partition id or type was invalid",
-                dataset_name,
+                profile_request.pretty_name,
             )
             return None
         if (
@@ -268,24 +229,20 @@ def get_bigquery_profile_request(
             and not self.config.profiling.partition_profiling_enabled
         ):
             logger.debug(
-                f"{dataset_name} and partition {partition} is skipped because profiling.partition_profiling_enabled property is disabled"
+                f"{profile_request.pretty_name} and partition {partition} is skipped because profiling.partition_profiling_enabled property is disabled"
             )
             self.report.profiling_skipped_partition_profiling_disabled.append(
-                dataset_name
+                profile_request.pretty_name
             )
             return None
 
-        self.report.report_entity_profiled(dataset_name)
-        logger.debug(f"Preparing profiling request for {dataset_name}")
-        profile_request = BigqueryProfilerRequest(
-            pretty_name=dataset_name,
-            batch_kwargs=dict(
-                schema=project,
-                table=f"{dataset}.{table.name}",
-                custom_sql=custom_sql,
-                partition=partition,
-            ),
-            table=table,
-            profile_table_level_only=profile_table_level_only,
-        )
+        if partition:
+            logger.debug("Updating profiling request for partitioned/sharded tables")
+            profile_request.batch_kwargs.update(
+                dict(
+                    custom_sql=custom_sql,
+                    partition=partition,
+                )
+            )
+
         return profile_request
diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
index 01e083d566168..9f6ac9dd21164 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
@@ -273,6 +273,7 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
     partition: Optional[str]
     config: GEProfilingConfig
     report: SQLSourceReport
+    custom_sql: Optional[str]
 
     query_combiner: SQLAlchemyQueryCombiner
 
@@ -596,16 +597,8 @@ def generate_dataset_profile(  # noqa: C901 (complexity)
             "catch_exceptions", self.config.catch_exceptions
         )
 
-        profile = DatasetProfileClass(timestampMillis=get_sys_time())
-        if self.partition:
-            profile.partitionSpec = PartitionSpecClass(partition=self.partition)
-        elif self.config.limit and self.config.offset:
-            profile.partitionSpec = PartitionSpecClass(
-                type=PartitionTypeClass.QUERY,
-                partition=json.dumps(
-                    dict(limit=self.config.limit, offset=self.config.offset)
-                ),
-            )
+        profile = self.init_profile()
+
         profile.fieldProfiles = []
         self._get_dataset_rows(profile)
 
@@ -740,6 +733,24 @@ def generate_dataset_profile(  # noqa: C901 (complexity)
         self.query_combiner.flush()
         return profile
 
+    def init_profile(self):
+        profile = DatasetProfileClass(timestampMillis=get_sys_time())
+        if self.partition:
+            profile.partitionSpec = PartitionSpecClass(partition=self.partition)
+        elif self.config.limit:
+            profile.partitionSpec = PartitionSpecClass(
+                type=PartitionTypeClass.QUERY,
+                partition=json.dumps(
+                    dict(limit=self.config.limit, offset=self.config.offset)
+                ),
+            )
+        elif self.custom_sql:
+            profile.partitionSpec = PartitionSpecClass(
+                type=PartitionTypeClass.QUERY, partition="SAMPLE"
+            )
+
+        return profile
+
     def update_dataset_batch_use_sampling(self, profile: DatasetProfileClass) -> None:
         if (
             self.dataset.engine.dialect.name.lower() == BIGQUERY
@@ -1064,6 +1075,7 @@ def _generate_single_profile(
                     partition,
                     self.config,
                     self.report,
+                    custom_sql,
                     query_combiner,
                 ).generate_dataset_profile()
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py
index 77761c529ba0b..24a3e520d8caf 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py
@@ -157,12 +157,12 @@ class GEProfilingConfig(ConfigModel):
     )
     use_sampling: bool = Field(
         default=True,
-        description="Whether to profile column level stats on sample of table. Only BigQuery supports this. "
+        description="Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. "
         "If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ",
     )
 
     sample_size: int = Field(
-        default=1000,
+        default=10000,
         description="Number of rows to be sampled from table for column level profiling."
         "Applicable only if `use_sampling` is set to True.",
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py
index e983734082b1d..771636e8498a3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py
@@ -1,33 +1,19 @@
-import dataclasses
 import logging
-from datetime import datetime
-from typing import Dict, Iterable, List, Optional, Union, cast
+from typing import Dict, Iterable, List, Optional, Union
 
-from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest
 from datahub.ingestion.source.redshift.config import RedshiftConfig
 from datahub.ingestion.source.redshift.redshift_schema import (
     RedshiftTable,
     RedshiftView,
 )
 from datahub.ingestion.source.redshift.report import RedshiftReport
-from datahub.ingestion.source.sql.sql_generic_profiler import (
-    GenericProfiler,
-    TableProfilerRequest,
-)
+from datahub.ingestion.source.sql.sql_generic_profiler import GenericProfiler
 from datahub.ingestion.source.state.profiling_state_handler import ProfilingHandler
 
 logger = logging.getLogger(__name__)
 
 
-@dataclasses.dataclass
-class RedshiftProfilerRequest(GEProfilerRequest):
-    table: Union[RedshiftTable, RedshiftView]
-    profile_table_level_only: bool = False
-
-
 class RedshiftProfiler(GenericProfiler):
     config: RedshiftConfig
     report: RedshiftReport
@@ -63,80 +49,21 @@ def get_workunits(
                     continue
                 for table in tables[db].get(schema, {}):
                     # Emit the profile work unit
-                    profile_request = self.get_redshift_profile_request(
-                        table, schema, db
-                    )
+                    profile_request = self.get_profile_request(table, schema, db)
                     if profile_request is not None:
+                        self.report.report_entity_profiled(profile_request.pretty_name)
                         profile_requests.append(profile_request)
 
             if len(profile_requests) == 0:
                 continue
-            table_profile_requests = cast(List[TableProfilerRequest], profile_requests)
-            for request, profile in self.generate_profiles(
-                table_profile_requests,
+
+            yield from self.generate_profile_workunits(
+                profile_requests,
                 self.config.profiling.max_workers,
                 db,
                 platform=self.platform,
                 profiler_args=self.get_profile_args(),
-            ):
-                if profile is None:
-                    continue
-                request = cast(RedshiftProfilerRequest, request)
-
-                profile.sizeInBytes = request.table.size_in_bytes
-                dataset_name = request.pretty_name
-                dataset_urn = make_dataset_urn_with_platform_instance(
-                    self.platform,
-                    dataset_name,
-                    self.config.platform_instance,
-                    self.config.env,
-                )
-
-                # We don't add to the profiler state if we only do table level profiling as it always happens
-                if self.state_handler and not request.profile_table_level_only:
-                    self.state_handler.add_to_state(
-                        dataset_urn, int(datetime.now().timestamp() * 1000)
-                    )
-
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=dataset_urn, aspect=profile
-                ).as_workunit()
-
-    def get_redshift_profile_request(
-        self,
-        table: Union[RedshiftTable, RedshiftView],
-        schema_name: str,
-        db_name: str,
-    ) -> Optional[RedshiftProfilerRequest]:
-        skip_profiling = False
-        profile_table_level_only = self.config.profiling.profile_table_level_only
-        dataset_name = f"{db_name}.{schema_name}.{table.name}".lower()
-        if not self.is_dataset_eligible_for_profiling(
-            dataset_name, table.last_altered, table.size_in_bytes, table.rows_count
-        ):
-            # Profile only table level if dataset is filtered from profiling
-            # due to size limits alone
-            if self.is_dataset_eligible_for_profiling(
-                dataset_name, table.last_altered, 0, 0
-            ):
-                profile_table_level_only = True
-            else:
-                skip_profiling = True
-
-        if len(table.columns) == 0:
-            skip_profiling = True
-
-        if skip_profiling:
-            if self.config.profiling.report_dropped_profiles:
-                self.report.report_dropped(f"profile of {dataset_name}")
-            return None
+            )
 
-        self.report.report_entity_profiled(dataset_name)
-        logger.debug(f"Preparing profiling request for {dataset_name}")
-        profile_request = RedshiftProfilerRequest(
-            pretty_name=dataset_name,
-            batch_kwargs=dict(schema=schema_name, table=table.name),
-            table=table,
-            profile_table_level_only=profile_table_level_only,
-        )
-        return profile_request
+    def get_dataset_name(self, table_name: str, schema_name: str, db_name: str) -> str:
+        return f"{db_name}.{schema_name}.{table_name}".lower()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
index 5f5e8e4bcdea3..24275dcdff34d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
@@ -1,20 +1,12 @@
-import dataclasses
 import logging
-from datetime import datetime
-from typing import Callable, Dict, Iterable, List, Optional, cast
+from typing import Callable, Dict, Iterable, List, Optional
 
 from snowflake.sqlalchemy import snowdialect
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.sql import sqltypes
 
-from datahub.configuration.pattern_utils import is_schema_allowed
-from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.ge_data_profiler import (
-    DatahubGEProfiler,
-    GEProfilerRequest,
-)
+from datahub.ingestion.source.ge_data_profiler import DatahubGEProfiler
 from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config
 from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery
 from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report
@@ -23,10 +15,8 @@
     SnowflakeTable,
 )
 from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeCommonMixin
-from datahub.ingestion.source.sql.sql_generic_profiler import (
-    GenericProfiler,
-    TableProfilerRequest,
-)
+from datahub.ingestion.source.sql.sql_generic import BaseTable
+from datahub.ingestion.source.sql.sql_generic_profiler import GenericProfiler
 from datahub.ingestion.source.state.profiling_state_handler import ProfilingHandler
 
 snowdialect.ischema_names["GEOGRAPHY"] = sqltypes.NullType
@@ -35,12 +25,6 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclasses.dataclass
-class SnowflakeProfilerRequest(GEProfilerRequest):
-    table: SnowflakeTable
-    profile_table_level_only: bool = False
-
-
 class SnowflakeProfiler(GenericProfiler, SnowflakeCommonMixin):
     def __init__(
         self,
@@ -65,101 +49,52 @@ def get_workunits(
 
         profile_requests = []
         for schema in database.schemas:
-            if not is_schema_allowed(
-                self.config.schema_pattern,
-                schema.name,
-                database.name,
-                self.config.match_fully_qualified_names,
-            ):
-                continue
-
             for table in db_tables[schema.name]:
-                profile_request = self.get_snowflake_profile_request(
+                profile_request = self.get_profile_request(
                     table, schema.name, database.name
                 )
                 if profile_request is not None:
+                    self.report.report_entity_profiled(profile_request.pretty_name)
                     profile_requests.append(profile_request)
 
         if len(profile_requests) == 0:
             return
 
-        table_profile_requests = cast(List[TableProfilerRequest], profile_requests)
-
-        for request, profile in self.generate_profiles(
-            table_profile_requests,
+        yield from self.generate_profile_workunits(
+            profile_requests,
             self.config.profiling.max_workers,
             database.name,
             platform=self.platform,
             profiler_args=self.get_profile_args(),
-        ):
-            if profile is None:
-                continue
-            profile.sizeInBytes = cast(
-                SnowflakeProfilerRequest, request
-            ).table.size_in_bytes
-            dataset_name = request.pretty_name
-            dataset_urn = make_dataset_urn_with_platform_instance(
-                self.platform,
-                dataset_name,
-                self.config.platform_instance,
-                self.config.env,
-            )
-
-            # We don't add to the profiler state if we only do table level profiling as it always happens
-            if self.state_handler:
-                self.state_handler.add_to_state(
-                    dataset_urn, int(datetime.now().timestamp() * 1000)
-                )
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn, aspect=profile
-            ).as_workunit()
+        )
 
-    def get_snowflake_profile_request(
-        self,
-        table: SnowflakeTable,
-        schema_name: str,
-        db_name: str,
-    ) -> Optional[SnowflakeProfilerRequest]:
-        skip_profiling = False
-        profile_table_level_only = self.config.profiling.profile_table_level_only
-        dataset_name = self.get_dataset_identifier(table.name, schema_name, db_name)
-        if not self.is_dataset_eligible_for_profiling(
-            dataset_name, table.last_altered, table.size_in_bytes, table.rows_count
+    def get_dataset_name(self, table_name: str, schema_name: str, db_name: str) -> str:
+        return self.get_dataset_identifier(table_name, schema_name, db_name)
+
+    def get_batch_kwargs(
+        self, table: BaseTable, schema_name: str, db_name: str
+    ) -> dict:
+        custom_sql = None
+        if (
+            not self.config.profiling.limit
+            and self.config.profiling.use_sampling
+            and table.rows_count
+            and table.rows_count > self.config.profiling.sample_size
         ):
-            # Profile only table level if dataset is filtered from profiling
-            # due to size limits alone
-            if self.is_dataset_eligible_for_profiling(
-                dataset_name, table.last_altered, 0, 0
-            ):
-                profile_table_level_only = True
-            else:
-                skip_profiling = True
-
-        if len(table.columns) == 0:
-            skip_profiling = True
-
-        if skip_profiling:
-            if self.config.profiling.report_dropped_profiles:
-                self.report.report_dropped(f"profile of {dataset_name}")
-            return None
-
-        self.report.report_entity_profiled(dataset_name)
-        logger.debug(f"Preparing profiling request for {dataset_name}")
-        profile_request = SnowflakeProfilerRequest(
-            pretty_name=dataset_name,
-            batch_kwargs=dict(
-                schema=schema_name,
-                table=table.name,
-                # Lowercase/Mixedcase table names in Snowflake do not work by default.
-                # We need to pass `use_quoted_name=True` for such tables as mentioned here -
-                # https://github.com/great-expectations/great_expectations/pull/2023
-                use_quoted_name=(table.name != table.name.upper()),
-            ),
-            table=table,
-            profile_table_level_only=profile_table_level_only,
-        )
-        return profile_request
+            # GX creates a temporary table from query if query is passed as batch kwargs.
+            # We are using fraction-based sampling here, instead of fixed-size sampling because
+            # Fixed-size sampling can be slower than equivalent fraction-based sampling
+            # as per https://docs.snowflake.com/en/sql-reference/constructs/sample#performance-considerations
+            sample_pc = 100 * self.config.profiling.sample_size / table.rows_count
+            custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE ({sample_pc:.3f})'
+        return {
+            **super().get_batch_kwargs(table, schema_name, db_name),
+            # Lowercase/Mixedcase table names in Snowflake do not work by default.
+            # We need to pass `use_quoted_name=True` for such tables as mentioned here -
+            # https://github.com/great-expectations/great_expectations/pull/2023
+            "use_quoted_name": (table.name != table.name.upper()),
+            "custom_sql": custom_sql,
+        }
 
     def get_profiler_instance(
         self, db_name: Optional[str] = None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py
index 344c114d464a9..aaeee5717a867 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py
@@ -1,12 +1,15 @@
 import logging
+from abc import abstractmethod
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
-from typing import Dict, Iterable, List, Optional, Tuple, Union, cast
+from typing import Dict, Iterable, List, Optional, Union, cast
 
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.engine.reflection import Inspector
 
 from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.ge_data_profiler import (
     DatahubGEProfiler,
     GEProfilerRequest,
@@ -16,7 +19,7 @@
 from datahub.ingestion.source.sql.sql_generic import BaseTable, BaseView
 from datahub.ingestion.source.state.profiling_state_handler import ProfilingHandler
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProfile
-from datahub.metadata.schema_classes import DatasetProfileClass
+from datahub.metadata.com.linkedin.pegasus2avro.timeseries import PartitionType
 from datahub.utilities.stats_collections import TopKDict, int_top_k_dict
 
 
@@ -63,14 +66,14 @@ def __init__(
         self.platform = platform
         self.state_handler = state_handler
 
-    def generate_profiles(
+    def generate_profile_workunits(
         self,
         requests: List[TableProfilerRequest],
         max_workers: int,
         db_name: Optional[str] = None,
         platform: Optional[str] = None,
         profiler_args: Optional[Dict] = None,
-    ) -> Iterable[Tuple[GEProfilerRequest, Optional[DatasetProfileClass]]]:
+    ) -> Iterable[MetadataWorkUnit]:
         ge_profile_requests: List[GEProfilerRequest] = [
             cast(GEProfilerRequest, request)
             for request in requests
@@ -80,21 +83,109 @@ def generate_profiles(
             request for request in requests if request.profile_table_level_only
         ]
         for request in table_level_profile_requests:
-            profile = DatasetProfile(
+            table_level_profile = DatasetProfile(
                 timestampMillis=int(datetime.now().timestamp() * 1000),
                 columnCount=request.table.column_count,
                 rowCount=request.table.rows_count,
                 sizeInBytes=request.table.size_in_bytes,
             )
-            yield (request, profile)
+            dataset_urn = self.dataset_urn_builder(request.pretty_name)
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn, aspect=table_level_profile
+            ).as_workunit()
 
         if not ge_profile_requests:
             return
 
         # Otherwise, if column level profiling is enabled, use  GE profiler.
         ge_profiler = self.get_profiler_instance(db_name)
-        yield from ge_profiler.generate_profiles(
+
+        for ge_profiler_request, profile in ge_profiler.generate_profiles(
             ge_profile_requests, max_workers, platform, profiler_args
+        ):
+            if profile is None:
+                continue
+
+            request = cast(TableProfilerRequest, ge_profiler_request)
+            profile.sizeInBytes = request.table.size_in_bytes
+
+            # If table is partitioned we profile only one partition (if nothing set then the last one)
+            # but for table level we can use the rows_count from the table metadata
+            # This way even though column statistics only reflects one partition data but the rows count
+            # shows the proper count.
+            if (
+                profile.partitionSpec
+                and profile.partitionSpec.type != PartitionType.FULL_TABLE
+            ):
+                profile.rowCount = request.table.rows_count
+
+            dataset_urn = self.dataset_urn_builder(request.pretty_name)
+
+            # We don't add to the profiler state if we only do table level profiling as it always happens
+            if self.state_handler:
+                self.state_handler.add_to_state(
+                    dataset_urn, int(datetime.now().timestamp() * 1000)
+                )
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn, aspect=profile
+            ).as_workunit()
+
+    def dataset_urn_builder(self, dataset_name: str) -> str:
+        return make_dataset_urn_with_platform_instance(
+            self.platform,
+            dataset_name,
+            self.config.platform_instance,
+            self.config.env,
+        )
+
+    @abstractmethod
+    def get_dataset_name(self, table_name: str, schema_name: str, db_name: str) -> str:
+        pass
+
+    def get_profile_request(
+        self, table: BaseTable, schema_name: str, db_name: str
+    ) -> Optional[TableProfilerRequest]:
+        skip_profiling = False
+        profile_table_level_only = self.config.profiling.profile_table_level_only
+        dataset_name = self.get_dataset_name(table.name, schema_name, db_name)
+        if not self.is_dataset_eligible_for_profiling(
+            dataset_name, table.last_altered, table.size_in_bytes, table.rows_count
+        ):
+            # Profile only table level if dataset is filtered from profiling
+            # due to size limits alone
+            if self.is_dataset_eligible_for_profiling(
+                dataset_name, table.last_altered, 0, 0
+            ):
+                profile_table_level_only = True
+            else:
+                skip_profiling = True
+                self.report.num_tables_not_eligible_profiling[
+                    f"{db_name}.{schema_name}"
+                ] += 1
+
+        if table.column_count == 0:
+            skip_profiling = True
+
+        if skip_profiling:
+            if self.config.profiling.report_dropped_profiles:
+                self.report.report_dropped(f"profile of {dataset_name}")
+            return None
+
+        logger.debug(f"Preparing profiling request for {dataset_name}")
+        profile_request = TableProfilerRequest(
+            pretty_name=dataset_name,
+            batch_kwargs=self.get_batch_kwargs(table, schema_name, db_name),
+            table=table,
+            profile_table_level_only=profile_table_level_only,
+        )
+        return profile_request
+
+    def get_batch_kwargs(
+        self, table: BaseTable, schema_name: str, db_name: str
+    ) -> dict:
+        return dict(
+            schema=schema_name,
+            table=table.name,
         )
 
     def get_inspectors(self) -> Iterable[Inspector]:

From c0feceb76fbf607e2883b7f2960eaf6c757629e4 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Fri, 6 Oct 2023 17:10:24 -0400
Subject: [PATCH 018/259] test(): Manage Access Tokens Cypress test (#8936)

---
 .../src/app/settings/AccessTokenModal.tsx     |  4 +-
 .../src/app/settings/AccessTokens.tsx         |  7 ++-
 .../src/app/settings/CreateTokenModal.tsx     | 18 +++++---
 .../e2e/settings/manage_access_tokens.js      | 43 +++++++++++++++++++
 .../tests/cypress/cypress/support/commands.js |  6 +++
 5 files changed, 70 insertions(+), 8 deletions(-)
 create mode 100644 smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js

diff --git a/datahub-web-react/src/app/settings/AccessTokenModal.tsx b/datahub-web-react/src/app/settings/AccessTokenModal.tsx
index 0303db656c2a8..10427210d0692 100644
--- a/datahub-web-react/src/app/settings/AccessTokenModal.tsx
+++ b/datahub-web-react/src/app/settings/AccessTokenModal.tsx
@@ -60,7 +60,7 @@ export const AccessTokenModal = ({ visible, onClose, accessToken, expiresInText
             onCancel={onClose}
             footer={
                 <>
-                    <Button id="createTokenButton" onClick={onClose}>
+                    <Button id="createTokenButton" onClick={onClose} data-testid="access-token-modal-close-button">
                         Close
                     </Button>
                 </>
@@ -81,7 +81,7 @@ export const AccessTokenModal = ({ visible, onClose, accessToken, expiresInText
                 <ModalSectionHeader strong>Token</ModalSectionHeader>
                 <ModalSectionParagraph>{expiresInText}</ModalSectionParagraph>
                 <Typography.Paragraph copyable={{ text: accessToken }}>
-                    <pre>{accessToken}</pre>
+                    <pre data-testid="access-token-value">{accessToken}</pre>
                 </Typography.Paragraph>
             </ModalSection>
             <ModalSection>
diff --git a/datahub-web-react/src/app/settings/AccessTokens.tsx b/datahub-web-react/src/app/settings/AccessTokens.tsx
index 02ff3f1cd304c..c7a015de392da 100644
--- a/datahub-web-react/src/app/settings/AccessTokens.tsx
+++ b/datahub-web-react/src/app/settings/AccessTokens.tsx
@@ -199,7 +199,12 @@ export const AccessTokens = () => {
             key: 'x',
             render: (_, record: any) => (
                 <ActionButtonContainer>
-                    <Button onClick={() => onRemoveToken(record)} icon={<DeleteOutlined />} danger>
+                    <Button
+                        onClick={() => onRemoveToken(record)}
+                        icon={<DeleteOutlined />}
+                        danger
+                        data-testid="revoke-token-button"
+                    >
                         Revoke
                     </Button>
                 </ActionButtonContainer>
diff --git a/datahub-web-react/src/app/settings/CreateTokenModal.tsx b/datahub-web-react/src/app/settings/CreateTokenModal.tsx
index 6038a86e23303..3cc446651efcb 100644
--- a/datahub-web-react/src/app/settings/CreateTokenModal.tsx
+++ b/datahub-web-react/src/app/settings/CreateTokenModal.tsx
@@ -117,10 +117,15 @@ export default function CreateTokenModal({ currentUserUrn, visible, onClose, onC
                 onCancel={onModalClose}
                 footer={
                     <>
-                        <Button onClick={onModalClose} type="text">
+                        <Button onClick={onModalClose} type="text" data-testid="cancel-create-access-token-button">
                             Cancel
                         </Button>
-                        <Button id="createTokenButton" onClick={onCreateNewToken} disabled={createButtonEnabled}>
+                        <Button
+                            id="createTokenButton"
+                            onClick={onCreateNewToken}
+                            disabled={createButtonEnabled}
+                            data-testid="create-access-token-button"
+                        >
                             Create
                         </Button>
                     </>
@@ -148,18 +153,21 @@ export default function CreateTokenModal({ currentUserUrn, visible, onClose, onC
                             ]}
                             hasFeedback
                         >
-                            <Input placeholder="A name for your token" />
+                            <Input placeholder="A name for your token" data-testid="create-access-token-name" />
                         </Form.Item>
                     </Form.Item>
                     <Form.Item label={<Typography.Text strong>Description</Typography.Text>}>
                         <Typography.Paragraph>An optional description for your new token.</Typography.Paragraph>
                         <Form.Item name="description" rules={[{ whitespace: true }, { min: 1, max: 500 }]} hasFeedback>
-                            <Input placeholder="A description for your token" />
+                            <Input
+                                placeholder="A description for your token"
+                                data-testid="create-access-token-description"
+                            />
                         </Form.Item>
                     </Form.Item>
                     <ExpirationSelectContainer>
                         <Typography.Text strong>Expires in</Typography.Text>
-                        <Form.Item name="duration" noStyle>
+                        <Form.Item name="duration" data-testid="create-access-token-duration" noStyle>
                             <ExpirationDurationSelect>
                                 {ACCESS_TOKEN_DURATIONS.map((duration) => (
                                     <Select.Option key={duration.text} value={duration.duration}>
diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js b/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js
new file mode 100644
index 0000000000000..7a77c2b77df5b
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/settings/manage_access_tokens.js
@@ -0,0 +1,43 @@
+import { aliasQuery, hasOperationName } from "../utils";
+const test_id = Math.floor(Math.random() * 100000);
+
+describe("manage access tokens", () => {
+    before(() => {
+      cy.intercept("POST", "/api/v2/graphql", (req) => {
+        aliasQuery(req, "appConfig");
+      });
+    });
+    
+    const setTokenAuthEnabledFlag = (isOn) => {
+      cy.intercept("POST", "/api/v2/graphql", (req) => {
+        if (hasOperationName(req, "appConfig")) {
+          req.reply((res) => {
+            res.body.data.appConfig.authConfig.tokenAuthEnabled = isOn;
+          });
+        }
+      });
+    };
+
+    it("create and revoke access token", () => {
+      //create access token, verify token on ui
+      setTokenAuthEnabledFlag(true);
+      cy.loginWithCredentials();
+      cy.goToAccessTokenSettings();
+      cy.clickOptionWithTestId("add-token-button");
+      cy.enterTextInTestId("create-access-token-name", "Token Name" + test_id);
+      cy.enterTextInTestId("create-access-token-description", "Token Description" + test_id);
+      cy.clickOptionWithTestId("create-access-token-button");
+      cy.waitTextVisible("New Personal Access Token");
+      cy.get('[data-testid="access-token-value"]').should("be.visible");
+      cy.get('[data-testid="access-token-value"]').invoke('text').should('match', /^[a-zA-Z0-9-_]+\.[a-zA-Z0-9-_]+\.[a-zA-Z0-9-_]+$/);
+      cy.clickOptionWithTestId("access-token-modal-close-button");
+      //revoke access token, verify token removed from ui
+      cy.waitTextVisible("Token Name" + test_id);
+      cy.waitTextVisible("Token Description" + test_id);
+      cy.clickOptionWithTestId("revoke-token-button");
+      cy.waitTextVisible("Are you sure you want to revoke this token?");
+      cy.clickOptionWithText("Yes");
+      cy.ensureTextNotPresent("Token Name" + test_id);
+      cy.ensureTextNotPresent("Token Description" + test_id);
+    });
+});
\ No newline at end of file
diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js
index 8bfe7305c001f..64bc1253fc383 100644
--- a/smoke-test/tests/cypress/cypress/support/commands.js
+++ b/smoke-test/tests/cypress/cypress/support/commands.js
@@ -84,6 +84,12 @@ Cypress.Commands.add("goToOwnershipTypesSettings", () => {
   cy.waitTextVisible("Manage Ownership");
 });
 
+Cypress.Commands.add("goToAccessTokenSettings", () => {
+  cy.visit("/settings/tokens");
+  cy.waitTextVisible("Manage Access Tokens");
+  cy.wait(3000);
+});
+
 Cypress.Commands.add("goToIngestionPage", () => {
   cy.visit("/ingestion");
   cy.waitTextVisible("Manage Ingestion");

From b191abbc5bb32a0a3c895facdff14d146da9fb74 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Fri, 6 Oct 2023 17:11:57 -0400
Subject: [PATCH 019/259] test(): Nested domains cypress test (#8879)

---
 .../src/app/domain/CreateDomainModal.tsx      |  5 +-
 .../nestedDomains/ManageDomainsPageV2.tsx     |  7 ++-
 .../domainNavigator/DomainNode.tsx            |  2 +-
 .../shared/EntityDropdown/EntityDropdown.tsx  |  4 +-
 .../shared/EntityDropdown/MoveDomainModal.tsx |  5 +-
 .../cypress/e2e/domains/nested_domains.js     | 53 +++++++++++++++++++
 6 files changed, 70 insertions(+), 6 deletions(-)
 create mode 100644 smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js

diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx
index ca1bc30596003..606444d34bdc9 100644
--- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx
+++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx
@@ -191,7 +191,10 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
                         rules={[{ whitespace: true }, { min: 1, max: 500 }]}
                         hasFeedback
                     >
-                        <Input.TextArea placeholder="A description for your domain" />
+                        <Input.TextArea
+                            placeholder="A description for your domain"
+                            data-testid="create-domain-description"
+                        />
                     </FormItemNoMargin>
                 </FormItemWithMargin>
                 <Collapse ghost>
diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx
index 0e5c035df00c1..b69f0c5458b5d 100644
--- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx
+++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx
@@ -42,7 +42,12 @@ export default function ManageDomainsPageV2() {
             <OnboardingTour stepIds={[DOMAINS_INTRO_ID, DOMAINS_CREATE_DOMAIN_ID]} />
             <Header>
                 <DomainsTitle />
-                <Button type="primary" id={DOMAINS_CREATE_DOMAIN_ID} onClick={() => setIsCreatingDomain(true)}>
+                <Button
+                    type="primary"
+                    id={DOMAINS_CREATE_DOMAIN_ID}
+                    onClick={() => setIsCreatingDomain(true)}
+                    data-testid="domains-new-domain-button"
+                >
                     <PlusOutlined /> New Domain
                 </Button>
             </Header>
diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx
index 09c8e13853bb7..bf70bd043fd4a 100644
--- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx
+++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx
@@ -103,7 +103,7 @@ export default function DomainNode({ domain, numDomainChildren, domainUrnToHide,
 
     return (
         <>
-            <RowWrapper>
+            <RowWrapper data-testid="domain-list-item">
                 {hasDomainChildren && (
                     <ButtonWrapper>
                         <RotatingTriangle isOpen={isOpen && !isClosing} onClick={toggle} />
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx
index be975249b2670..bfb7ff7e540c4 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx
@@ -203,7 +203,7 @@ function EntityDropdown(props: Props) {
                                 disabled={isMoveDisabled(entityType, entityData, me.platformPrivileges)}
                                 onClick={() => setIsMoveModalVisible(true)}
                             >
-                                <MenuItem>
+                                <MenuItem data-testid="entity-menu-move-button">
                                     <FolderOpenOutlined /> &nbsp;Move
                                 </MenuItem>
                             </StyledMenuItem>
@@ -223,7 +223,7 @@ function EntityDropdown(props: Props) {
                                             : undefined
                                     }
                                 >
-                                    <MenuItem>
+                                    <MenuItem data-testid="entity-menu-delete-button">
                                         <DeleteOutlined /> &nbsp;Delete
                                     </MenuItem>
                                 </Tooltip>
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx
index cdbf6fdabf3c9..3826f934c1c25 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx
@@ -67,6 +67,7 @@ function MoveDomainModal(props: Props) {
     return (
         <Modal
             title="Move"
+            data-testid="move-domain-modal"
             visible
             onCancel={onClose}
             footer={
@@ -74,7 +75,9 @@ function MoveDomainModal(props: Props) {
                     <Button onClick={onClose} type="text">
                         Cancel
                     </Button>
-                    <Button onClick={moveDomain}>Move</Button>
+                    <Button onClick={moveDomain} data-testid="move-domain-modal-move-button">
+                        Move
+                    </Button>
                 </>
             }
         >
diff --git a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js
new file mode 100644
index 0000000000000..a2d4de0f51659
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js
@@ -0,0 +1,53 @@
+const domainName = "CypressNestedDomain";
+const domainDescription = "CypressNestedDomainDescription";
+
+describe("nested domains test", () => {
+
+    it("create a domain, move under parent, remove domain", () => {
+        // Create a new domain without a parent
+        cy.loginWithCredentials();
+        cy.goToDomainList();
+        cy.clickOptionWithTestId("domains-new-domain-button");
+        cy.get('[data-testid="create-domain-name"]').click().type(domainName);
+        cy.get('[data-testid="create-domain-description"]').click().type(domainDescription);
+        cy.clickOptionWithTestId("create-domain-button");
+        cy.waitTextVisible(domainName);
+
+        // Ensure the new domain has no parent in the navigation sidebar
+        cy.waitTextVisible(domainDescription);
+
+        // Move a domain from the root level to be under a parent domain
+        cy.clickOptionWithText(domainName);
+        cy.openThreeDotDropdown();
+        cy.clickOptionWithTestId("entity-menu-move-button");
+        cy.get('[data-testid="move-domain-modal"]').contains("Marketing").click({force: true});
+        cy.get('[data-testid="move-domain-modal"]').contains("Marketing").should("be.visible");
+        cy.clickOptionWithTestId("move-domain-modal-move-button").wait(5000);
+
+        // Wnsure domain is no longer on the sidebar navigator at the top level but shows up under the parent
+        cy.goToDomainList();
+        cy.ensureTextNotPresent(domainName);
+        cy.ensureTextNotPresent(domainDescription);
+        cy.waitTextVisible("1 sub-domain");
+
+        // Move a domain from under a parent domain to the root level
+        cy.get('[data-testid="domain-list-item"]').contains("Marketing").prev().click();
+        cy.clickOptionWithText(domainName);
+        cy.openThreeDotDropdown();
+        cy.clickOptionWithTestId("entity-menu-move-button");
+        cy.clickOptionWithTestId("move-domain-modal-move-button").wait(5000);
+        cy.goToDomainList();
+        cy.waitTextVisible(domainName);
+        cy.waitTextVisible(domainDescription);
+
+        // Delete a domain
+        cy.clickOptionWithText(domainName).wait(3000);
+        cy.openThreeDotDropdown();
+        cy.clickOptionWithTestId("entity-menu-delete-button");
+        cy.waitTextVisible("Are you sure you want to remove this Domain?");
+        cy.clickOptionWithText("Yes");
+        cy.waitTextVisible("Deleted Domain!");
+        cy.ensureTextNotPresent(domainName);
+        cy.ensureTextNotPresent(domainDescription);
+    });
+});
\ No newline at end of file

From 93958302d529a65021c78f880347930297854692 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Sun, 8 Oct 2023 13:26:48 -0400
Subject: [PATCH 020/259] feat(models/assertion): Add SQL Assertions (#8969)

---
 .../com/linkedin/assertion/AssertionInfo.pdl  | 17 ++++-
 .../linkedin/assertion/SqlAssertionInfo.pdl   | 67 +++++++++++++++++++
 2 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/SqlAssertionInfo.pdl

diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl
index ae2a58028057b..e161270145a88 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl
@@ -32,6 +32,11 @@ record AssertionInfo includes CustomProperties, ExternalReference {
        */
       VOLUME
 
+      /**
+       * A raw SQL-statement based assertion
+       */
+      SQL
+
       /**
        * A schema or structural assertion.
        *
@@ -56,7 +61,12 @@ record AssertionInfo includes CustomProperties, ExternalReference {
     volumeAssertion: optional VolumeAssertionInfo
 
     /**
-    * An schema Assertion definition. This field is populated when the type is DATASET_SCHEMA
+    * A SQL Assertion definition. This field is populated when the type is SQL.
+    */
+    sqlAssertion: optional SqlAssertionInfo
+
+    /**
+    * An schema Assertion definition. This field is populated when the type is DATA_SCHEMA
     */
     schemaAssertion: optional SchemaAssertionInfo
 
@@ -67,4 +77,9 @@ record AssertionInfo includes CustomProperties, ExternalReference {
     * the platform where it was ingested from.
     */
     source: optional AssertionSource
+
+    /**
+    * An optional human-readable description of the assertion
+    */
+    description: optional string
 }
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/SqlAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/SqlAssertionInfo.pdl
new file mode 100644
index 0000000000000..f6ce738252f35
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/SqlAssertionInfo.pdl
@@ -0,0 +1,67 @@
+namespace com.linkedin.assertion
+
+import com.linkedin.common.Urn
+import com.linkedin.dataset.DatasetFilter
+
+/**
+* Attributes defining a SQL Assertion
+*/
+record SqlAssertionInfo {
+    /**
+     * The type of the SQL assertion being monitored.
+     */
+    @Searchable = {}
+    type: enum SqlAssertionType {
+      /**
+       * A SQL Metric Assertion, e.g. one based on a numeric value returned by an arbitrary SQL query.
+       */
+       METRIC
+       /**
+       * A SQL assertion that is evaluated against the CHANGE in a metric assertion
+       * over time.
+       */
+       METRIC_CHANGE
+    }
+
+    /**
+    * The entity targeted by this SQL check.
+    */
+    @Searchable = {
+      "fieldType": "URN"
+    }
+    @Relationship = {
+      "name": "Asserts",
+      "entityTypes": [ "dataset" ]
+    }
+    entity: Urn
+
+    /**
+    * The SQL statement to be executed when evaluating the assertion (or computing the metric).
+    * This should be a valid and complete statement, executable by itself.
+    *
+    * Usually this should be a SELECT query statement.
+    */
+    statement: string
+
+    /**
+    * The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage.
+    * This value is required if the type is METRIC_CHANGE.
+    */
+    changeType: optional AssertionValueChangeType
+
+    /**
+    * The operator you'd like to apply to the result of the SQL query.
+    *
+    * Note that at this time, only numeric operators are valid inputs:
+    * GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO,
+    * BETWEEN.
+    */
+    operator: AssertionStdOperator
+
+    /**
+    * The parameters you'd like to provide as input to the operator.
+    *
+    * Note that only numeric parameter types are valid inputs: NUMBER.
+    */
+    parameters: AssertionStdParameters
+}
\ No newline at end of file

From 8d175ef7ef1ae8ffada7b2df2fb711ac02a6785d Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Tue, 10 Oct 2023 02:04:25 +0530
Subject: [PATCH 021/259] feat(ingest): incremental lineage source helper
 (#8941)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../datahub/ingestion/api/source_helpers.py   | 138 +++++++++-
 .../ingestion/source/bigquery_v2/bigquery.py  |   3 +-
 .../source/snowflake/snowflake_v2.py          |   9 +
 .../snowflake_privatelink_golden.json         | 243 +++++++++++------
 .../integration/snowflake/test_snowflake.py   |   2 +
 .../snowflake/test_snowflake_failures.py      |   6 +-
 .../snowflake/test_snowflake_stateful.py      |   3 +-
 ...l_less_upstreams_in_gms_aspect_golden.json | 106 ++++++++
 ...l_more_upstreams_in_gms_aspect_golden.json | 120 +++++++++
 .../incremental_table_lineage_golden.json     |  41 +++
 .../test_incremental_lineage_helper.py        | 244 ++++++++++++++++++
 .../source_helpers}/test_source_helpers.py    |   0
 12 files changed, 829 insertions(+), 86 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_less_upstreams_in_gms_aspect_golden.json
 create mode 100644 metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_more_upstreams_in_gms_aspect_golden.json
 create mode 100644 metadata-ingestion/tests/unit/api/source_helpers/incremental_table_lineage_golden.json
 create mode 100644 metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
 rename metadata-ingestion/tests/unit/{ => api/source_helpers}/test_source_helpers.py (100%)

diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
index 7fc15cf829678..42f970e97c95f 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
@@ -1,3 +1,4 @@
+import copy
 import logging
 from datetime import datetime, timezone
 from typing import (
@@ -15,9 +16,14 @@
 )
 
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
-from datahub.emitter.mce_builder import make_dataplatform_instance_urn
+from datahub.emitter.mce_builder import (
+    datahub_guid,
+    make_dataplatform_instance_urn,
+    set_aspect,
+)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import (
     BrowsePathEntryClass,
     BrowsePathsClass,
@@ -25,12 +31,17 @@
     ChangeTypeClass,
     ContainerClass,
     DatasetUsageStatisticsClass,
+    FineGrainedLineageClass,
     MetadataChangeEventClass,
     MetadataChangeProposalClass,
     StatusClass,
+    SystemMetadataClass,
     TagKeyClass,
     TimeWindowSizeClass,
+    UpstreamClass,
+    UpstreamLineageClass,
 )
+from datahub.specific.dataset import DatasetPatchBuilder
 from datahub.telemetry import telemetry
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.tag_urn import TagUrn
@@ -366,3 +377,128 @@ def _prepend_platform_instance(
         return [BrowsePathEntryClass(id=urn, urn=urn)] + entries
 
     return entries
+
+
+def auto_incremental_lineage(
+    graph: Optional[DataHubGraph],
+    incremental_lineage: bool,
+    include_column_level_lineage: bool,
+    stream: Iterable[MetadataWorkUnit],
+) -> Iterable[MetadataWorkUnit]:
+    if not incremental_lineage:
+        yield from stream
+        return  # early exit
+
+    for wu in stream:
+        lineage_aspect: Optional[UpstreamLineageClass] = wu.get_aspect_of_type(
+            UpstreamLineageClass
+        )
+        urn = wu.get_urn()
+
+        if lineage_aspect:
+            if isinstance(wu.metadata, MetadataChangeEventClass):
+                set_aspect(
+                    wu.metadata, None, UpstreamLineageClass
+                )  # we'll emit upstreamLineage separately below
+                if len(wu.metadata.proposedSnapshot.aspects) > 0:
+                    yield wu
+
+            yield _lineage_wu_via_read_modify_write(
+                graph, urn, lineage_aspect, wu.metadata.systemMetadata
+            ) if include_column_level_lineage else _convert_upstream_lineage_to_patch(
+                urn, lineage_aspect, wu.metadata.systemMetadata
+            )
+        else:
+            yield wu
+
+
+def _convert_upstream_lineage_to_patch(
+    urn: str,
+    aspect: UpstreamLineageClass,
+    system_metadata: Optional[SystemMetadataClass],
+) -> MetadataWorkUnit:
+    patch_builder = DatasetPatchBuilder(urn, system_metadata)
+    for upstream in aspect.upstreams:
+        patch_builder.add_upstream_lineage(upstream)
+    mcp = next(iter(patch_builder.build()))
+    return MetadataWorkUnit(id=f"{urn}-upstreamLineage", mcp_raw=mcp)
+
+
+def _lineage_wu_via_read_modify_write(
+    graph: Optional[DataHubGraph],
+    urn: str,
+    aspect: UpstreamLineageClass,
+    system_metadata: Optional[SystemMetadataClass],
+) -> MetadataWorkUnit:
+    if graph is None:
+        raise ValueError(
+            "Failed to handle incremental lineage, DataHubGraph is missing. "
+            "Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
+        )
+    gms_aspect = graph.get_aspect(urn, UpstreamLineageClass)
+    if gms_aspect:
+        new_aspect = _merge_upstream_lineage(aspect, gms_aspect)
+    else:
+        new_aspect = aspect
+
+    return MetadataChangeProposalWrapper(
+        entityUrn=urn, aspect=new_aspect, systemMetadata=system_metadata
+    ).as_workunit()
+
+
+def _merge_upstream_lineage(
+    new_aspect: UpstreamLineageClass, gms_aspect: UpstreamLineageClass
+) -> UpstreamLineageClass:
+    merged_aspect = copy.deepcopy(gms_aspect)
+
+    upstreams_map: Dict[str, UpstreamClass] = {
+        upstream.dataset: upstream for upstream in merged_aspect.upstreams
+    }
+
+    upstreams_updated = False
+    fine_upstreams_updated = False
+
+    for table_upstream in new_aspect.upstreams:
+        if table_upstream.dataset not in upstreams_map or (
+            table_upstream.auditStamp.time
+            > upstreams_map[table_upstream.dataset].auditStamp.time
+        ):
+            upstreams_map[table_upstream.dataset] = table_upstream
+            upstreams_updated = True
+
+    if upstreams_updated:
+        merged_aspect.upstreams = list(upstreams_map.values())
+
+    if new_aspect.fineGrainedLineages and merged_aspect.fineGrainedLineages:
+        fine_upstreams_map: Dict[str, FineGrainedLineageClass] = {
+            get_fine_grained_lineage_key(fine_upstream): fine_upstream
+            for fine_upstream in merged_aspect.fineGrainedLineages
+        }
+        for column_upstream in new_aspect.fineGrainedLineages:
+            column_upstream_key = get_fine_grained_lineage_key(column_upstream)
+
+            if column_upstream_key not in fine_upstreams_map or (
+                column_upstream.confidenceScore
+                > fine_upstreams_map[column_upstream_key].confidenceScore
+            ):
+                fine_upstreams_map[column_upstream_key] = column_upstream
+                fine_upstreams_updated = True
+
+        if fine_upstreams_updated:
+            merged_aspect.fineGrainedLineages = list(fine_upstreams_map.values())
+    else:
+        merged_aspect.fineGrainedLineages = (
+            new_aspect.fineGrainedLineages or gms_aspect.fineGrainedLineages
+        )
+
+    return merged_aspect
+
+
+def get_fine_grained_lineage_key(fine_upstream: FineGrainedLineageClass) -> str:
+    return datahub_guid(
+        {
+            "upstreams": sorted(fine_upstream.upstreams or []),
+            "downstreams": sorted(fine_upstream.downstreams or []),
+            "transformOperation": fine_upstream.transformOperation,
+        }
+    )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index fee181864a2d6..b4a04d96b532b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -461,7 +461,8 @@ def _init_schema_resolver(self) -> SchemaResolver:
                 )
             else:
                 logger.warning(
-                    "Failed to load schema info from DataHub as DataHubGraph is missing.",
+                    "Failed to load schema info from DataHub as DataHubGraph is missing. "
+                    "Use `datahub-rest` sink OR provide `datahub-api` config in recipe. ",
                 )
         return SchemaResolver(platform=self.platform, env=self.config.env)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
index 215116b4c33fb..e0848b5f9ab34 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
@@ -4,6 +4,7 @@
 import os.path
 import platform
 from dataclasses import dataclass
+from functools import partial
 from typing import Callable, Dict, Iterable, List, Optional, Union
 
 import pandas as pd
@@ -35,6 +36,7 @@
     TestableSource,
     TestConnectionReport,
 )
+from datahub.ingestion.api.source_helpers import auto_incremental_lineage
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.glossary.classification_mixin import ClassificationHandler
 from datahub.ingestion.source.common.subtypes import (
@@ -511,6 +513,13 @@ def _init_schema_resolver(self) -> SchemaResolver:
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
+            partial(
+                auto_incremental_lineage,
+                self.ctx.graph,
+                self.config.incremental_lineage,
+                self.config.include_column_lineage
+                or self.config.include_view_column_lineage,
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
index 7687b99ac8d6d..5057dacd5b0c8 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
@@ -24,7 +24,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -39,7 +40,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -54,7 +56,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -71,7 +74,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -86,7 +90,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -115,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -130,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -145,7 +152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -162,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -177,7 +186,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -197,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -212,7 +223,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -375,7 +387,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -401,7 +414,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +430,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -433,7 +448,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -457,7 +473,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +489,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -635,7 +653,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -661,7 +680,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -676,7 +696,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +714,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -717,7 +739,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -732,7 +755,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -895,7 +919,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -921,7 +946,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -936,7 +962,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -953,7 +980,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -977,7 +1005,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -992,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1155,7 +1185,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1181,7 +1212,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1196,7 +1228,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1213,7 +1246,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1237,7 +1271,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1252,7 +1287,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1415,7 +1451,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1441,7 +1478,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1456,7 +1494,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1473,7 +1512,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1497,7 +1537,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1512,7 +1553,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1675,7 +1717,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1701,7 +1744,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1716,7 +1760,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1733,7 +1778,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1757,7 +1803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1772,7 +1819,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +1983,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1961,7 +2010,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1976,7 +2026,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1993,7 +2044,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2017,7 +2069,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2032,7 +2085,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2195,7 +2249,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2221,7 +2276,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2236,7 +2292,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2253,7 +2310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2277,7 +2335,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2292,7 +2351,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2455,7 +2515,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2481,7 +2542,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2496,7 +2558,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2513,7 +2576,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2537,7 +2601,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2552,7 +2617,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2715,7 +2781,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2741,7 +2808,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2756,7 +2824,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2773,7 +2842,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2797,7 +2867,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2821,7 +2892,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2845,7 +2917,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2869,7 +2942,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2893,7 +2967,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2917,7 +2992,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2941,7 +3017,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2965,7 +3042,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2989,7 +3067,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3013,7 +3092,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3037,7 +3117,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
index 2c77ace8b53e5..3dafe85ef950a 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
@@ -125,6 +125,7 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
                         validate_upstreams_against_patterns=False,
                         include_operational_stats=True,
                         email_as_user_identifier=True,
+                        incremental_lineage=False,
                         start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace(
                             tzinfo=timezone.utc
                         ),
@@ -213,6 +214,7 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_
                         include_views=False,
                         include_view_lineage=False,
                         include_usage_stats=False,
+                        incremental_lineage=False,
                         include_operational_stats=False,
                         start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace(
                             tzinfo=timezone.utc
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
index bba53c1e97a47..cd53b8f7db4f6 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
@@ -283,10 +283,12 @@ def test_snowflake_unexpected_snowflake_view_lineage_error_causes_pipeline_warni
         )
 
         snowflake_pipeline_config1 = snowflake_pipeline_config.copy()
-        cast(
+        config = cast(
             SnowflakeV2Config,
             cast(PipelineConfig, snowflake_pipeline_config1).source.config,
-        ).include_view_lineage = True
+        )
+        config.include_view_lineage = True
+        config.incremental_lineage = False
         pipeline = Pipeline(snowflake_pipeline_config1)
         pipeline.run()
         pipeline.raise_from_status()  # pipeline should not fail
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_stateful.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_stateful.py
index f72bd5b72d2cd..7e2ac94fa4e35 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_stateful.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_stateful.py
@@ -31,6 +31,7 @@ def stateful_pipeline_config(include_tables: bool) -> PipelineConfig:
                 match_fully_qualified_names=True,
                 schema_pattern=AllowDenyPattern(allow=["test_db.test_schema"]),
                 include_tables=include_tables,
+                incremental_lineage=False,
                 stateful_ingestion=StatefulStaleMetadataRemovalConfig.parse_obj(
                     {
                         "enabled": True,
@@ -49,7 +50,7 @@ def stateful_pipeline_config(include_tables: bool) -> PipelineConfig:
 
 
 @freeze_time(FROZEN_TIME)
-def test_tableau_stateful(mock_datahub_graph):
+def test_stale_metadata_removal(mock_datahub_graph):
     with mock.patch(
         "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
         mock_datahub_graph,
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_less_upstreams_in_gms_aspect_golden.json b/metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_less_upstreams_in_gms_aspect_golden.json
new file mode 100644
index 0000000000000..812566143014b
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_less_upstreams_in_gms_aspect_golden.json
@@ -0,0 +1,106 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_a)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_b)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_c)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_c)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_a)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_a)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_b)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_b)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_c)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_c)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_c)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "run-id",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_more_upstreams_in_gms_aspect_golden.json b/metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_more_upstreams_in_gms_aspect_golden.json
new file mode 100644
index 0000000000000..17f4d10728268
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/source_helpers/incremental_cll_more_upstreams_in_gms_aspect_golden.json
@@ -0,0 +1,120 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream3,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_a)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_a)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream3,PROD),col_a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_a)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_b)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_b)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream3,PROD),col_b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_b)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_c)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_c)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream3,PROD),col_c)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_c)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_a)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_a)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_a)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_b)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_b)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_b)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD),col_c)",
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD),col_c)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD),col_c)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "run-id",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/incremental_table_lineage_golden.json b/metadata-ingestion/tests/unit/api/source_helpers/incremental_table_lineage_golden.json
new file mode 100644
index 0000000000000..c828373c73080
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/source_helpers/incremental_table_lineage_golden.json
@@ -0,0 +1,41 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:platform,dataset1,PROD)",
+    "changeType": "PATCH",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aplatform%2Cupstream1%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream1,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            },
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aplatform%2Cupstream2%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:platform,upstream2,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "run-id",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
new file mode 100644
index 0000000000000..4078bda26c743
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
@@ -0,0 +1,244 @@
+from typing import List, Optional
+from unittest.mock import MagicMock
+
+import pytest
+
+import datahub.metadata.schema_classes as models
+from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.source_helpers import auto_incremental_lineage
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.sink.file import write_metadata_file
+from tests.test_helpers import mce_helpers
+
+platform = "platform"
+system_metadata = models.SystemMetadataClass(lastObserved=1643871600000, runId="run-id")
+
+
+def make_lineage_aspect(
+    dataset_name: str,
+    upstreams: List[str],
+    timestamp: int = 0,
+    columns: List[str] = [],
+    include_cll: bool = False,
+) -> models.UpstreamLineageClass:
+    """
+    Generates dataset properties and upstream lineage aspects
+    with simple column to column lineage between current dataset and all upstreams
+    """
+
+    dataset_urn = make_dataset_urn(platform, dataset_name)
+    return models.UpstreamLineageClass(
+        upstreams=[
+            models.UpstreamClass(
+                dataset=upstream_urn,
+                type=models.DatasetLineageTypeClass.TRANSFORMED,
+                auditStamp=models.AuditStampClass(
+                    time=timestamp, actor="urn:li:corpuser:unknown"
+                ),
+            )
+            for upstream_urn in upstreams
+        ],
+        fineGrainedLineages=[
+            models.FineGrainedLineageClass(
+                upstreamType=models.FineGrainedLineageUpstreamTypeClass.FIELD_SET,
+                downstreamType=models.FineGrainedLineageDownstreamTypeClass.FIELD,
+                upstreams=[
+                    make_schema_field_urn(upstream_urn, col)
+                    for upstream_urn in upstreams
+                ],
+                downstreams=[make_schema_field_urn(dataset_urn, col)],
+            )
+            for col in columns
+        ]
+        if include_cll
+        else None,
+    )
+
+
+def base_table_lineage_aspect() -> models.UpstreamLineageClass:
+    return make_lineage_aspect(
+        "dataset1",
+        upstreams=[
+            make_dataset_urn(platform, name) for name in ["upstream1", "upstream2"]
+        ],
+    )
+
+
+def base_cll_aspect(timestamp: int = 0) -> models.UpstreamLineageClass:
+    return make_lineage_aspect(
+        "dataset1",
+        upstreams=[
+            make_dataset_urn(platform, name) for name in ["upstream1", "upstream2"]
+        ],
+        timestamp=timestamp,
+        columns=["col_a", "col_b", "col_c"],
+        include_cll=True,
+    )
+
+
+def test_incremental_table_lineage(tmp_path, pytestconfig):
+    test_resources_dir = pytestconfig.rootpath / "tests/unit/api/source_helpers"
+    test_file = tmp_path / "incremental_table_lineage.json"
+    golden_file = test_resources_dir / "incremental_table_lineage_golden.json"
+
+    urn = make_dataset_urn(platform, "dataset1")
+    aspect = base_table_lineage_aspect()
+
+    processed_wus = auto_incremental_lineage(
+        graph=None,
+        incremental_lineage=True,
+        include_column_level_lineage=False,
+        stream=[
+            MetadataChangeProposalWrapper(
+                entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
+            ).as_workunit()
+        ],
+    )
+
+    write_metadata_file(
+        test_file,
+        [wu.metadata for wu in processed_wus],
+    )
+    mce_helpers.check_golden_file(
+        pytestconfig=pytestconfig, output_path=test_file, golden_path=golden_file
+    )
+
+
+@pytest.mark.parametrize(
+    "gms_aspect,current_aspect,output_aspect",
+    [
+        # emitting CLL upstreamLineage over table level upstreamLineage
+        [
+            base_table_lineage_aspect(),
+            base_cll_aspect(),
+            base_cll_aspect(),
+        ],
+        # emitting upstreamLineage for the first time
+        [
+            None,
+            base_cll_aspect(),
+            base_cll_aspect(),
+        ],
+        # emitting CLL upstreamLineage over same CLL upstreamLineage
+        [
+            base_cll_aspect(),
+            base_cll_aspect(),
+            base_cll_aspect(),
+        ],
+        # emitting CLL upstreamLineage over same CLL upstreamLineage but with earlier timestamp
+        [
+            base_cll_aspect(),  # default timestamp is 0
+            base_cll_aspect(timestamp=1643871600000),
+            base_cll_aspect(timestamp=1643871600000),
+        ],
+    ],
+)
+def test_incremental_column_level_lineage(
+    gms_aspect: Optional[models.UpstreamLineageClass],
+    current_aspect: models.UpstreamLineageClass,
+    output_aspect: models.UpstreamLineageClass,
+) -> None:
+    mock_graph = MagicMock()
+    mock_graph.get_aspect.return_value = gms_aspect
+    dataset_urn = make_dataset_urn(platform, "dataset1")
+
+    processed_wus = auto_incremental_lineage(
+        graph=mock_graph,
+        incremental_lineage=True,
+        include_column_level_lineage=True,
+        stream=[
+            MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=current_aspect,
+                systemMetadata=system_metadata,
+            ).as_workunit()
+        ],
+    )
+
+    wu: MetadataWorkUnit = next(iter(processed_wus))
+    aspect = wu.get_aspect_of_type(models.UpstreamLineageClass)
+    assert aspect == output_aspect
+
+
+def test_incremental_column_lineage_less_upstreams_in_gms_aspect(
+    tmp_path, pytestconfig
+):
+    test_resources_dir = pytestconfig.rootpath / "tests/unit/api/source_helpers"
+    test_file = tmp_path / "incremental_cll_less_upstreams_in_gms_aspect.json"
+    golden_file = (
+        test_resources_dir / "incremental_cll_less_upstreams_in_gms_aspect_golden.json"
+    )
+
+    urn = make_dataset_urn(platform, "dataset1")
+    aspect = base_cll_aspect()
+
+    mock_graph = MagicMock()
+    mock_graph.get_aspect.return_value = make_lineage_aspect(
+        "dataset1",
+        upstreams=[make_dataset_urn(platform, name) for name in ["upstream1"]],
+        columns=["col_a", "col_b", "col_c"],
+        include_cll=True,
+    )
+
+    processed_wus = auto_incremental_lineage(
+        graph=mock_graph,
+        incremental_lineage=True,
+        include_column_level_lineage=True,
+        stream=[
+            MetadataChangeProposalWrapper(
+                entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
+            ).as_workunit()
+        ],
+    )
+
+    write_metadata_file(
+        test_file,
+        [wu.metadata for wu in processed_wus],
+    )
+    mce_helpers.check_golden_file(
+        pytestconfig=pytestconfig, output_path=test_file, golden_path=golden_file
+    )
+
+
+def test_incremental_column_lineage_more_upstreams_in_gms_aspect(
+    tmp_path, pytestconfig
+):
+    test_resources_dir = pytestconfig.rootpath / "tests/unit/api/source_helpers"
+    test_file = tmp_path / "incremental_cll_more_upstreams_in_gms_aspect.json"
+    golden_file = (
+        test_resources_dir / "incremental_cll_more_upstreams_in_gms_aspect_golden.json"
+    )
+
+    urn = make_dataset_urn(platform, "dataset1")
+    aspect = base_cll_aspect()
+
+    mock_graph = MagicMock()
+    mock_graph.get_aspect.return_value = make_lineage_aspect(
+        "dataset1",
+        upstreams=[
+            make_dataset_urn(platform, name)
+            for name in ["upstream1", "upstream2", "upstream3"]
+        ],
+        columns=["col_a", "col_b", "col_c"],
+        include_cll=True,
+    )
+
+    processed_wus = auto_incremental_lineage(
+        graph=mock_graph,
+        incremental_lineage=True,
+        include_column_level_lineage=True,
+        stream=[
+            MetadataChangeProposalWrapper(
+                entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
+            ).as_workunit()
+        ],
+    )
+
+    write_metadata_file(
+        test_file,
+        [wu.metadata for wu in processed_wus],
+    )
+    mce_helpers.check_golden_file(
+        pytestconfig=pytestconfig, output_path=test_file, golden_path=golden_file
+    )
diff --git a/metadata-ingestion/tests/unit/test_source_helpers.py b/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
similarity index 100%
rename from metadata-ingestion/tests/unit/test_source_helpers.py
rename to metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py

From 57f855ecd11632e884b12fda0fc57e2694ee26a5 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Tue, 10 Oct 2023 12:18:21 +0530
Subject: [PATCH 022/259] feat(ingest): refactor + simplify incremental lineage
 helper (#8976)

---
 .../api/incremental_lineage_helper.py         | 139 ++++++++++++++++++
 .../datahub/ingestion/api/source_helpers.py   | 138 +----------------
 .../source/snowflake/snowflake_v2.py          |   4 +-
 .../test_incremental_lineage_helper.py        |   6 +-
 4 files changed, 142 insertions(+), 145 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py

diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
new file mode 100644
index 0000000000000..9478c5cf7efa2
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
@@ -0,0 +1,139 @@
+import copy
+from typing import Dict, Iterable, Optional
+
+from datahub.emitter.mce_builder import datahub_guid, set_aspect
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.metadata.schema_classes import (
+    FineGrainedLineageClass,
+    MetadataChangeEventClass,
+    SystemMetadataClass,
+    UpstreamClass,
+    UpstreamLineageClass,
+)
+from datahub.specific.dataset import DatasetPatchBuilder
+
+
+def _convert_upstream_lineage_to_patch(
+    urn: str,
+    aspect: UpstreamLineageClass,
+    system_metadata: Optional[SystemMetadataClass],
+) -> MetadataWorkUnit:
+    patch_builder = DatasetPatchBuilder(urn, system_metadata)
+    for upstream in aspect.upstreams:
+        patch_builder.add_upstream_lineage(upstream)
+    mcp = next(iter(patch_builder.build()))
+    return MetadataWorkUnit(id=f"{urn}-upstreamLineage", mcp_raw=mcp)
+
+
+def get_fine_grained_lineage_key(fine_upstream: FineGrainedLineageClass) -> str:
+    return datahub_guid(
+        {
+            "upstreams": sorted(fine_upstream.upstreams or []),
+            "downstreams": sorted(fine_upstream.downstreams or []),
+            "transformOperation": fine_upstream.transformOperation,
+        }
+    )
+
+
+def _merge_upstream_lineage(
+    new_aspect: UpstreamLineageClass, gms_aspect: UpstreamLineageClass
+) -> UpstreamLineageClass:
+    merged_aspect = copy.deepcopy(gms_aspect)
+
+    upstreams_map: Dict[str, UpstreamClass] = {
+        upstream.dataset: upstream for upstream in merged_aspect.upstreams
+    }
+
+    upstreams_updated = False
+    fine_upstreams_updated = False
+
+    for table_upstream in new_aspect.upstreams:
+        if table_upstream.dataset not in upstreams_map or (
+            table_upstream.auditStamp.time
+            > upstreams_map[table_upstream.dataset].auditStamp.time
+        ):
+            upstreams_map[table_upstream.dataset] = table_upstream
+            upstreams_updated = True
+
+    if upstreams_updated:
+        merged_aspect.upstreams = list(upstreams_map.values())
+
+    if new_aspect.fineGrainedLineages and merged_aspect.fineGrainedLineages:
+        fine_upstreams_map: Dict[str, FineGrainedLineageClass] = {
+            get_fine_grained_lineage_key(fine_upstream): fine_upstream
+            for fine_upstream in merged_aspect.fineGrainedLineages
+        }
+        for column_upstream in new_aspect.fineGrainedLineages:
+            column_upstream_key = get_fine_grained_lineage_key(column_upstream)
+
+            if column_upstream_key not in fine_upstreams_map or (
+                column_upstream.confidenceScore
+                > fine_upstreams_map[column_upstream_key].confidenceScore
+            ):
+                fine_upstreams_map[column_upstream_key] = column_upstream
+                fine_upstreams_updated = True
+
+        if fine_upstreams_updated:
+            merged_aspect.fineGrainedLineages = list(fine_upstreams_map.values())
+    else:
+        merged_aspect.fineGrainedLineages = (
+            new_aspect.fineGrainedLineages or gms_aspect.fineGrainedLineages
+        )
+
+    return merged_aspect
+
+
+def _lineage_wu_via_read_modify_write(
+    graph: Optional[DataHubGraph],
+    urn: str,
+    aspect: UpstreamLineageClass,
+    system_metadata: Optional[SystemMetadataClass],
+) -> MetadataWorkUnit:
+    if graph is None:
+        raise ValueError(
+            "Failed to handle incremental lineage, DataHubGraph is missing. "
+            "Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
+        )
+    gms_aspect = graph.get_aspect(urn, UpstreamLineageClass)
+    if gms_aspect:
+        new_aspect = _merge_upstream_lineage(aspect, gms_aspect)
+    else:
+        new_aspect = aspect
+
+    return MetadataChangeProposalWrapper(
+        entityUrn=urn, aspect=new_aspect, systemMetadata=system_metadata
+    ).as_workunit()
+
+
+def auto_incremental_lineage(
+    graph: Optional[DataHubGraph],
+    incremental_lineage: bool,
+    stream: Iterable[MetadataWorkUnit],
+) -> Iterable[MetadataWorkUnit]:
+    if not incremental_lineage:
+        yield from stream
+        return  # early exit
+
+    for wu in stream:
+        lineage_aspect: Optional[UpstreamLineageClass] = wu.get_aspect_of_type(
+            UpstreamLineageClass
+        )
+        urn = wu.get_urn()
+
+        if lineage_aspect:
+            if isinstance(wu.metadata, MetadataChangeEventClass):
+                set_aspect(
+                    wu.metadata, None, UpstreamLineageClass
+                )  # we'll emit upstreamLineage separately below
+                if len(wu.metadata.proposedSnapshot.aspects) > 0:
+                    yield wu
+
+            yield _lineage_wu_via_read_modify_write(
+                graph, urn, lineage_aspect, wu.metadata.systemMetadata
+            ) if lineage_aspect.fineGrainedLineages else _convert_upstream_lineage_to_patch(
+                urn, lineage_aspect, wu.metadata.systemMetadata
+            )
+        else:
+            yield wu
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
index 42f970e97c95f..7fc15cf829678 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
@@ -1,4 +1,3 @@
-import copy
 import logging
 from datetime import datetime, timezone
 from typing import (
@@ -16,14 +15,9 @@
 )
 
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
-from datahub.emitter.mce_builder import (
-    datahub_guid,
-    make_dataplatform_instance_urn,
-    set_aspect,
-)
+from datahub.emitter.mce_builder import make_dataplatform_instance_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import (
     BrowsePathEntryClass,
     BrowsePathsClass,
@@ -31,17 +25,12 @@
     ChangeTypeClass,
     ContainerClass,
     DatasetUsageStatisticsClass,
-    FineGrainedLineageClass,
     MetadataChangeEventClass,
     MetadataChangeProposalClass,
     StatusClass,
-    SystemMetadataClass,
     TagKeyClass,
     TimeWindowSizeClass,
-    UpstreamClass,
-    UpstreamLineageClass,
 )
-from datahub.specific.dataset import DatasetPatchBuilder
 from datahub.telemetry import telemetry
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.tag_urn import TagUrn
@@ -377,128 +366,3 @@ def _prepend_platform_instance(
         return [BrowsePathEntryClass(id=urn, urn=urn)] + entries
 
     return entries
-
-
-def auto_incremental_lineage(
-    graph: Optional[DataHubGraph],
-    incremental_lineage: bool,
-    include_column_level_lineage: bool,
-    stream: Iterable[MetadataWorkUnit],
-) -> Iterable[MetadataWorkUnit]:
-    if not incremental_lineage:
-        yield from stream
-        return  # early exit
-
-    for wu in stream:
-        lineage_aspect: Optional[UpstreamLineageClass] = wu.get_aspect_of_type(
-            UpstreamLineageClass
-        )
-        urn = wu.get_urn()
-
-        if lineage_aspect:
-            if isinstance(wu.metadata, MetadataChangeEventClass):
-                set_aspect(
-                    wu.metadata, None, UpstreamLineageClass
-                )  # we'll emit upstreamLineage separately below
-                if len(wu.metadata.proposedSnapshot.aspects) > 0:
-                    yield wu
-
-            yield _lineage_wu_via_read_modify_write(
-                graph, urn, lineage_aspect, wu.metadata.systemMetadata
-            ) if include_column_level_lineage else _convert_upstream_lineage_to_patch(
-                urn, lineage_aspect, wu.metadata.systemMetadata
-            )
-        else:
-            yield wu
-
-
-def _convert_upstream_lineage_to_patch(
-    urn: str,
-    aspect: UpstreamLineageClass,
-    system_metadata: Optional[SystemMetadataClass],
-) -> MetadataWorkUnit:
-    patch_builder = DatasetPatchBuilder(urn, system_metadata)
-    for upstream in aspect.upstreams:
-        patch_builder.add_upstream_lineage(upstream)
-    mcp = next(iter(patch_builder.build()))
-    return MetadataWorkUnit(id=f"{urn}-upstreamLineage", mcp_raw=mcp)
-
-
-def _lineage_wu_via_read_modify_write(
-    graph: Optional[DataHubGraph],
-    urn: str,
-    aspect: UpstreamLineageClass,
-    system_metadata: Optional[SystemMetadataClass],
-) -> MetadataWorkUnit:
-    if graph is None:
-        raise ValueError(
-            "Failed to handle incremental lineage, DataHubGraph is missing. "
-            "Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
-        )
-    gms_aspect = graph.get_aspect(urn, UpstreamLineageClass)
-    if gms_aspect:
-        new_aspect = _merge_upstream_lineage(aspect, gms_aspect)
-    else:
-        new_aspect = aspect
-
-    return MetadataChangeProposalWrapper(
-        entityUrn=urn, aspect=new_aspect, systemMetadata=system_metadata
-    ).as_workunit()
-
-
-def _merge_upstream_lineage(
-    new_aspect: UpstreamLineageClass, gms_aspect: UpstreamLineageClass
-) -> UpstreamLineageClass:
-    merged_aspect = copy.deepcopy(gms_aspect)
-
-    upstreams_map: Dict[str, UpstreamClass] = {
-        upstream.dataset: upstream for upstream in merged_aspect.upstreams
-    }
-
-    upstreams_updated = False
-    fine_upstreams_updated = False
-
-    for table_upstream in new_aspect.upstreams:
-        if table_upstream.dataset not in upstreams_map or (
-            table_upstream.auditStamp.time
-            > upstreams_map[table_upstream.dataset].auditStamp.time
-        ):
-            upstreams_map[table_upstream.dataset] = table_upstream
-            upstreams_updated = True
-
-    if upstreams_updated:
-        merged_aspect.upstreams = list(upstreams_map.values())
-
-    if new_aspect.fineGrainedLineages and merged_aspect.fineGrainedLineages:
-        fine_upstreams_map: Dict[str, FineGrainedLineageClass] = {
-            get_fine_grained_lineage_key(fine_upstream): fine_upstream
-            for fine_upstream in merged_aspect.fineGrainedLineages
-        }
-        for column_upstream in new_aspect.fineGrainedLineages:
-            column_upstream_key = get_fine_grained_lineage_key(column_upstream)
-
-            if column_upstream_key not in fine_upstreams_map or (
-                column_upstream.confidenceScore
-                > fine_upstreams_map[column_upstream_key].confidenceScore
-            ):
-                fine_upstreams_map[column_upstream_key] = column_upstream
-                fine_upstreams_updated = True
-
-        if fine_upstreams_updated:
-            merged_aspect.fineGrainedLineages = list(fine_upstreams_map.values())
-    else:
-        merged_aspect.fineGrainedLineages = (
-            new_aspect.fineGrainedLineages or gms_aspect.fineGrainedLineages
-        )
-
-    return merged_aspect
-
-
-def get_fine_grained_lineage_key(fine_upstream: FineGrainedLineageClass) -> str:
-    return datahub_guid(
-        {
-            "upstreams": sorted(fine_upstream.upstreams or []),
-            "downstreams": sorted(fine_upstream.downstreams or []),
-            "transformOperation": fine_upstream.transformOperation,
-        }
-    )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
index e0848b5f9ab34..a5c07d9a3870c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
@@ -27,6 +27,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
 from datahub.ingestion.api.source import (
     CapabilityReport,
     MetadataWorkUnitProcessor,
@@ -36,7 +37,6 @@
     TestableSource,
     TestConnectionReport,
 )
-from datahub.ingestion.api.source_helpers import auto_incremental_lineage
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.glossary.classification_mixin import ClassificationHandler
 from datahub.ingestion.source.common.subtypes import (
@@ -517,8 +517,6 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
                 auto_incremental_lineage,
                 self.ctx.graph,
                 self.config.incremental_lineage,
-                self.config.include_column_lineage
-                or self.config.include_view_column_lineage,
             ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
index 4078bda26c743..54a22d860285c 100644
--- a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
@@ -6,7 +6,7 @@
 import datahub.metadata.schema_classes as models
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.source_helpers import auto_incremental_lineage
+from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.sink.file import write_metadata_file
 from tests.test_helpers import mce_helpers
@@ -88,7 +88,6 @@ def test_incremental_table_lineage(tmp_path, pytestconfig):
     processed_wus = auto_incremental_lineage(
         graph=None,
         incremental_lineage=True,
-        include_column_level_lineage=False,
         stream=[
             MetadataChangeProposalWrapper(
                 entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
@@ -146,7 +145,6 @@ def test_incremental_column_level_lineage(
     processed_wus = auto_incremental_lineage(
         graph=mock_graph,
         incremental_lineage=True,
-        include_column_level_lineage=True,
         stream=[
             MetadataChangeProposalWrapper(
                 entityUrn=dataset_urn,
@@ -184,7 +182,6 @@ def test_incremental_column_lineage_less_upstreams_in_gms_aspect(
     processed_wus = auto_incremental_lineage(
         graph=mock_graph,
         incremental_lineage=True,
-        include_column_level_lineage=True,
         stream=[
             MetadataChangeProposalWrapper(
                 entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
@@ -227,7 +224,6 @@ def test_incremental_column_lineage_more_upstreams_in_gms_aspect(
     processed_wus = auto_incremental_lineage(
         graph=mock_graph,
         incremental_lineage=True,
-        include_column_level_lineage=True,
         stream=[
             MetadataChangeProposalWrapper(
                 entityUrn=urn, aspect=aspect, systemMetadata=system_metadata

From bb39d5418fcbf8bebbae1b510c63a1170865a072 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Tue, 10 Oct 2023 16:08:34 +0530
Subject: [PATCH 023/259] fix(lint): run black, isort (#8978)

---
 .../tests/assertions/assertions_test.py       |  33 ++--
 smoke-test/tests/browse/browse_test.py        |  51 +++++--
 smoke-test/tests/cli/datahub-cli.py           |  76 +++++++---
 smoke-test/tests/cli/datahub_graph_test.py    |  12 +-
 .../cli/delete_cmd/test_timeseries_delete.py  |  12 +-
 .../ingest_cmd/test_timeseries_rollback.py    |   6 +-
 .../cli/user_groups_cmd/test_group_cmd.py     |   3 +-
 smoke-test/tests/conftest.py                  |   4 +-
 smoke-test/tests/consistency_utils.py         |  16 +-
 .../tests/containers/containers_test.py       |   4 +-
 smoke-test/tests/cypress/integration_test.py  |  23 ++-
 .../tests/dataproduct/test_dataproduct.py     |   4 +-
 smoke-test/tests/delete/delete_test.py        |  18 +--
 .../tests/deprecation/deprecation_test.py     |   9 +-
 smoke-test/tests/domains/domains_test.py      |  15 +-
 .../managed_ingestion_test.py                 |   3 +-
 smoke-test/tests/patch/common_patch_tests.py  |  52 ++-----
 .../tests/patch/test_datajob_patches.py       |  23 +--
 .../tests/patch/test_dataset_patches.py       |  18 ++-
 smoke-test/tests/policies/test_policies.py    |  10 +-
 .../tests/setup/lineage/helper_classes.py     |   5 +-
 .../setup/lineage/ingest_data_job_change.py   |  42 ++----
 .../lineage/ingest_dataset_join_change.py     |  36 ++---
 .../lineage/ingest_input_datasets_change.py   |  42 ++----
 .../setup/lineage/ingest_time_lineage.py      |  18 ++-
 smoke-test/tests/setup/lineage/utils.py       |  85 +++++------
 .../tags-and-terms/tags_and_terms_test.py     |   4 +-
 smoke-test/tests/telemetry/telemetry_test.py  |   4 +-
 smoke-test/tests/test_result_msg.py           |  23 ++-
 smoke-test/tests/test_stateful_ingestion.py   |  14 +-
 smoke-test/tests/tests/tests_test.py          |   7 +-
 smoke-test/tests/timeline/timeline_test.py    |  67 +++++----
 .../tokens/revokable_access_token_test.py     |  12 +-
 smoke-test/tests/utils.py                     |  17 +--
 smoke-test/tests/views/views_test.py          | 142 +++++++++---------
 35 files changed, 457 insertions(+), 453 deletions(-)

diff --git a/smoke-test/tests/assertions/assertions_test.py b/smoke-test/tests/assertions/assertions_test.py
index 4aa64c512f684..48f3564e6cd97 100644
--- a/smoke-test/tests/assertions/assertions_test.py
+++ b/smoke-test/tests/assertions/assertions_test.py
@@ -2,28 +2,29 @@
 import urllib
 
 import pytest
-import requests_wrapper as requests
 import tenacity
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
 from datahub.ingestion.api.sink import NoopWriteCallback
 from datahub.ingestion.sink.file import FileSink, FileSinkConfig
-from datahub.metadata.com.linkedin.pegasus2avro.assertion import AssertionStdAggregation
-from datahub.metadata.schema_classes import (
-    AssertionInfoClass,
-    AssertionResultClass,
-    AssertionResultTypeClass,
-    AssertionRunEventClass,
-    AssertionRunStatusClass,
-    AssertionStdOperatorClass,
-    AssertionTypeClass,
-    DatasetAssertionInfoClass,
-    DatasetAssertionScopeClass,
-    PartitionSpecClass,
-    PartitionTypeClass,
-)
-from tests.utils import delete_urns_from_file, get_gms_url, ingest_file_via_rest, wait_for_healthcheck_util, get_sleep_info
+from datahub.metadata.com.linkedin.pegasus2avro.assertion import \
+    AssertionStdAggregation
+from datahub.metadata.schema_classes import (AssertionInfoClass,
+                                             AssertionResultClass,
+                                             AssertionResultTypeClass,
+                                             AssertionRunEventClass,
+                                             AssertionRunStatusClass,
+                                             AssertionStdOperatorClass,
+                                             AssertionTypeClass,
+                                             DatasetAssertionInfoClass,
+                                             DatasetAssertionScopeClass,
+                                             PartitionSpecClass,
+                                             PartitionTypeClass)
+
+import requests_wrapper as requests
+from tests.utils import (delete_urns_from_file, get_gms_url, get_sleep_info,
+                         ingest_file_via_rest, wait_for_healthcheck_util)
 
 restli_default_headers = {
     "X-RestLi-Protocol-Version": "2.0.0",
diff --git a/smoke-test/tests/browse/browse_test.py b/smoke-test/tests/browse/browse_test.py
index b9d2143d13ec7..550f0062d5a39 100644
--- a/smoke-test/tests/browse/browse_test.py
+++ b/smoke-test/tests/browse/browse_test.py
@@ -1,9 +1,10 @@
 import time
 
 import pytest
-import requests_wrapper as requests
-from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest
 
+import requests_wrapper as requests
+from tests.utils import (delete_urns_from_file, get_frontend_url,
+                         ingest_file_via_rest)
 
 TEST_DATASET_1_URN = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-browse-1,PROD)"
 TEST_DATASET_2_URN = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-browse-2,PROD)"
@@ -51,7 +52,9 @@ def test_get_browse_paths(frontend_session, ingest_cleanup_data):
     # /prod -- There should be one entity
     get_browse_paths_json = {
         "query": get_browse_paths_query,
-        "variables": {"input": { "type": "DATASET", "path": ["prod"], "start": 0, "count": 100 } },
+        "variables": {
+            "input": {"type": "DATASET", "path": ["prod"], "start": 0, "count": 100}
+        },
     }
 
     response = frontend_session.post(
@@ -67,12 +70,19 @@ def test_get_browse_paths(frontend_session, ingest_cleanup_data):
 
     browse = res_data["data"]["browse"]
     print(browse)
-    assert browse["entities"] == [{ "urn": TEST_DATASET_3_URN }]
+    assert browse["entities"] == [{"urn": TEST_DATASET_3_URN}]
 
     # /prod/kafka1
     get_browse_paths_json = {
         "query": get_browse_paths_query,
-        "variables": {"input": { "type": "DATASET", "path": ["prod", "kafka1"], "start": 0, "count": 10 } },
+        "variables": {
+            "input": {
+                "type": "DATASET",
+                "path": ["prod", "kafka1"],
+                "start": 0,
+                "count": 10,
+            }
+        },
     }
 
     response = frontend_session.post(
@@ -88,16 +98,27 @@ def test_get_browse_paths(frontend_session, ingest_cleanup_data):
 
     browse = res_data["data"]["browse"]
     assert browse == {
-      "total": 3,
-      "entities": [{ "urn": TEST_DATASET_1_URN }, { "urn": TEST_DATASET_2_URN }, { "urn": TEST_DATASET_3_URN }],
-      "groups": [],
-      "metadata": { "path": ["prod", "kafka1"], "totalNumEntities": 0 }
+        "total": 3,
+        "entities": [
+            {"urn": TEST_DATASET_1_URN},
+            {"urn": TEST_DATASET_2_URN},
+            {"urn": TEST_DATASET_3_URN},
+        ],
+        "groups": [],
+        "metadata": {"path": ["prod", "kafka1"], "totalNumEntities": 0},
     }
 
     # /prod/kafka2
     get_browse_paths_json = {
         "query": get_browse_paths_query,
-        "variables": {"input": { "type": "DATASET", "path": ["prod", "kafka2"], "start": 0, "count": 10 } },
+        "variables": {
+            "input": {
+                "type": "DATASET",
+                "path": ["prod", "kafka2"],
+                "start": 0,
+                "count": 10,
+            }
+        },
     }
 
     response = frontend_session.post(
@@ -113,10 +134,8 @@ def test_get_browse_paths(frontend_session, ingest_cleanup_data):
 
     browse = res_data["data"]["browse"]
     assert browse == {
-      "total": 2,
-      "entities": [{ "urn": TEST_DATASET_1_URN }, { "urn": TEST_DATASET_2_URN }],
-      "groups": [],
-      "metadata": { "path": ["prod", "kafka2"], "totalNumEntities": 0 }
+        "total": 2,
+        "entities": [{"urn": TEST_DATASET_1_URN}, {"urn": TEST_DATASET_2_URN}],
+        "groups": [],
+        "metadata": {"path": ["prod", "kafka2"], "totalNumEntities": 0},
     }
-
-
diff --git a/smoke-test/tests/cli/datahub-cli.py b/smoke-test/tests/cli/datahub-cli.py
index 1d0080bdd9d48..c3db6028efceb 100644
--- a/smoke-test/tests/cli/datahub-cli.py
+++ b/smoke-test/tests/cli/datahub-cli.py
@@ -1,8 +1,11 @@
 import json
-import pytest
 from time import sleep
-from datahub.cli.cli_utils import guess_entity_type, post_entity, get_aspects_for_entity
+
+import pytest
+from datahub.cli.cli_utils import (get_aspects_for_entity, guess_entity_type,
+                                   post_entity)
 from datahub.cli.ingest_cli import get_session_and_host, rollback
+
 from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 ingested_dataset_run_id = ""
@@ -24,24 +27,46 @@ def test_setup():
 
     session, gms_host = get_session_and_host()
 
-    assert "browsePaths" not in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["browsePaths"], typed=False)
-    assert "editableDatasetProperties" not in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False)
+    assert "browsePaths" not in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["browsePaths"], typed=False
+    )
+    assert "editableDatasetProperties" not in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False
+    )
 
-    ingested_dataset_run_id = ingest_file_via_rest("tests/cli/cli_test_data.json").config.run_id
+    ingested_dataset_run_id = ingest_file_via_rest(
+        "tests/cli/cli_test_data.json"
+    ).config.run_id
     print("Setup ingestion id: " + ingested_dataset_run_id)
 
-    assert "browsePaths" in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["browsePaths"], typed=False)
+    assert "browsePaths" in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["browsePaths"], typed=False
+    )
 
     yield
 
     # Clean up
     rollback_url = f"{gms_host}/runs?action=rollback"
 
-    session.post(rollback_url, data=json.dumps({"runId": ingested_editable_run_id, "dryRun": False, "hardDelete": True}))
-    session.post(rollback_url, data=json.dumps({"runId": ingested_dataset_run_id, "dryRun": False, "hardDelete": True}))
+    session.post(
+        rollback_url,
+        data=json.dumps(
+            {"runId": ingested_editable_run_id, "dryRun": False, "hardDelete": True}
+        ),
+    )
+    session.post(
+        rollback_url,
+        data=json.dumps(
+            {"runId": ingested_dataset_run_id, "dryRun": False, "hardDelete": True}
+        ),
+    )
 
-    assert "browsePaths" not in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["browsePaths"], typed=False)
-    assert "editableDatasetProperties" not in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False)
+    assert "browsePaths" not in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["browsePaths"], typed=False
+    )
+    assert "editableDatasetProperties" not in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False
+    )
 
 
 @pytest.mark.dependency()
@@ -49,9 +74,7 @@ def test_rollback_editable():
     global ingested_dataset_run_id
     global ingested_editable_run_id
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-rollback"
-    )
+    dataset_name = "test-rollback"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
@@ -59,23 +82,38 @@ def test_rollback_editable():
 
     print("Ingested dataset id:", ingested_dataset_run_id)
     # Assert that second data ingestion worked
-    assert "browsePaths" in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["browsePaths"], typed=False)
+    assert "browsePaths" in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["browsePaths"], typed=False
+    )
 
     # Make editable change
-    ingested_editable_run_id = ingest_file_via_rest("tests/cli/cli_editable_test_data.json").config.run_id
+    ingested_editable_run_id = ingest_file_via_rest(
+        "tests/cli/cli_editable_test_data.json"
+    ).config.run_id
     print("ingested editable id:", ingested_editable_run_id)
     # Assert that second data ingestion worked
-    assert "editableDatasetProperties" in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False)
+    assert "editableDatasetProperties" in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False
+    )
 
     # rollback ingestion 1
     rollback_url = f"{gms_host}/runs?action=rollback"
 
-    session.post(rollback_url, data=json.dumps({"runId": ingested_dataset_run_id, "dryRun": False, "hardDelete": False}))
+    session.post(
+        rollback_url,
+        data=json.dumps(
+            {"runId": ingested_dataset_run_id, "dryRun": False, "hardDelete": False}
+        ),
+    )
 
     # Allow async MCP processor to handle ingestions & rollbacks
     wait_for_writes_to_sync()
 
     # EditableDatasetProperties should still be part of the entity that was soft deleted.
-    assert "editableDatasetProperties" in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False)
+    assert "editableDatasetProperties" in get_aspects_for_entity(
+        entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False
+    )
     # But first ingestion aspects should not be present
-    assert "browsePaths" not in get_aspects_for_entity(entity_urn=dataset_urn, typed=False)
+    assert "browsePaths" not in get_aspects_for_entity(
+        entity_urn=dataset_urn, typed=False
+    )
diff --git a/smoke-test/tests/cli/datahub_graph_test.py b/smoke-test/tests/cli/datahub_graph_test.py
index 16925d26f6983..17c8924fb0998 100644
--- a/smoke-test/tests/cli/datahub_graph_test.py
+++ b/smoke-test/tests/cli/datahub_graph_test.py
@@ -1,13 +1,11 @@
 import pytest
 import tenacity
 from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
-from datahub.metadata.schema_classes import KafkaSchemaClass, SchemaMetadataClass
-from tests.utils import (
-    delete_urns_from_file,
-    get_gms_url,
-    get_sleep_info,
-    ingest_file_via_rest,
-)
+from datahub.metadata.schema_classes import (KafkaSchemaClass,
+                                             SchemaMetadataClass)
+
+from tests.utils import (delete_urns_from_file, get_gms_url, get_sleep_info,
+                         ingest_file_via_rest)
 
 sleep_sec, sleep_times = get_sleep_info()
 
diff --git a/smoke-test/tests/cli/delete_cmd/test_timeseries_delete.py b/smoke-test/tests/cli/delete_cmd/test_timeseries_delete.py
index 4288a61b7a0c1..106da7cd8d71e 100644
--- a/smoke-test/tests/cli/delete_cmd/test_timeseries_delete.py
+++ b/smoke-test/tests/cli/delete_cmd/test_timeseries_delete.py
@@ -1,21 +1,22 @@
 import json
 import logging
+import sys
 import tempfile
 import time
-import sys
 from json import JSONDecodeError
 from typing import Any, Dict, List, Optional
 
-from click.testing import CliRunner, Result
-
 import datahub.emitter.mce_builder as builder
+from click.testing import CliRunner, Result
 from datahub.emitter.serialization_helper import pre_json_transform
 from datahub.entrypoints import datahub
 from datahub.metadata.schema_classes import DatasetProfileClass
+
+import requests_wrapper as requests
 from tests.aspect_generators.timeseries.dataset_profile_gen import \
     gen_dataset_profiles
-from tests.utils import get_strftime_from_timestamp_millis, wait_for_writes_to_sync
-import requests_wrapper as requests
+from tests.utils import (get_strftime_from_timestamp_millis,
+                         wait_for_writes_to_sync)
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +34,7 @@
 def sync_elastic() -> None:
     wait_for_writes_to_sync()
 
+
 def datahub_put_profile(dataset_profile: DatasetProfileClass) -> None:
     with tempfile.NamedTemporaryFile("w+t", suffix=".json") as aspect_file:
         aspect_text: str = json.dumps(pre_json_transform(dataset_profile.to_obj()))
diff --git a/smoke-test/tests/cli/ingest_cmd/test_timeseries_rollback.py b/smoke-test/tests/cli/ingest_cmd/test_timeseries_rollback.py
index 61e7a5a65b494..e962b1a5cafd6 100644
--- a/smoke-test/tests/cli/ingest_cmd/test_timeseries_rollback.py
+++ b/smoke-test/tests/cli/ingest_cmd/test_timeseries_rollback.py
@@ -2,14 +2,14 @@
 import time
 from typing import Any, Dict, List, Optional
 
-from click.testing import CliRunner, Result
-
 import datahub.emitter.mce_builder as builder
+from click.testing import CliRunner, Result
 from datahub.emitter.serialization_helper import post_json_transform
 from datahub.entrypoints import datahub
 from datahub.metadata.schema_classes import DatasetProfileClass
-from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
+
 import requests_wrapper as requests
+from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync
 
 runner = CliRunner(mix_stderr=False)
 
diff --git a/smoke-test/tests/cli/user_groups_cmd/test_group_cmd.py b/smoke-test/tests/cli/user_groups_cmd/test_group_cmd.py
index 405e061c016f9..7b986d3be0444 100644
--- a/smoke-test/tests/cli/user_groups_cmd/test_group_cmd.py
+++ b/smoke-test/tests/cli/user_groups_cmd/test_group_cmd.py
@@ -1,6 +1,7 @@
 import json
 import sys
 import tempfile
+import time
 from typing import Any, Dict, Iterable, List
 
 import yaml
@@ -8,7 +9,7 @@
 from datahub.api.entities.corpgroup.corpgroup import CorpGroup
 from datahub.entrypoints import datahub
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
-import time
+
 import requests_wrapper as requests
 from tests.utils import wait_for_writes_to_sync
 
diff --git a/smoke-test/tests/conftest.py b/smoke-test/tests/conftest.py
index eed7a983197ef..57b92a2db1c19 100644
--- a/smoke-test/tests/conftest.py
+++ b/smoke-test/tests/conftest.py
@@ -2,8 +2,8 @@
 
 import pytest
 
-from tests.utils import wait_for_healthcheck_util, get_frontend_session
 from tests.test_result_msg import send_message
+from tests.utils import get_frontend_session, wait_for_healthcheck_util
 
 # Disable telemetry
 os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
@@ -28,5 +28,5 @@ def test_healthchecks(wait_for_healthchecks):
 
 
 def pytest_sessionfinish(session, exitstatus):
-    """ whole test run finishes. """
+    """whole test run finishes."""
     send_message(exitstatus)
diff --git a/smoke-test/tests/consistency_utils.py b/smoke-test/tests/consistency_utils.py
index 15993733c592b..607835bf3649c 100644
--- a/smoke-test/tests/consistency_utils.py
+++ b/smoke-test/tests/consistency_utils.py
@@ -1,10 +1,16 @@
-import time
+import logging
 import os
 import subprocess
+import time
 
 _ELASTIC_BUFFER_WRITES_TIME_IN_SEC: int = 1
 USE_STATIC_SLEEP: bool = bool(os.getenv("USE_STATIC_SLEEP", False))
-ELASTICSEARCH_REFRESH_INTERVAL_SECONDS: int = int(os.getenv("ELASTICSEARCH_REFRESH_INTERVAL_SECONDS", 5))
+ELASTICSEARCH_REFRESH_INTERVAL_SECONDS: int = int(
+    os.getenv("ELASTICSEARCH_REFRESH_INTERVAL_SECONDS", 5)
+)
+
+logger = logging.getLogger(__name__)
+
 
 def wait_for_writes_to_sync(max_timeout_in_sec: int = 120) -> None:
     if USE_STATIC_SLEEP:
@@ -30,7 +36,9 @@ def wait_for_writes_to_sync(max_timeout_in_sec: int = 120) -> None:
             lag_zero = True
 
     if not lag_zero:
-        logger.warning(f"Exiting early from waiting for elastic to catch up due to a timeout. Current lag is {lag_values}")
+        logger.warning(
+            f"Exiting early from waiting for elastic to catch up due to a timeout. Current lag is {lag_values}"
+        )
     else:
         # we want to sleep for an additional period of time for Elastic writes buffer to clear
-        time.sleep(_ELASTIC_BUFFER_WRITES_TIME_IN_SEC)
\ No newline at end of file
+        time.sleep(_ELASTIC_BUFFER_WRITES_TIME_IN_SEC)
diff --git a/smoke-test/tests/containers/containers_test.py b/smoke-test/tests/containers/containers_test.py
index 575e3def6cf23..05a45239dabf8 100644
--- a/smoke-test/tests/containers/containers_test.py
+++ b/smoke-test/tests/containers/containers_test.py
@@ -1,5 +1,7 @@
 import pytest
-from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest
+
+from tests.utils import (delete_urns_from_file, get_frontend_url,
+                         ingest_file_via_rest)
 
 
 @pytest.fixture(scope="module", autouse=False)
diff --git a/smoke-test/tests/cypress/integration_test.py b/smoke-test/tests/cypress/integration_test.py
index b3bacf39ac7ae..4ad2bc53fa87d 100644
--- a/smoke-test/tests/cypress/integration_test.py
+++ b/smoke-test/tests/cypress/integration_test.py
@@ -1,18 +1,16 @@
-from typing import Set, List
-
 import datetime
-import pytest
-import subprocess
 import os
+import subprocess
+from typing import List, Set
+
+import pytest
+
+from tests.setup.lineage.ingest_time_lineage import (get_time_lineage_urns,
+                                                     ingest_time_lineage)
+from tests.utils import (create_datahub_step_state_aspects, delete_urns,
+                         delete_urns_from_file, get_admin_username,
+                         ingest_file_via_rest)
 
-from tests.utils import (
-    create_datahub_step_state_aspects,
-    get_admin_username,
-    ingest_file_via_rest,
-    delete_urns_from_file,
-    delete_urns,
-)
-from tests.setup.lineage.ingest_time_lineage import ingest_time_lineage, get_time_lineage_urns
 CYPRESS_TEST_DATA_DIR = "tests/cypress"
 
 TEST_DATA_FILENAME = "data.json"
@@ -145,7 +143,6 @@ def ingest_cleanup_data():
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
     delete_urns(get_time_lineage_urns())
 
-
     print_now()
     print("deleting onboarding data file")
     if os.path.exists(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}"):
diff --git a/smoke-test/tests/dataproduct/test_dataproduct.py b/smoke-test/tests/dataproduct/test_dataproduct.py
index db198098f21fa..baef1cb1cb3ba 100644
--- a/smoke-test/tests/dataproduct/test_dataproduct.py
+++ b/smoke-test/tests/dataproduct/test_dataproduct.py
@@ -1,4 +1,6 @@
+import logging
 import os
+import subprocess
 import tempfile
 import time
 from random import randint
@@ -17,8 +19,6 @@
                                              DomainPropertiesClass,
                                              DomainsClass)
 from datahub.utilities.urns.urn import Urn
-import subprocess
-import logging
 
 logger = logging.getLogger(__name__)
 
diff --git a/smoke-test/tests/delete/delete_test.py b/smoke-test/tests/delete/delete_test.py
index 68e001f983fbf..d920faaf3a89a 100644
--- a/smoke-test/tests/delete/delete_test.py
+++ b/smoke-test/tests/delete/delete_test.py
@@ -1,16 +1,14 @@
-import os
 import json
-import pytest
+import os
 from time import sleep
+
+import pytest
 from datahub.cli.cli_utils import get_aspects_for_entity
 from datahub.cli.ingest_cli import get_session_and_host
-from tests.utils import (
-    ingest_file_via_rest,
-    wait_for_healthcheck_util,
-    delete_urns_from_file,
-    wait_for_writes_to_sync,
-    get_datahub_graph,
-)
+
+from tests.utils import (delete_urns_from_file, get_datahub_graph,
+                         ingest_file_via_rest, wait_for_healthcheck_util,
+                         wait_for_writes_to_sync)
 
 # Disable telemetry
 os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
@@ -102,7 +100,7 @@ def test_delete_reference(test_setup, depends=["test_healthchecks"]):
     graph.delete_references_to_urn(tag_urn, dry_run=False)
 
     wait_for_writes_to_sync()
-    
+
     # Validate that references no longer exist
     references_count, related_aspects = graph.delete_references_to_urn(
         tag_urn, dry_run=True
diff --git a/smoke-test/tests/deprecation/deprecation_test.py b/smoke-test/tests/deprecation/deprecation_test.py
index 1149a970aa8e5..a8969804d03d7 100644
--- a/smoke-test/tests/deprecation/deprecation_test.py
+++ b/smoke-test/tests/deprecation/deprecation_test.py
@@ -1,10 +1,7 @@
 import pytest
-from tests.utils import (
-    delete_urns_from_file,
-    get_frontend_url,
-    ingest_file_via_rest,
-    get_root_urn,
-)
+
+from tests.utils import (delete_urns_from_file, get_frontend_url, get_root_urn,
+                         ingest_file_via_rest)
 
 
 @pytest.fixture(scope="module", autouse=True)
diff --git a/smoke-test/tests/domains/domains_test.py b/smoke-test/tests/domains/domains_test.py
index 7ffe1682cafd8..fa8c918e3cbe1 100644
--- a/smoke-test/tests/domains/domains_test.py
+++ b/smoke-test/tests/domains/domains_test.py
@@ -1,12 +1,8 @@
 import pytest
 import tenacity
-from tests.utils import (
-    delete_urns_from_file,
-    get_frontend_url,
-    get_gms_url,
-    ingest_file_via_rest,
-    get_sleep_info,
-)
+
+from tests.utils import (delete_urns_from_file, get_frontend_url, get_gms_url,
+                         get_sleep_info, ingest_file_via_rest)
 
 sleep_sec, sleep_times = get_sleep_info()
 
@@ -240,4 +236,7 @@ def test_set_unset_domain(frontend_session, ingest_cleanup_data):
 
     assert res_data
     assert res_data["data"]["dataset"]["domain"]["domain"]["urn"] == domain_urn
-    assert res_data["data"]["dataset"]["domain"]["domain"]["properties"]["name"] == "Engineering"
+    assert (
+        res_data["data"]["dataset"]["domain"]["domain"]["properties"]["name"]
+        == "Engineering"
+    )
diff --git a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py
index 1238a1dd5730a..b5e408731334e 100644
--- a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py
+++ b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py
@@ -3,7 +3,8 @@
 import pytest
 import tenacity
 
-from tests.utils import get_frontend_url, get_sleep_info, wait_for_healthcheck_util
+from tests.utils import (get_frontend_url, get_sleep_info,
+                         wait_for_healthcheck_util)
 
 sleep_sec, sleep_times = get_sleep_info()
 
diff --git a/smoke-test/tests/patch/common_patch_tests.py b/smoke-test/tests/patch/common_patch_tests.py
index 574e4fd4e4c88..f1d6abf5da794 100644
--- a/smoke-test/tests/patch/common_patch_tests.py
+++ b/smoke-test/tests/patch/common_patch_tests.py
@@ -2,25 +2,17 @@
 import uuid
 from typing import Dict, Optional, Type
 
-from datahub.emitter.mce_builder import (
-    make_tag_urn,
-    make_term_urn,
-    make_user_urn,
-)
+from datahub.emitter.mce_builder import (make_tag_urn, make_term_urn,
+                                         make_user_urn)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
 from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
-from datahub.metadata.schema_classes import (
-    AuditStampClass,
-    GlobalTagsClass,
-    GlossaryTermAssociationClass,
-    GlossaryTermsClass,
-    OwnerClass,
-    OwnershipClass,
-    OwnershipTypeClass,
-    TagAssociationClass,
-    _Aspect,
-)
+from datahub.metadata.schema_classes import (AuditStampClass, GlobalTagsClass,
+                                             GlossaryTermAssociationClass,
+                                             GlossaryTermsClass, OwnerClass,
+                                             OwnershipClass,
+                                             OwnershipTypeClass,
+                                             TagAssociationClass, _Aspect)
 
 
 def helper_test_entity_terms_patch(
@@ -34,18 +26,14 @@ def get_terms(graph, entity_urn):
 
     term_urn = make_term_urn(term=f"testTerm-{uuid.uuid4()}")
 
-    term_association = GlossaryTermAssociationClass(
-        urn=term_urn, context="test"
-    )
+    term_association = GlossaryTermAssociationClass(urn=term_urn, context="test")
     global_terms = GlossaryTermsClass(
         terms=[term_association],
         auditStamp=AuditStampClass(
             time=int(time.time() * 1000.0), actor=make_user_urn("tester")
         ),
     )
-    mcpw = MetadataChangeProposalWrapper(
-        entityUrn=test_entity_urn, aspect=global_terms
-    )
+    mcpw = MetadataChangeProposalWrapper(entityUrn=test_entity_urn, aspect=global_terms)
 
     with DataHubGraph(DataHubGraphConfig()) as graph:
         graph.emit_mcp(mcpw)
@@ -88,9 +76,7 @@ def helper_test_dataset_tags_patch(
 
     tag_association = TagAssociationClass(tag=tag_urn, context="test")
     global_tags = GlobalTagsClass(tags=[tag_association])
-    mcpw = MetadataChangeProposalWrapper(
-        entityUrn=test_entity_urn, aspect=global_tags
-    )
+    mcpw = MetadataChangeProposalWrapper(entityUrn=test_entity_urn, aspect=global_tags)
 
     with DataHubGraph(DataHubGraphConfig()) as graph:
         graph.emit_mcp(mcpw)
@@ -153,15 +139,11 @@ def helper_test_ownership_patch(
         assert owner.owners[0].owner == make_user_urn("jdoe")
 
         for patch_mcp in (
-            patch_builder_class(test_entity_urn)
-            .add_owner(owner_to_add)
-            .build()
+            patch_builder_class(test_entity_urn).add_owner(owner_to_add).build()
         ):
             graph.emit_mcp(patch_mcp)
 
-        owner = graph.get_aspect(
-            entity_urn=test_entity_urn, aspect_type=OwnershipClass
-        )
+        owner = graph.get_aspect(entity_urn=test_entity_urn, aspect_type=OwnershipClass)
         assert len(owner.owners) == 2
 
         for patch_mcp in (
@@ -171,9 +153,7 @@ def helper_test_ownership_patch(
         ):
             graph.emit_mcp(patch_mcp)
 
-        owner = graph.get_aspect(
-            entity_urn=test_entity_urn, aspect_type=OwnershipClass
-        )
+        owner = graph.get_aspect(entity_urn=test_entity_urn, aspect_type=OwnershipClass)
         assert len(owner.owners) == 1
         assert owner.owners[0].owner == make_user_urn("jdoe")
 
@@ -199,9 +179,7 @@ def get_custom_properties(
     orig_aspect = base_aspect
     assert hasattr(orig_aspect, "customProperties")
     orig_aspect.customProperties = base_property_map
-    mcpw = MetadataChangeProposalWrapper(
-        entityUrn=test_entity_urn, aspect=orig_aspect
-    )
+    mcpw = MetadataChangeProposalWrapper(entityUrn=test_entity_urn, aspect=orig_aspect)
 
     with DataHubGraph(DataHubGraphConfig()) as graph:
         graph.emit(mcpw)
diff --git a/smoke-test/tests/patch/test_datajob_patches.py b/smoke-test/tests/patch/test_datajob_patches.py
index 407410ee89914..342d5d683228a 100644
--- a/smoke-test/tests/patch/test_datajob_patches.py
+++ b/smoke-test/tests/patch/test_datajob_patches.py
@@ -3,19 +3,14 @@
 from datahub.emitter.mce_builder import make_data_job_urn, make_dataset_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig
-from datahub.metadata.schema_classes import (
-    DataJobInfoClass,
-    DataJobInputOutputClass,
-    EdgeClass,
-)
+from datahub.metadata.schema_classes import (DataJobInfoClass,
+                                             DataJobInputOutputClass,
+                                             EdgeClass)
 from datahub.specific.datajob import DataJobPatchBuilder
 
 from tests.patch.common_patch_tests import (
-    helper_test_custom_properties_patch,
-    helper_test_dataset_tags_patch,
-    helper_test_entity_terms_patch,
-    helper_test_ownership_patch,
-)
+    helper_test_custom_properties_patch, helper_test_dataset_tags_patch,
+    helper_test_entity_terms_patch, helper_test_ownership_patch)
 
 
 def _make_test_datajob_urn(
@@ -37,16 +32,12 @@ def test_datajob_ownership_patch(wait_for_healthchecks):
 
 # Tags
 def test_datajob_tags_patch(wait_for_healthchecks):
-    helper_test_dataset_tags_patch(
-        _make_test_datajob_urn(), DataJobPatchBuilder
-    )
+    helper_test_dataset_tags_patch(_make_test_datajob_urn(), DataJobPatchBuilder)
 
 
 # Terms
 def test_dataset_terms_patch(wait_for_healthchecks):
-    helper_test_entity_terms_patch(
-        _make_test_datajob_urn(), DataJobPatchBuilder
-    )
+    helper_test_entity_terms_patch(_make_test_datajob_urn(), DataJobPatchBuilder)
 
 
 # Custom Properties
diff --git a/smoke-test/tests/patch/test_dataset_patches.py b/smoke-test/tests/patch/test_dataset_patches.py
index 239aab64675d8..6704d19760fb9 100644
--- a/smoke-test/tests/patch/test_dataset_patches.py
+++ b/smoke-test/tests/patch/test_dataset_patches.py
@@ -20,7 +20,10 @@
                                              UpstreamClass,
                                              UpstreamLineageClass)
 from datahub.specific.dataset import DatasetPatchBuilder
-from tests.patch.common_patch_tests import helper_test_entity_terms_patch, helper_test_dataset_tags_patch, helper_test_ownership_patch, helper_test_custom_properties_patch
+
+from tests.patch.common_patch_tests import (
+    helper_test_custom_properties_patch, helper_test_dataset_tags_patch,
+    helper_test_entity_terms_patch, helper_test_ownership_patch)
 
 
 # Common Aspect Patch Tests
@@ -31,6 +34,7 @@ def test_dataset_ownership_patch(wait_for_healthchecks):
     )
     helper_test_ownership_patch(dataset_urn, DatasetPatchBuilder)
 
+
 # Tags
 def test_dataset_tags_patch(wait_for_healthchecks):
     dataset_urn = make_dataset_urn(
@@ -38,6 +42,7 @@ def test_dataset_tags_patch(wait_for_healthchecks):
     )
     helper_test_dataset_tags_patch(dataset_urn, DatasetPatchBuilder)
 
+
 # Terms
 def test_dataset_terms_patch(wait_for_healthchecks):
     dataset_urn = make_dataset_urn(
@@ -284,8 +289,15 @@ def test_custom_properties_patch(wait_for_healthchecks):
     dataset_urn = make_dataset_urn(
         platform="hive", name=f"SampleHiveDataset-{uuid.uuid4()}", env="PROD"
     )
-    orig_dataset_properties = DatasetPropertiesClass(name="test_name", description="test_description")
-    helper_test_custom_properties_patch(test_entity_urn=dataset_urn, patch_builder_class=DatasetPatchBuilder, custom_properties_aspect_class=DatasetPropertiesClass, base_aspect=orig_dataset_properties)
+    orig_dataset_properties = DatasetPropertiesClass(
+        name="test_name", description="test_description"
+    )
+    helper_test_custom_properties_patch(
+        test_entity_urn=dataset_urn,
+        patch_builder_class=DatasetPatchBuilder,
+        custom_properties_aspect_class=DatasetPropertiesClass,
+        base_aspect=orig_dataset_properties,
+    )
 
     with DataHubGraph(DataHubGraphConfig()) as graph:
         # Patch custom properties along with name
diff --git a/smoke-test/tests/policies/test_policies.py b/smoke-test/tests/policies/test_policies.py
index b7091541894dd..67142181d2b96 100644
--- a/smoke-test/tests/policies/test_policies.py
+++ b/smoke-test/tests/policies/test_policies.py
@@ -1,12 +1,8 @@
 import pytest
 import tenacity
-from tests.utils import (
-    get_frontend_url,
-    wait_for_healthcheck_util,
-    get_frontend_session,
-    get_sleep_info,
-    get_root_urn,
-)
+
+from tests.utils import (get_frontend_session, get_frontend_url, get_root_urn,
+                         get_sleep_info, wait_for_healthcheck_util)
 
 TEST_POLICY_NAME = "Updated Platform Policy"
 
diff --git a/smoke-test/tests/setup/lineage/helper_classes.py b/smoke-test/tests/setup/lineage/helper_classes.py
index 53f77b08d15ed..d550f3093be85 100644
--- a/smoke-test/tests/setup/lineage/helper_classes.py
+++ b/smoke-test/tests/setup/lineage/helper_classes.py
@@ -1,10 +1,7 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional
 
-from datahub.metadata.schema_classes import (
-    EdgeClass,
-    SchemaFieldDataTypeClass,
-)
+from datahub.metadata.schema_classes import EdgeClass, SchemaFieldDataTypeClass
 
 
 @dataclass
diff --git a/smoke-test/tests/setup/lineage/ingest_data_job_change.py b/smoke-test/tests/setup/lineage/ingest_data_job_change.py
index 8e3e9c5352922..588a1625419bc 100644
--- a/smoke-test/tests/setup/lineage/ingest_data_job_change.py
+++ b/smoke-test/tests/setup/lineage/ingest_data_job_change.py
@@ -1,36 +1,20 @@
 from typing import List
 
-from datahub.emitter.mce_builder import (
-    make_dataset_urn,
-    make_data_flow_urn,
-    make_data_job_urn_with_flow,
-)
+from datahub.emitter.mce_builder import (make_data_flow_urn,
+                                         make_data_job_urn_with_flow,
+                                         make_dataset_urn)
 from datahub.emitter.rest_emitter import DatahubRestEmitter
-from datahub.metadata.schema_classes import (
-    DateTypeClass,
-    NumberTypeClass,
-    SchemaFieldDataTypeClass,
-    StringTypeClass,
-)
+from datahub.metadata.schema_classes import (DateTypeClass, NumberTypeClass,
+                                             SchemaFieldDataTypeClass,
+                                             StringTypeClass)
 
-from tests.setup.lineage.constants import (
-    AIRFLOW_DATA_PLATFORM,
-    SNOWFLAKE_DATA_PLATFORM,
-    TIMESTAMP_MILLIS_EIGHT_DAYS_AGO,
-    TIMESTAMP_MILLIS_ONE_DAY_AGO,
-)
-from tests.setup.lineage.helper_classes import (
-    Field,
-    Dataset,
-    Task,
-    Pipeline,
-)
-from tests.setup.lineage.utils import (
-    create_edge,
-    create_node,
-    create_nodes_and_edges,
-    emit_mcps,
-)
+from tests.setup.lineage.constants import (AIRFLOW_DATA_PLATFORM,
+                                           SNOWFLAKE_DATA_PLATFORM,
+                                           TIMESTAMP_MILLIS_EIGHT_DAYS_AGO,
+                                           TIMESTAMP_MILLIS_ONE_DAY_AGO)
+from tests.setup.lineage.helper_classes import Dataset, Field, Pipeline, Task
+from tests.setup.lineage.utils import (create_edge, create_node,
+                                       create_nodes_and_edges, emit_mcps)
 
 # Constants for Case 2
 DAILY_TEMPERATURE_DATASET_ID = "climate.daily_temperature"
diff --git a/smoke-test/tests/setup/lineage/ingest_dataset_join_change.py b/smoke-test/tests/setup/lineage/ingest_dataset_join_change.py
index 35a8e6d5cf02e..bb9f51b6b5e9b 100644
--- a/smoke-test/tests/setup/lineage/ingest_dataset_join_change.py
+++ b/smoke-test/tests/setup/lineage/ingest_dataset_join_change.py
@@ -1,32 +1,18 @@
 from typing import List
 
-from datahub.emitter.mce_builder import (
-    make_dataset_urn,
-)
+from datahub.emitter.mce_builder import make_dataset_urn
 from datahub.emitter.rest_emitter import DatahubRestEmitter
-from datahub.metadata.schema_classes import (
-    NumberTypeClass,
-    SchemaFieldDataTypeClass,
-    StringTypeClass,
-    UpstreamClass,
-)
+from datahub.metadata.schema_classes import (NumberTypeClass,
+                                             SchemaFieldDataTypeClass,
+                                             StringTypeClass, UpstreamClass)
 
-from tests.setup.lineage.constants import (
-    DATASET_ENTITY_TYPE,
-    SNOWFLAKE_DATA_PLATFORM,
-    TIMESTAMP_MILLIS_EIGHT_DAYS_AGO,
-    TIMESTAMP_MILLIS_ONE_DAY_AGO,
-)
-from tests.setup.lineage.helper_classes import (
-    Field,
-    Dataset,
-)
-from tests.setup.lineage.utils import (
-    create_node,
-    create_upstream_edge,
-    create_upstream_mcp,
-    emit_mcps,
-)
+from tests.setup.lineage.constants import (DATASET_ENTITY_TYPE,
+                                           SNOWFLAKE_DATA_PLATFORM,
+                                           TIMESTAMP_MILLIS_EIGHT_DAYS_AGO,
+                                           TIMESTAMP_MILLIS_ONE_DAY_AGO)
+from tests.setup.lineage.helper_classes import Dataset, Field
+from tests.setup.lineage.utils import (create_node, create_upstream_edge,
+                                       create_upstream_mcp, emit_mcps)
 
 # Constants for Case 3
 GDP_DATASET_ID = "economic_data.gdp"
diff --git a/smoke-test/tests/setup/lineage/ingest_input_datasets_change.py b/smoke-test/tests/setup/lineage/ingest_input_datasets_change.py
index f4fb795147478..6079d7a3d2b63 100644
--- a/smoke-test/tests/setup/lineage/ingest_input_datasets_change.py
+++ b/smoke-test/tests/setup/lineage/ingest_input_datasets_change.py
@@ -1,36 +1,20 @@
 from typing import List
 
-from datahub.emitter.mce_builder import (
-    make_dataset_urn,
-    make_data_flow_urn,
-    make_data_job_urn_with_flow,
-)
+from datahub.emitter.mce_builder import (make_data_flow_urn,
+                                         make_data_job_urn_with_flow,
+                                         make_dataset_urn)
 from datahub.emitter.rest_emitter import DatahubRestEmitter
-from datahub.metadata.schema_classes import (
-    NumberTypeClass,
-    SchemaFieldDataTypeClass,
-    StringTypeClass,
-)
-
-from tests.setup.lineage.constants import (
-    AIRFLOW_DATA_PLATFORM,
-    BQ_DATA_PLATFORM,
-    TIMESTAMP_MILLIS_EIGHT_DAYS_AGO,
-    TIMESTAMP_MILLIS_ONE_DAY_AGO,
-)
-from tests.setup.lineage.helper_classes import (
-    Field,
-    Dataset,
-    Task,
-    Pipeline,
-)
-from tests.setup.lineage.utils import (
-    create_edge,
-    create_node,
-    create_nodes_and_edges,
-    emit_mcps,
-)
+from datahub.metadata.schema_classes import (NumberTypeClass,
+                                             SchemaFieldDataTypeClass,
+                                             StringTypeClass)
 
+from tests.setup.lineage.constants import (AIRFLOW_DATA_PLATFORM,
+                                           BQ_DATA_PLATFORM,
+                                           TIMESTAMP_MILLIS_EIGHT_DAYS_AGO,
+                                           TIMESTAMP_MILLIS_ONE_DAY_AGO)
+from tests.setup.lineage.helper_classes import Dataset, Field, Pipeline, Task
+from tests.setup.lineage.utils import (create_edge, create_node,
+                                       create_nodes_and_edges, emit_mcps)
 
 # Constants for Case 1
 TRANSACTIONS_DATASET_ID = "transactions.transactions"
diff --git a/smoke-test/tests/setup/lineage/ingest_time_lineage.py b/smoke-test/tests/setup/lineage/ingest_time_lineage.py
index cae8e0124d501..3aec979707290 100644
--- a/smoke-test/tests/setup/lineage/ingest_time_lineage.py
+++ b/smoke-test/tests/setup/lineage/ingest_time_lineage.py
@@ -1,12 +1,14 @@
+import os
 from typing import List
 
 from datahub.emitter.rest_emitter import DatahubRestEmitter
 
-from tests.setup.lineage.ingest_input_datasets_change import ingest_input_datasets_change, get_input_datasets_change_urns
-from tests.setup.lineage.ingest_data_job_change import ingest_data_job_change, get_data_job_change_urns
-from tests.setup.lineage.ingest_dataset_join_change import ingest_dataset_join_change, get_dataset_join_change_urns
-
-import os
+from tests.setup.lineage.ingest_data_job_change import (
+    get_data_job_change_urns, ingest_data_job_change)
+from tests.setup.lineage.ingest_dataset_join_change import (
+    get_dataset_join_change_urns, ingest_dataset_join_change)
+from tests.setup.lineage.ingest_input_datasets_change import (
+    get_input_datasets_change_urns, ingest_input_datasets_change)
 
 SERVER = os.getenv("DATAHUB_SERVER") or "http://localhost:8080"
 TOKEN = os.getenv("DATAHUB_TOKEN") or ""
@@ -20,4 +22,8 @@ def ingest_time_lineage() -> None:
 
 
 def get_time_lineage_urns() -> List[str]:
-    return get_input_datasets_change_urns() + get_data_job_change_urns() + get_dataset_join_change_urns()
+    return (
+        get_input_datasets_change_urns()
+        + get_data_job_change_urns()
+        + get_dataset_join_change_urns()
+    )
diff --git a/smoke-test/tests/setup/lineage/utils.py b/smoke-test/tests/setup/lineage/utils.py
index 672f7a945a6af..c72f6ccb89b7a 100644
--- a/smoke-test/tests/setup/lineage/utils.py
+++ b/smoke-test/tests/setup/lineage/utils.py
@@ -1,41 +1,30 @@
 import datetime
-from datahub.emitter.mce_builder import (
-    make_data_platform_urn,
-    make_dataset_urn,
-    make_data_job_urn_with_flow,
-    make_data_flow_urn,
-)
+from typing import List
+
+from datahub.emitter.mce_builder import (make_data_flow_urn,
+                                         make_data_job_urn_with_flow,
+                                         make_data_platform_urn,
+                                         make_dataset_urn)
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage
-from datahub.metadata.schema_classes import (
-    AuditStampClass,
-    ChangeTypeClass,
-    DatasetLineageTypeClass,
-    DatasetPropertiesClass,
-    DataFlowInfoClass,
-    DataJobInputOutputClass,
-    DataJobInfoClass,
-    EdgeClass,
-    MySqlDDLClass,
-    SchemaFieldClass,
-    SchemaMetadataClass,
-    UpstreamClass,
-)
-from typing import List
-
-from tests.setup.lineage.constants import (
-    DATASET_ENTITY_TYPE,
-    DATA_JOB_ENTITY_TYPE,
-    DATA_FLOW_ENTITY_TYPE,
-    DATA_FLOW_INFO_ASPECT_NAME,
-    DATA_JOB_INFO_ASPECT_NAME,
-    DATA_JOB_INPUT_OUTPUT_ASPECT_NAME,
-)
-from tests.setup.lineage.helper_classes import (
-    Dataset,
-    Pipeline,
-)
+from datahub.metadata.schema_classes import (AuditStampClass, ChangeTypeClass,
+                                             DataFlowInfoClass,
+                                             DataJobInfoClass,
+                                             DataJobInputOutputClass,
+                                             DatasetLineageTypeClass,
+                                             DatasetPropertiesClass, EdgeClass,
+                                             MySqlDDLClass, SchemaFieldClass,
+                                             SchemaMetadataClass,
+                                             UpstreamClass)
+
+from tests.setup.lineage.constants import (DATA_FLOW_ENTITY_TYPE,
+                                           DATA_FLOW_INFO_ASPECT_NAME,
+                                           DATA_JOB_ENTITY_TYPE,
+                                           DATA_JOB_INFO_ASPECT_NAME,
+                                           DATA_JOB_INPUT_OUTPUT_ASPECT_NAME,
+                                           DATASET_ENTITY_TYPE)
+from tests.setup.lineage.helper_classes import Dataset, Pipeline
 
 
 def create_node(dataset: Dataset) -> List[MetadataChangeProposalWrapper]:
@@ -85,10 +74,10 @@ def create_node(dataset: Dataset) -> List[MetadataChangeProposalWrapper]:
 
 
 def create_edge(
-        source_urn: str,
-        destination_urn: str,
-        created_timestamp_millis: int,
-        updated_timestamp_millis: int,
+    source_urn: str,
+    destination_urn: str,
+    created_timestamp_millis: int,
+    updated_timestamp_millis: int,
 ) -> EdgeClass:
     created_audit_stamp: AuditStampClass = AuditStampClass(
         time=created_timestamp_millis, actor="urn:li:corpuser:unknown"
@@ -105,7 +94,7 @@ def create_edge(
 
 
 def create_nodes_and_edges(
-        airflow_dag: Pipeline,
+    airflow_dag: Pipeline,
 ) -> List[MetadataChangeProposalWrapper]:
     mcps = []
     data_flow_urn = make_data_flow_urn(
@@ -160,9 +149,9 @@ def create_nodes_and_edges(
 
 
 def create_upstream_edge(
-        upstream_entity_urn: str,
-        created_timestamp_millis: int,
-        updated_timestamp_millis: int,
+    upstream_entity_urn: str,
+    created_timestamp_millis: int,
+    updated_timestamp_millis: int,
 ):
     created_audit_stamp: AuditStampClass = AuditStampClass(
         time=created_timestamp_millis, actor="urn:li:corpuser:unknown"
@@ -180,11 +169,11 @@ def create_upstream_edge(
 
 
 def create_upstream_mcp(
-        entity_type: str,
-        entity_urn: str,
-        upstreams: List[UpstreamClass],
-        timestamp_millis: int,
-        run_id: str = "",
+    entity_type: str,
+    entity_urn: str,
+    upstreams: List[UpstreamClass],
+    timestamp_millis: int,
+    run_id: str = "",
 ) -> MetadataChangeProposalWrapper:
     print(f"Creating upstreamLineage aspect for {entity_urn}")
     timestamp_millis: int = int(datetime.datetime.now().timestamp() * 1000)
@@ -203,7 +192,7 @@ def create_upstream_mcp(
 
 
 def emit_mcps(
-        emitter: DatahubRestEmitter, mcps: List[MetadataChangeProposalWrapper]
+    emitter: DatahubRestEmitter, mcps: List[MetadataChangeProposalWrapper]
 ) -> None:
     for mcp in mcps:
         emitter.emit_mcp(mcp)
diff --git a/smoke-test/tests/tags-and-terms/tags_and_terms_test.py b/smoke-test/tests/tags-and-terms/tags_and_terms_test.py
index b0ca29b544cfe..6ac75765286f0 100644
--- a/smoke-test/tests/tags-and-terms/tags_and_terms_test.py
+++ b/smoke-test/tests/tags-and-terms/tags_and_terms_test.py
@@ -1,5 +1,7 @@
 import pytest
-from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest, wait_for_healthcheck_util
+
+from tests.utils import (delete_urns_from_file, get_frontend_url,
+                         ingest_file_via_rest, wait_for_healthcheck_util)
 
 
 @pytest.fixture(scope="module", autouse=True)
diff --git a/smoke-test/tests/telemetry/telemetry_test.py b/smoke-test/tests/telemetry/telemetry_test.py
index 3672abcda948d..3127061c9f506 100644
--- a/smoke-test/tests/telemetry/telemetry_test.py
+++ b/smoke-test/tests/telemetry/telemetry_test.py
@@ -7,5 +7,7 @@ def test_no_clientID():
     client_id_urn = "urn:li:telemetry:clientId"
     aspect = ["telemetryClientId"]
 
-    res_data = json.dumps(get_aspects_for_entity(entity_urn=client_id_urn, aspects=aspect, typed=False))
+    res_data = json.dumps(
+        get_aspects_for_entity(entity_urn=client_id_urn, aspects=aspect, typed=False)
+    )
     assert res_data == "{}"
diff --git a/smoke-test/tests/test_result_msg.py b/smoke-test/tests/test_result_msg.py
index e3b336db9d66c..b9775e8ee4acd 100644
--- a/smoke-test/tests/test_result_msg.py
+++ b/smoke-test/tests/test_result_msg.py
@@ -1,6 +1,6 @@
-from slack_sdk import WebClient
 import os
 
+from slack_sdk import WebClient
 
 datahub_stats = {}
 
@@ -10,10 +10,10 @@ def add_datahub_stats(stat_name, stat_val):
 
 
 def send_to_slack(passed: str):
-    slack_api_token = os.getenv('SLACK_API_TOKEN')
-    slack_channel = os.getenv('SLACK_CHANNEL')
-    slack_thread_ts = os.getenv('SLACK_THREAD_TS')
-    test_identifier = os.getenv('TEST_IDENTIFIER', 'LOCAL_TEST')
+    slack_api_token = os.getenv("SLACK_API_TOKEN")
+    slack_channel = os.getenv("SLACK_CHANNEL")
+    slack_thread_ts = os.getenv("SLACK_THREAD_TS")
+    test_identifier = os.getenv("TEST_IDENTIFIER", "LOCAL_TEST")
     if slack_api_token is None or slack_channel is None:
         return
     client = WebClient(token=slack_api_token)
@@ -26,14 +26,21 @@ def send_to_slack(passed: str):
             message += f"Num {entity_type} is {val}\n"
 
     if slack_thread_ts is None:
-        client.chat_postMessage(channel=slack_channel, text=f'{test_identifier} Status - {passed}\n{message}')
+        client.chat_postMessage(
+            channel=slack_channel,
+            text=f"{test_identifier} Status - {passed}\n{message}",
+        )
     else:
-        client.chat_postMessage(channel=slack_channel, text=f'{test_identifier} Status - {passed}\n{message}', thread_ts=slack_thread_ts)
+        client.chat_postMessage(
+            channel=slack_channel,
+            text=f"{test_identifier} Status - {passed}\n{message}",
+            thread_ts=slack_thread_ts,
+        )
 
 
 def send_message(exitstatus):
     try:
-        send_to_slack('PASSED' if exitstatus == 0 else 'FAILED')
+        send_to_slack("PASSED" if exitstatus == 0 else "FAILED")
     except Exception as e:
         # We don't want to fail pytest at all
         print(f"Exception happened for sending msg to slack {e}")
diff --git a/smoke-test/tests/test_stateful_ingestion.py b/smoke-test/tests/test_stateful_ingestion.py
index a10cf13a08029..c6adb402e5d51 100644
--- a/smoke-test/tests/test_stateful_ingestion.py
+++ b/smoke-test/tests/test_stateful_ingestion.py
@@ -4,17 +4,15 @@
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource
 from datahub.ingestion.source.state.checkpoint import Checkpoint
-from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState
-from datahub.ingestion.source.state.stale_entity_removal_handler import StaleEntityRemovalHandler
+from datahub.ingestion.source.state.entity_removal_state import \
+    GenericCheckpointState
+from datahub.ingestion.source.state.stale_entity_removal_handler import \
+    StaleEntityRemovalHandler
 from sqlalchemy import create_engine
 from sqlalchemy.sql import text
 
-from tests.utils import (
-    get_gms_url,
-    get_mysql_password,
-    get_mysql_url,
-    get_mysql_username,
-)
+from tests.utils import (get_gms_url, get_mysql_password, get_mysql_url,
+                         get_mysql_username)
 
 
 def test_stateful_ingestion(wait_for_healthchecks):
diff --git a/smoke-test/tests/tests/tests_test.py b/smoke-test/tests/tests/tests_test.py
index 0b87f90a92c58..213a2ea087b7a 100644
--- a/smoke-test/tests/tests/tests_test.py
+++ b/smoke-test/tests/tests/tests_test.py
@@ -1,9 +1,13 @@
 import pytest
 import tenacity
-from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest, wait_for_healthcheck_util, get_sleep_info
+
+from tests.utils import (delete_urns_from_file, get_frontend_url,
+                         get_sleep_info, ingest_file_via_rest,
+                         wait_for_healthcheck_util)
 
 sleep_sec, sleep_times = get_sleep_info()
 
+
 @pytest.fixture(scope="module", autouse=True)
 def ingest_cleanup_data(request):
     print("ingesting test data")
@@ -18,6 +22,7 @@ def wait_for_healthchecks():
     wait_for_healthcheck_util()
     yield
 
+
 @pytest.mark.dependency()
 def test_healthchecks(wait_for_healthchecks):
     # Call to wait_for_healthchecks fixture will do the actual functionality.
diff --git a/smoke-test/tests/timeline/timeline_test.py b/smoke-test/tests/timeline/timeline_test.py
index a73d585c6c72d..4705343c1a2ba 100644
--- a/smoke-test/tests/timeline/timeline_test.py
+++ b/smoke-test/tests/timeline/timeline_test.py
@@ -3,14 +3,14 @@
 
 from datahub.cli import timeline_cli
 from datahub.cli.cli_utils import guess_entity_type, post_entity
-from tests.utils import ingest_file_via_rest, wait_for_writes_to_sync, get_datahub_graph
+
+from tests.utils import (get_datahub_graph, ingest_file_via_rest,
+                         wait_for_writes_to_sync)
 
 
 def test_all():
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-timeline-sample-kafka"
-    )
+    dataset_name = "test-timeline-sample-kafka"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
@@ -18,8 +18,13 @@ def test_all():
     ingest_file_via_rest("tests/timeline/timeline_test_datav2.json")
     ingest_file_via_rest("tests/timeline/timeline_test_datav3.json")
 
-    res_data = timeline_cli.get_timeline(dataset_urn, ["TAG", "DOCUMENTATION", "TECHNICAL_SCHEMA", "GLOSSARY_TERM",
-                                                       "OWNER"], None, None, False)
+    res_data = timeline_cli.get_timeline(
+        dataset_urn,
+        ["TAG", "DOCUMENTATION", "TECHNICAL_SCHEMA", "GLOSSARY_TERM", "OWNER"],
+        None,
+        None,
+        False,
+    )
     get_datahub_graph().hard_delete_entity(urn=dataset_urn)
 
     assert res_data
@@ -35,9 +40,7 @@ def test_all():
 
 def test_schema():
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-timeline-sample-kafka"
-    )
+    dataset_name = "test-timeline-sample-kafka"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
@@ -45,7 +48,9 @@ def test_schema():
     put(dataset_urn, "schemaMetadata", "test_resources/timeline/newschemav2.json")
     put(dataset_urn, "schemaMetadata", "test_resources/timeline/newschemav3.json")
 
-    res_data = timeline_cli.get_timeline(dataset_urn, ["TECHNICAL_SCHEMA"], None, None, False)
+    res_data = timeline_cli.get_timeline(
+        dataset_urn, ["TECHNICAL_SCHEMA"], None, None, False
+    )
 
     get_datahub_graph().hard_delete_entity(urn=dataset_urn)
     assert res_data
@@ -61,9 +66,7 @@ def test_schema():
 
 def test_glossary():
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-timeline-sample-kafka"
-    )
+    dataset_name = "test-timeline-sample-kafka"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
@@ -71,7 +74,9 @@ def test_glossary():
     put(dataset_urn, "glossaryTerms", "test_resources/timeline/newglossaryv2.json")
     put(dataset_urn, "glossaryTerms", "test_resources/timeline/newglossaryv3.json")
 
-    res_data = timeline_cli.get_timeline(dataset_urn, ["GLOSSARY_TERM"], None, None, False)
+    res_data = timeline_cli.get_timeline(
+        dataset_urn, ["GLOSSARY_TERM"], None, None, False
+    )
 
     get_datahub_graph().hard_delete_entity(urn=dataset_urn)
     assert res_data
@@ -87,17 +92,29 @@ def test_glossary():
 
 def test_documentation():
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-timeline-sample-kafka"
-    )
+    dataset_name = "test-timeline-sample-kafka"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
-    put(dataset_urn, "institutionalMemory", "test_resources/timeline/newdocumentation.json")
-    put(dataset_urn, "institutionalMemory", "test_resources/timeline/newdocumentationv2.json")
-    put(dataset_urn, "institutionalMemory", "test_resources/timeline/newdocumentationv3.json")
+    put(
+        dataset_urn,
+        "institutionalMemory",
+        "test_resources/timeline/newdocumentation.json",
+    )
+    put(
+        dataset_urn,
+        "institutionalMemory",
+        "test_resources/timeline/newdocumentationv2.json",
+    )
+    put(
+        dataset_urn,
+        "institutionalMemory",
+        "test_resources/timeline/newdocumentationv3.json",
+    )
 
-    res_data = timeline_cli.get_timeline(dataset_urn, ["DOCUMENTATION"], None, None, False)
+    res_data = timeline_cli.get_timeline(
+        dataset_urn, ["DOCUMENTATION"], None, None, False
+    )
 
     get_datahub_graph().hard_delete_entity(urn=dataset_urn)
     assert res_data
@@ -113,9 +130,7 @@ def test_documentation():
 
 def test_tags():
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-timeline-sample-kafka"
-    )
+    dataset_name = "test-timeline-sample-kafka"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
@@ -139,9 +154,7 @@ def test_tags():
 
 def test_ownership():
     platform = "urn:li:dataPlatform:kafka"
-    dataset_name = (
-        "test-timeline-sample-kafka"
-    )
+    dataset_name = "test-timeline-sample-kafka"
     env = "PROD"
     dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
 
diff --git a/smoke-test/tests/tokens/revokable_access_token_test.py b/smoke-test/tests/tokens/revokable_access_token_test.py
index b10ad3aa3fc2a..55f3de594af4e 100644
--- a/smoke-test/tests/tokens/revokable_access_token_test.py
+++ b/smoke-test/tests/tokens/revokable_access_token_test.py
@@ -1,15 +1,11 @@
 import os
-import pytest
-import requests
 from time import sleep
 
-from tests.utils import (
-    get_frontend_url,
-    wait_for_healthcheck_util,
-    get_admin_credentials,
-    wait_for_writes_to_sync,
-)
+import pytest
+import requests
 
+from tests.utils import (get_admin_credentials, get_frontend_url,
+                         wait_for_healthcheck_util, wait_for_writes_to_sync)
 
 # Disable telemetry
 os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
diff --git a/smoke-test/tests/utils.py b/smoke-test/tests/utils.py
index af03efd4f71f8..bd75b13d1910f 100644
--- a/smoke-test/tests/utils.py
+++ b/smoke-test/tests/utils.py
@@ -1,19 +1,20 @@
 import functools
 import json
+import logging
 import os
-from datetime import datetime, timedelta, timezone
 import subprocess
 import time
-from typing import Any, Dict, List, Tuple
+from datetime import datetime, timedelta, timezone
 from time import sleep
-from joblib import Parallel, delayed
+from typing import Any, Dict, List, Tuple
 
-import requests_wrapper as requests
-import logging
 from datahub.cli import cli_utils
 from datahub.cli.cli_utils import get_system_auth
-from datahub.ingestion.graph.client import DataHubGraph, DatahubClientConfig
+from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
 from datahub.ingestion.run.pipeline import Pipeline
+from joblib import Parallel, delayed
+
+import requests_wrapper as requests
 from tests.consistency_utils import wait_for_writes_to_sync
 
 TIME: int = 1581407189000
@@ -174,6 +175,7 @@ def delete(entry):
 
     wait_for_writes_to_sync()
 
+
 # Fixed now value
 NOW: datetime = datetime.now()
 
@@ -232,6 +234,3 @@ def create_datahub_step_state_aspects(
     ]
     with open(onboarding_filename, "w") as f:
         json.dump(aspects_dict, f, indent=2)
-
-
-
diff --git a/smoke-test/tests/views/views_test.py b/smoke-test/tests/views/views_test.py
index 4da69750a167b..685c3bd80b04d 100644
--- a/smoke-test/tests/views/views_test.py
+++ b/smoke-test/tests/views/views_test.py
@@ -1,16 +1,14 @@
-import pytest
 import time
+
+import pytest
 import tenacity
-from tests.utils import (
-    delete_urns_from_file,
-    get_frontend_url,
-    get_gms_url,
-    ingest_file_via_rest,
-    get_sleep_info,
-)
+
+from tests.utils import (delete_urns_from_file, get_frontend_url, get_gms_url,
+                         get_sleep_info, ingest_file_via_rest)
 
 sleep_sec, sleep_times = get_sleep_info()
 
+
 @pytest.mark.dependency()
 def test_healthchecks(wait_for_healthchecks):
     # Call to wait_for_healthchecks fixture will do the actual functionality.
@@ -40,6 +38,7 @@ def _ensure_more_views(frontend_session, list_views_json, query_name, before_cou
     assert after_count == before_count + 1
     return after_count
 
+
 @tenacity.retry(
     stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
 )
@@ -111,18 +110,18 @@ def test_create_list_delete_global_view(frontend_session):
     new_view_name = "Test View"
     new_view_description = "Test Description"
     new_view_definition = {
-      "entityTypes": ["DATASET", "DASHBOARD"],
-      "filter": {
-        "operator": "AND",
-        "filters": [
-          {
-            "field": "tags",
-            "values": ["urn:li:tag:test"],
-            "negated": False,
-            "condition": "EQUAL"
-          }
-        ]
-      }
+        "entityTypes": ["DATASET", "DASHBOARD"],
+        "filter": {
+            "operator": "AND",
+            "filters": [
+                {
+                    "field": "tags",
+                    "values": ["urn:li:tag:test"],
+                    "negated": False,
+                    "condition": "EQUAL",
+                }
+            ],
+        },
     }
 
     # Create new View
@@ -137,7 +136,7 @@ def test_create_list_delete_global_view(frontend_session):
                 "viewType": "GLOBAL",
                 "name": new_view_name,
                 "description": new_view_description,
-                "definition": new_view_definition
+                "definition": new_view_definition,
             }
         },
     }
@@ -169,9 +168,7 @@ def test_create_list_delete_global_view(frontend_session):
         "query": """mutation deleteView($urn: String!) {\n
             deleteView(urn: $urn)
         }""",
-        "variables": {
-            "urn": view_urn
-        },
+        "variables": {"urn": view_urn},
     }
 
     response = frontend_session.post(
@@ -189,7 +186,9 @@ def test_create_list_delete_global_view(frontend_session):
     )
 
 
-@pytest.mark.dependency(depends=["test_healthchecks", "test_create_list_delete_global_view"])
+@pytest.mark.dependency(
+    depends=["test_healthchecks", "test_create_list_delete_global_view"]
+)
 def test_create_list_delete_personal_view(frontend_session):
 
     # Get count of existing views
@@ -237,18 +236,18 @@ def test_create_list_delete_personal_view(frontend_session):
     new_view_name = "Test View"
     new_view_description = "Test Description"
     new_view_definition = {
-      "entityTypes": ["DATASET", "DASHBOARD"],
-      "filter": {
-        "operator": "AND",
-        "filters": [
-          {
-            "field": "tags",
-            "values": ["urn:li:tag:test"],
-            "negated": False,
-            "condition": "EQUAL"
-          }
-        ]
-      }
+        "entityTypes": ["DATASET", "DASHBOARD"],
+        "filter": {
+            "operator": "AND",
+            "filters": [
+                {
+                    "field": "tags",
+                    "values": ["urn:li:tag:test"],
+                    "negated": False,
+                    "condition": "EQUAL",
+                }
+            ],
+        },
     }
 
     # Create new View
@@ -263,7 +262,7 @@ def test_create_list_delete_personal_view(frontend_session):
                 "viewType": "PERSONAL",
                 "name": new_view_name,
                 "description": new_view_description,
-                "definition": new_view_definition
+                "definition": new_view_definition,
             }
         },
     }
@@ -293,9 +292,7 @@ def test_create_list_delete_personal_view(frontend_session):
         "query": """mutation deleteView($urn: String!) {\n
             deleteView(urn: $urn)
         }""",
-        "variables": {
-            "urn": view_urn
-        },
+        "variables": {"urn": view_urn},
     }
 
     response = frontend_session.post(
@@ -312,25 +309,28 @@ def test_create_list_delete_personal_view(frontend_session):
         before_count=new_count,
     )
 
-@pytest.mark.dependency(depends=["test_healthchecks", "test_create_list_delete_personal_view"])
+
+@pytest.mark.dependency(
+    depends=["test_healthchecks", "test_create_list_delete_personal_view"]
+)
 def test_update_global_view(frontend_session):
 
     # First create a view
     new_view_name = "Test View"
     new_view_description = "Test Description"
     new_view_definition = {
-      "entityTypes": ["DATASET", "DASHBOARD"],
-      "filter": {
-        "operator": "AND",
-        "filters": [
-          {
-            "field": "tags",
-            "values": ["urn:li:tag:test"],
-            "negated": False,
-            "condition": "EQUAL"
-          }
-        ]
-      }
+        "entityTypes": ["DATASET", "DASHBOARD"],
+        "filter": {
+            "operator": "AND",
+            "filters": [
+                {
+                    "field": "tags",
+                    "values": ["urn:li:tag:test"],
+                    "negated": False,
+                    "condition": "EQUAL",
+                }
+            ],
+        },
     }
 
     # Create new View
@@ -345,7 +345,7 @@ def test_update_global_view(frontend_session):
                 "viewType": "PERSONAL",
                 "name": new_view_name,
                 "description": new_view_description,
-                "definition": new_view_definition
+                "definition": new_view_definition,
             }
         },
     }
@@ -366,18 +366,18 @@ def test_update_global_view(frontend_session):
     new_view_name = "New Test View"
     new_view_description = "New Test Description"
     new_view_definition = {
-      "entityTypes": ["DATASET", "DASHBOARD", "CHART", "DATA_FLOW"],
-      "filter": {
-        "operator": "OR",
-        "filters": [
-          {
-            "field": "glossaryTerms",
-            "values": ["urn:li:glossaryTerm:test"],
-            "negated": True,
-            "condition": "CONTAIN"
-          }
-        ]
-      }
+        "entityTypes": ["DATASET", "DASHBOARD", "CHART", "DATA_FLOW"],
+        "filter": {
+            "operator": "OR",
+            "filters": [
+                {
+                    "field": "glossaryTerms",
+                    "values": ["urn:li:glossaryTerm:test"],
+                    "negated": True,
+                    "condition": "CONTAIN",
+                }
+            ],
+        },
     }
 
     update_view_json = {
@@ -391,8 +391,8 @@ def test_update_global_view(frontend_session):
             "input": {
                 "name": new_view_name,
                 "description": new_view_description,
-                "definition": new_view_definition
-            }
+                "definition": new_view_definition,
+            },
         },
     }
 
@@ -411,9 +411,7 @@ def test_update_global_view(frontend_session):
         "query": """mutation deleteView($urn: String!) {\n
             deleteView(urn: $urn)
         }""",
-        "variables": {
-            "urn": view_urn
-        },
+        "variables": {"urn": view_urn},
     }
 
     response = frontend_session.post(

From 6ecdeda5ff590456c6bfadfa5c37821f7281169e Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Tue, 10 Oct 2023 16:28:40 +0530
Subject: [PATCH 024/259] fix(setup): drop older table if exists (#8979)

---
 docker/mariadb/init.sql        | 2 ++
 docker/mysql-setup/init.sql    | 2 ++
 docker/mysql/init.sql          | 2 ++
 docker/postgres-setup/init.sql | 2 ++
 docker/postgres/init.sql       | 2 ++
 5 files changed, 10 insertions(+)

diff --git a/docker/mariadb/init.sql b/docker/mariadb/init.sql
index c4132575cf442..95c8cabbc5ca4 100644
--- a/docker/mariadb/init.sql
+++ b/docker/mariadb/init.sql
@@ -28,3 +28,5 @@ insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat
   now(),
   'urn:li:corpuser:__datahub_system'
 );
+
+DROP TABLE IF EXISTS metadata_index;
diff --git a/docker/mysql-setup/init.sql b/docker/mysql-setup/init.sql
index 2370a971941d2..b789329ddfd17 100644
--- a/docker/mysql-setup/init.sql
+++ b/docker/mysql-setup/init.sql
@@ -39,3 +39,5 @@ INSERT INTO metadata_aspect_v2
 SELECT * FROM temp_metadata_aspect_v2
 WHERE NOT EXISTS (SELECT * from metadata_aspect_v2);
 DROP TABLE temp_metadata_aspect_v2;
+
+DROP TABLE IF EXISTS metadata_index;
diff --git a/docker/mysql/init.sql b/docker/mysql/init.sql
index b4b4e4617806c..aca57d7cd444c 100644
--- a/docker/mysql/init.sql
+++ b/docker/mysql/init.sql
@@ -27,3 +27,5 @@ INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat
   now(),
   'urn:li:corpuser:__datahub_system'
 );
+
+DROP TABLE IF EXISTS metadata_index;
diff --git a/docker/postgres-setup/init.sql b/docker/postgres-setup/init.sql
index 12fff7aec7fe6..72b2f73192e00 100644
--- a/docker/postgres-setup/init.sql
+++ b/docker/postgres-setup/init.sql
@@ -35,3 +35,5 @@ INSERT INTO metadata_aspect_v2
 SELECT * FROM temp_metadata_aspect_v2
 WHERE NOT EXISTS (SELECT * from metadata_aspect_v2);
 DROP TABLE temp_metadata_aspect_v2;
+
+DROP TABLE IF EXISTS metadata_index;
diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql
index cf477c135422e..87c8dd3337fac 100644
--- a/docker/postgres/init.sql
+++ b/docker/postgres/init.sql
@@ -28,3 +28,5 @@ insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, creat
   now(),
   'urn:li:corpuser:__datahub_system'
 );
+
+DROP TABLE IF EXISTS metadata_index;

From 1a72fa499c3404c6c3d2961e9575495f2dd021d2 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 10 Oct 2023 17:34:06 -0400
Subject: [PATCH 025/259] feat(ingest/tableau): Allow parsing of database name
 from fullName (#8981)

---
 .../src/datahub/ingestion/source/tableau.py   |  74 ++------
 .../ingestion/source/tableau_common.py        | 162 +++++++++++++-----
 .../tableau/test_tableau_ingest.py            |  34 ++--
 3 files changed, 151 insertions(+), 119 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
index e347cd26d245a..bad7ae49d325e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
@@ -77,6 +77,7 @@
     FIELD_TYPE_MAPPING,
     MetadataQueryException,
     TableauLineageOverrides,
+    TableauUpstreamReference,
     clean_query,
     custom_sql_graphql_query,
     dashboard_graphql_query,
@@ -85,7 +86,6 @@
     get_overridden_info,
     get_unique_custom_sql,
     make_fine_grained_lineage_class,
-    make_table_urn,
     make_upstream_class,
     published_datasource_graphql_query,
     query_metadata,
@@ -271,7 +271,7 @@ class TableauConfig(
         "You can change this if your Tableau projects contain slashes in their names, and you'd like to filter by project.",
     )
 
-    default_schema_map: dict = Field(
+    default_schema_map: Dict[str, str] = Field(
         default={}, description="Default schema to use when schema is not found."
     )
     ingest_tags: Optional[bool] = Field(
@@ -997,41 +997,16 @@ def get_upstream_tables(
                 )
                 continue
 
-            schema = table.get(tableau_constant.SCHEMA) or ""
-            table_name = table.get(tableau_constant.NAME) or ""
-            full_name = table.get(tableau_constant.FULL_NAME) or ""
-            upstream_db = (
-                table[tableau_constant.DATABASE][tableau_constant.NAME]
-                if table.get(tableau_constant.DATABASE)
-                and table[tableau_constant.DATABASE].get(tableau_constant.NAME)
-                else ""
-            )
-            logger.debug(
-                "Processing Table with Connection Type: {0} and id {1}".format(
-                    table.get(tableau_constant.CONNECTION_TYPE) or "",
-                    table.get(tableau_constant.ID) or "",
+            try:
+                ref = TableauUpstreamReference.create(
+                    table, default_schema_map=self.config.default_schema_map
                 )
-            )
-            schema = self._get_schema(schema, upstream_db, full_name)
-            # if the schema is included within the table name we omit it
-            if (
-                schema
-                and table_name
-                and full_name
-                and table_name == full_name
-                and schema in table_name
-            ):
-                logger.debug(
-                    f"Omitting schema for upstream table {table[tableau_constant.ID]}, schema included in table name"
-                )
-                schema = ""
+            except Exception as e:
+                logger.info(f"Failed to generate upstream reference for {table}: {e}")
+                continue
 
-            table_urn = make_table_urn(
+            table_urn = ref.make_dataset_urn(
                 self.config.env,
-                upstream_db,
-                table.get(tableau_constant.CONNECTION_TYPE) or "",
-                schema,
-                table_name,
                 self.config.platform_instance_map,
                 self.config.lineage_overrides,
             )
@@ -1052,7 +1027,7 @@ def get_upstream_tables(
                     urn=table_urn,
                     id=table[tableau_constant.ID],
                     num_cols=num_tbl_cols,
-                    paths=set([table_path]) if table_path else set(),
+                    paths={table_path} if table_path else set(),
                 )
             else:
                 self.database_tables[table_urn].update_table(
@@ -2462,35 +2437,6 @@ def emit_embedded_datasources(self) -> Iterable[MetadataWorkUnit]:
                 is_embedded_ds=True,
             )
 
-    @lru_cache(maxsize=None)
-    def _get_schema(self, schema_provided: str, database: str, fullName: str) -> str:
-        # For some databases, the schema attribute in tableau api does not return
-        # correct schema name for the table. For more information, see
-        # https://help.tableau.com/current/api/metadata_api/en-us/docs/meta_api_model.html#schema_attribute.
-        # Hence we extract schema from fullName whenever fullName is available
-        schema = self._extract_schema_from_fullName(fullName) if fullName else ""
-        if not schema:
-            schema = schema_provided
-        elif schema != schema_provided:
-            logger.debug(
-                "Correcting schema, provided {0}, corrected {1}".format(
-                    schema_provided, schema
-                )
-            )
-
-        if not schema and database in self.config.default_schema_map:
-            schema = self.config.default_schema_map[database]
-
-        return schema
-
-    @lru_cache(maxsize=None)
-    def _extract_schema_from_fullName(self, fullName: str) -> str:
-        # fullName is observed to be in format [schemaName].[tableName]
-        # OR simply tableName OR [tableName]
-        if fullName.startswith("[") and "].[" in fullName:
-            return fullName[1 : fullName.index("]")]
-        return ""
-
     @lru_cache(maxsize=None)
     def get_last_modified(
         self, creator: Optional[str], created_at: bytes, updated_at: bytes
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
index 2c92285fdba77..7c4852042ce7c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
@@ -1,4 +1,6 @@
 import html
+import logging
+from dataclasses import dataclass
 from functools import lru_cache
 from typing import Dict, List, Optional, Tuple
 
@@ -6,6 +8,7 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import ConfigModel
+from datahub.ingestion.source import tableau_constant as tc
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageType,
     FineGrainedLineage,
@@ -31,6 +34,8 @@
 )
 from datahub.utilities.sqlglot_lineage import ColumnLineageInfo, SqlParsingResult
 
+logger = logging.getLogger(__name__)
+
 
 class TableauLineageOverrides(ConfigModel):
     platform_override_map: Optional[Dict[str, str]] = Field(
@@ -537,12 +542,12 @@ def get_fully_qualified_table_name(
     platform: str,
     upstream_db: str,
     schema: str,
-    full_name: str,
+    table_name: str,
 ) -> str:
     if platform == "athena":
         upstream_db = ""
     database_name = f"{upstream_db}." if upstream_db else ""
-    final_name = full_name.replace("[", "").replace("]", "")
+    final_name = table_name.replace("[", "").replace("]", "")
 
     schema_name = f"{schema}." if schema else ""
 
@@ -573,17 +578,123 @@ def get_fully_qualified_table_name(
     return fully_qualified_table_name
 
 
-def get_platform_instance(
-    platform: str, platform_instance_map: Optional[Dict[str, str]]
-) -> Optional[str]:
-    if platform_instance_map is not None and platform in platform_instance_map.keys():
-        return platform_instance_map[platform]
+@dataclass
+class TableauUpstreamReference:
+    database: Optional[str]
+    schema: Optional[str]
+    table: str
+
+    connection_type: str
+
+    @classmethod
+    def create(
+        cls, d: dict, default_schema_map: Optional[Dict[str, str]] = None
+    ) -> "TableauUpstreamReference":
+        # Values directly from `table` object from Tableau
+        database = t_database = d.get(tc.DATABASE, {}).get(tc.NAME)
+        schema = t_schema = d.get(tc.SCHEMA)
+        table = t_table = d.get(tc.NAME) or ""
+        t_full_name = d.get(tc.FULL_NAME)
+        t_connection_type = d[tc.CONNECTION_TYPE]  # required to generate urn
+        t_id = d[tc.ID]
+
+        parsed_full_name = cls.parse_full_name(t_full_name)
+        if parsed_full_name and len(parsed_full_name) == 3:
+            database, schema, table = parsed_full_name
+        elif parsed_full_name and len(parsed_full_name) == 2:
+            schema, table = parsed_full_name
+        else:
+            logger.debug(
+                f"Upstream urn generation ({t_id}):"
+                f"  Did not parse full name {t_full_name}: unexpected number of values",
+            )
+
+        if not schema and default_schema_map and database in default_schema_map:
+            schema = default_schema_map[database]
+
+        if database != t_database:
+            logger.debug(
+                f"Upstream urn generation ({t_id}):"
+                f" replacing database {t_database} with {database} from full name {t_full_name}"
+            )
+        if schema != t_schema:
+            logger.debug(
+                f"Upstream urn generation ({t_id}):"
+                f" replacing schema {t_schema} with {schema} from full name {t_full_name}"
+            )
+        if table != t_table:
+            logger.debug(
+                f"Upstream urn generation ({t_id}):"
+                f" replacing table {t_table} with {table} from full name {t_full_name}"
+            )
+
+        # TODO: See if we can remove this -- made for redshift
+        if (
+            schema
+            and t_table
+            and t_full_name
+            and t_table == t_full_name
+            and schema in t_table
+        ):
+            logger.debug(
+                f"Omitting schema for upstream table {t_id}, schema included in table name"
+            )
+            schema = ""
+
+        return cls(
+            database=database,
+            schema=schema,
+            table=table,
+            connection_type=t_connection_type,
+        )
+
+    @staticmethod
+    def parse_full_name(full_name: Optional[str]) -> Optional[List[str]]:
+        # fullName is observed to be in formats:
+        #  [database].[schema].[table]
+        #  [schema].[table]
+        #  [table]
+        #  table
+        #  schema
+
+        # TODO: Validate the startswith check. Currently required for our integration tests
+        if full_name is None or not full_name.startswith("["):
+            return None
+
+        return full_name.replace("[", "").replace("]", "").split(".")
+
+    def make_dataset_urn(
+        self,
+        env: str,
+        platform_instance_map: Optional[Dict[str, str]],
+        lineage_overrides: Optional[TableauLineageOverrides] = None,
+    ) -> str:
+        (
+            upstream_db,
+            platform_instance,
+            platform,
+            original_platform,
+        ) = get_overridden_info(
+            connection_type=self.connection_type,
+            upstream_db=self.database,
+            lineage_overrides=lineage_overrides,
+            platform_instance_map=platform_instance_map,
+        )
+
+        table_name = get_fully_qualified_table_name(
+            original_platform,
+            upstream_db or "",
+            self.schema,
+            self.table,
+        )
 
-    return None
+        return builder.make_dataset_urn_with_platform_instance(
+            platform, table_name, platform_instance, env
+        )
 
 
 def get_overridden_info(
-    connection_type: str,
+    connection_type: Optional[str],
     upstream_db: Optional[str],
     platform_instance_map: Optional[Dict[str, str]],
     lineage_overrides: Optional[TableauLineageOverrides] = None,
@@ -605,7 +716,9 @@ def get_overridden_info(
     ):
         upstream_db = lineage_overrides.database_override_map[upstream_db]
 
-    platform_instance = get_platform_instance(original_platform, platform_instance_map)
+    platform_instance = (
+        platform_instance_map.get(original_platform) if platform_instance_map else None
+    )
 
     if original_platform in ("athena", "hive", "mysql"):  # Two tier databases
         upstream_db = None
@@ -613,35 +726,6 @@ def get_overridden_info(
     return upstream_db, platform_instance, platform, original_platform
 
 
-def make_table_urn(
-    env: str,
-    upstream_db: Optional[str],
-    connection_type: str,
-    schema: str,
-    full_name: str,
-    platform_instance_map: Optional[Dict[str, str]],
-    lineage_overrides: Optional[TableauLineageOverrides] = None,
-) -> str:
-
-    upstream_db, platform_instance, platform, original_platform = get_overridden_info(
-        connection_type=connection_type,
-        upstream_db=upstream_db,
-        lineage_overrides=lineage_overrides,
-        platform_instance_map=platform_instance_map,
-    )
-
-    table_name = get_fully_qualified_table_name(
-        original_platform,
-        upstream_db if upstream_db is not None else "",
-        schema,
-        full_name,
-    )
-
-    return builder.make_dataset_urn_with_platform_instance(
-        platform, table_name, platform_instance, env
-    )
-
-
 def make_description_from_params(description, formula):
     """
     Generate column description
diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
index c31867f5aa904..0510f4a40f659 100644
--- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
+++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
@@ -20,7 +20,7 @@
 from datahub.ingestion.source.tableau import TableauConfig, TableauSource
 from datahub.ingestion.source.tableau_common import (
     TableauLineageOverrides,
-    make_table_urn,
+    TableauUpstreamReference,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageType,
@@ -546,13 +546,13 @@ def test_lineage_overrides():
     enable_logging()
     # Simple - specify platform instance to presto table
     assert (
-        make_table_urn(
-            DEFAULT_ENV,
+        TableauUpstreamReference(
             "presto_catalog",
-            "presto",
             "test-schema",
-            "presto_catalog.test-schema.test-table",
-            platform_instance_map={"presto": "my_presto_instance"},
+            "test-table",
+            "presto",
+        ).make_dataset_urn(
+            env=DEFAULT_ENV, platform_instance_map={"presto": "my_presto_instance"}
         )
         == "urn:li:dataset:(urn:li:dataPlatform:presto,my_presto_instance.presto_catalog.test-schema.test-table,PROD)"
     )
@@ -560,12 +560,13 @@ def test_lineage_overrides():
     # Transform presto urn to hive urn
     # resulting platform instance for hive = mapped platform instance + presto_catalog
     assert (
-        make_table_urn(
-            DEFAULT_ENV,
+        TableauUpstreamReference(
             "presto_catalog",
-            "presto",
             "test-schema",
-            "presto_catalog.test-schema.test-table",
+            "test-table",
+            "presto",
+        ).make_dataset_urn(
+            env=DEFAULT_ENV,
             platform_instance_map={"presto": "my_instance"},
             lineage_overrides=TableauLineageOverrides(
                 platform_override_map={"presto": "hive"},
@@ -574,14 +575,15 @@ def test_lineage_overrides():
         == "urn:li:dataset:(urn:li:dataPlatform:hive,my_instance.presto_catalog.test-schema.test-table,PROD)"
     )
 
-    # tranform hive urn to presto urn
+    # transform hive urn to presto urn
     assert (
-        make_table_urn(
-            DEFAULT_ENV,
-            "",
-            "hive",
+        TableauUpstreamReference(
+            None,
             "test-schema",
-            "test-schema.test-table",
+            "test-table",
+            "hive",
+        ).make_dataset_urn(
+            env=DEFAULT_ENV,
             platform_instance_map={"hive": "my_presto_instance.presto_catalog"},
             lineage_overrides=TableauLineageOverrides(
                 platform_override_map={"hive": "presto"},

From e2988017c23270acd95e25ec3289983ecc3895f7 Mon Sep 17 00:00:00 2001
From: Amanda Hernando <110099762+amanda-her@users.noreply.github.com>
Date: Wed, 11 Oct 2023 01:36:01 +0200
Subject: [PATCH 026/259] feat(auth): add data platform instance field resolver
 provider (#8828)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Sergio Gómez Villamor <sgomezvillamor@gmail.com>
Co-authored-by: Adrián Pertíñez <khurzak92@gmail.com>
---
 .../authorization/ResolvedResourceSpec.java   |  17 ++
 .../authorization/ResourceFieldType.java      |   6 +-
 .../DefaultResourceSpecResolver.java          |   9 +-
 ...PlatformInstanceFieldResolverProvider.java |  70 +++++++
 ...formInstanceFieldResolverProviderTest.java | 188 ++++++++++++++++++
 5 files changed, 286 insertions(+), 4 deletions(-)
 create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
 create mode 100644 metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java

diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java
index 53dd0be44f963..8e429a8ca1b94 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java
@@ -3,6 +3,7 @@
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
+import javax.annotation.Nullable;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.ToString;
@@ -35,4 +36,20 @@ public Set<String> getOwners() {
     }
     return fieldResolvers.get(ResourceFieldType.OWNER).getFieldValuesFuture().join().getValues();
   }
+
+  /**
+   * Fetch the platform instance for a Resolved Resource Spec
+   * @return a Platform Instance or null if one does not exist.
+   */
+  @Nullable
+  public String getDataPlatformInstance() {
+    if (!fieldResolvers.containsKey(ResourceFieldType.DATA_PLATFORM_INSTANCE)) {
+      return null;
+    }
+    Set<String> dataPlatformInstance = fieldResolvers.get(ResourceFieldType.DATA_PLATFORM_INSTANCE).getFieldValuesFuture().join().getValues();
+    if (dataPlatformInstance.size() > 0) {
+      return dataPlatformInstance.stream().findFirst().get();
+    }
+    return null;
+  }
 }
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java
index ee54d2bfbba1d..478522dc7c331 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java
@@ -19,5 +19,9 @@ public enum ResourceFieldType {
   /**
    * Domains of resource
    */
-  DOMAIN
+  DOMAIN,
+  /**
+   * Data platform instance of resource
+   */
+  DATA_PLATFORM_INSTANCE
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java
index cd4e0b0967829..64c43dc8aa591 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java
@@ -1,13 +1,15 @@
 package com.datahub.authorization;
 
-import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
 import com.datahub.authentication.Authentication;
+import com.datahub.authorization.fieldresolverprovider.DataPlatformInstanceFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.DomainFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.ResourceFieldResolverProvider;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.entity.client.EntityClient;
+
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -20,7 +22,8 @@ public DefaultResourceSpecResolver(Authentication systemAuthentication, EntityCl
     _resourceFieldResolverProviders =
         ImmutableList.of(new EntityTypeFieldResolverProvider(), new EntityUrnFieldResolverProvider(),
             new DomainFieldResolverProvider(entityClient, systemAuthentication),
-            new OwnerFieldResolverProvider(entityClient, systemAuthentication));
+            new OwnerFieldResolverProvider(entityClient, systemAuthentication),
+            new DataPlatformInstanceFieldResolverProvider(entityClient, systemAuthentication));
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
new file mode 100644
index 0000000000000..cd838625c2ca1
--- /dev/null
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
@@ -0,0 +1,70 @@
+package com.datahub.authorization.fieldresolverprovider;
+
+import com.datahub.authentication.Authentication;
+import com.datahub.authorization.FieldResolver;
+import com.datahub.authorization.ResourceFieldType;
+import com.datahub.authorization.ResourceSpec;
+import com.linkedin.common.DataPlatformInstance;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.client.EntityClient;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Collections;
+import java.util.Objects;
+
+import static com.linkedin.metadata.Constants.*;
+
+/**
+ * Provides field resolver for domain given resourceSpec
+ */
+@Slf4j
+@RequiredArgsConstructor
+public class DataPlatformInstanceFieldResolverProvider implements ResourceFieldResolverProvider {
+
+  private final EntityClient _entityClient;
+  private final Authentication _systemAuthentication;
+
+  @Override
+  public ResourceFieldType getFieldType() {
+    return ResourceFieldType.DATA_PLATFORM_INSTANCE;
+  }
+
+  @Override
+  public FieldResolver getFieldResolver(ResourceSpec resourceSpec) {
+    return FieldResolver.getResolverFromFunction(resourceSpec, this::getDataPlatformInstance);
+  }
+
+  private FieldResolver.FieldValue getDataPlatformInstance(ResourceSpec resourceSpec) {
+    Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource());
+    // In the case that the entity is a platform instance, the associated platform instance entity is the instance itself
+    if (entityUrn.getEntityType().equals(DATA_PLATFORM_INSTANCE_ENTITY_NAME)) {
+      return FieldResolver.FieldValue.builder()
+          .values(Collections.singleton(entityUrn.toString()))
+          .build();
+    }
+
+    EnvelopedAspect dataPlatformInstanceAspect;
+    try {
+      EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn,
+          Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME), _systemAuthentication);
+      if (response == null || !response.getAspects().containsKey(DATA_PLATFORM_INSTANCE_ASPECT_NAME)) {
+        return FieldResolver.emptyFieldValue();
+      }
+      dataPlatformInstanceAspect = response.getAspects().get(DATA_PLATFORM_INSTANCE_ASPECT_NAME);
+    } catch (Exception e) {
+      log.error("Error while retrieving platform instance aspect for urn {}", entityUrn, e);
+      return FieldResolver.emptyFieldValue();
+    }
+    DataPlatformInstance dataPlatformInstance = new DataPlatformInstance(dataPlatformInstanceAspect.getValue().data());
+    if (dataPlatformInstance.getInstance() == null) {
+      return FieldResolver.emptyFieldValue();
+    }
+    return FieldResolver.FieldValue.builder()
+        .values(Collections.singleton(Objects.requireNonNull(dataPlatformInstance.getInstance()).toString()))
+        .build();
+  }
+}
\ No newline at end of file
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
new file mode 100644
index 0000000000000..e525c602c2620
--- /dev/null
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
@@ -0,0 +1,188 @@
+package com.datahub.authorization.fieldresolverprovider;
+
+import com.datahub.authentication.Authentication;
+import com.datahub.authorization.ResourceFieldType;
+import com.datahub.authorization.ResourceSpec;
+import com.linkedin.common.DataPlatformInstance;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.entity.Aspect;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.r2.RemoteInvocationException;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.Set;
+
+import static com.linkedin.metadata.Constants.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class DataPlatformInstanceFieldResolverProviderTest {
+
+  private static final String DATA_PLATFORM_INSTANCE_URN =
+      "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)";
+  private static final String RESOURCE_URN =
+      "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.testDataset,PROD)";
+  private static final ResourceSpec RESOURCE_SPEC = new ResourceSpec(DATASET_ENTITY_NAME, RESOURCE_URN);
+
+  @Mock
+  private EntityClient entityClientMock;
+  @Mock
+  private Authentication systemAuthenticationMock;
+
+  private DataPlatformInstanceFieldResolverProvider dataPlatformInstanceFieldResolverProvider;
+
+  @BeforeMethod
+  public void setup() {
+    MockitoAnnotations.initMocks(this);
+    dataPlatformInstanceFieldResolverProvider =
+        new DataPlatformInstanceFieldResolverProvider(entityClientMock, systemAuthenticationMock);
+  }
+
+  @Test
+  public void shouldReturnDataPlatformInstanceType() {
+    assertEquals(ResourceFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldType());
+  }
+
+  @Test
+  public void shouldReturnFieldValueWithResourceSpecIfTypeIsDataPlatformInstance() {
+    var resourceSpec = new ResourceSpec(DATA_PLATFORM_INSTANCE_ENTITY_NAME, DATA_PLATFORM_INSTANCE_URN);
+
+    var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(resourceSpec);
+
+    assertEquals(Set.of(DATA_PLATFORM_INSTANCE_URN), result.getFieldValuesFuture().join().getValues());
+    verifyZeroInteractions(entityClientMock);
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenResponseIsNull() throws RemoteInvocationException, URISyntaxException {
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(null);
+
+    var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenResourceHasNoDataPlatformInstance()
+      throws RemoteInvocationException, URISyntaxException {
+    var entityResponseMock = mock(EntityResponse.class);
+    when(entityResponseMock.getAspects()).thenReturn(new EnvelopedAspectMap());
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenThereIsAnException() throws RemoteInvocationException, URISyntaxException {
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenThrow(new RemoteInvocationException());
+
+    var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenDataPlatformInstanceHasNoInstance()
+      throws RemoteInvocationException, URISyntaxException {
+
+    var dataPlatform = new DataPlatformInstance()
+        .setPlatform(Urn.createFromString("urn:li:dataPlatform:s3"));
+    var entityResponseMock = mock(EntityResponse.class);
+    var envelopedAspectMap = new EnvelopedAspectMap();
+    envelopedAspectMap.put(DATA_PLATFORM_INSTANCE_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(dataPlatform.data())));
+    when(entityResponseMock.getAspects()).thenReturn(envelopedAspectMap);
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnFieldValueWithDataPlatformInstanceOfTheResource()
+      throws RemoteInvocationException, URISyntaxException {
+
+    var dataPlatformInstance = new DataPlatformInstance()
+        .setPlatform(Urn.createFromString("urn:li:dataPlatform:s3"))
+        .setInstance(Urn.createFromString(DATA_PLATFORM_INSTANCE_URN));
+    var entityResponseMock = mock(EntityResponse.class);
+    var envelopedAspectMap = new EnvelopedAspectMap();
+    envelopedAspectMap.put(DATA_PLATFORM_INSTANCE_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(dataPlatformInstance.data())));
+    when(entityResponseMock.getAspects()).thenReturn(envelopedAspectMap);
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertEquals(Set.of(DATA_PLATFORM_INSTANCE_URN), result.getFieldValuesFuture().join().getValues());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(Collections.singleton(DATA_PLATFORM_INSTANCE_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+}

From a17db676e37d90ec47f16a43ab95e0d562952939 Mon Sep 17 00:00:00 2001
From: siladitya <68184387+siladitya2@users.noreply.github.com>
Date: Wed, 11 Oct 2023 02:43:36 +0200
Subject: [PATCH 027/259] feat(graphql): Added datafetcher for
 DataPlatformInstance entity (#8935)

Co-authored-by: si-chakraborty <si.chakraborty@adevinta.com>
Co-authored-by: John Joyce <john@acryl.io>
---
 .../datahub/graphql/GmsGraphQLEngine.java     |  1 +
 .../DataPlatformInstanceType.java             | 34 ++++++++++++++++++-
 .../src/main/resources/entity.graphql         |  5 +++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index 3ba0cc1f747e3..ebb5c7d62c7d3 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -821,6 +821,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("glossaryNode", getResolver(glossaryNodeType))
             .dataFetcher("domain", getResolver((domainType)))
             .dataFetcher("dataPlatform", getResolver(dataPlatformType))
+            .dataFetcher("dataPlatformInstance", getResolver(dataPlatformInstanceType))
             .dataFetcher("mlFeatureTable", getResolver(mlFeatureTableType))
             .dataFetcher("mlFeature", getResolver(mlFeatureType))
             .dataFetcher("mlPrimaryKey", getResolver(mlPrimaryKeyType))
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatforminstance/DataPlatformInstanceType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatforminstance/DataPlatformInstanceType.java
index 2423fc31ea52e..87614e1332528 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatforminstance/DataPlatformInstanceType.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatforminstance/DataPlatformInstanceType.java
@@ -4,16 +4,25 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.AutoCompleteResults;
 import com.linkedin.datahub.graphql.generated.DataPlatformInstance;
 import com.linkedin.datahub.graphql.generated.Entity;
 import com.linkedin.datahub.graphql.generated.EntityType;
+import com.linkedin.datahub.graphql.generated.FacetFilterInput;
+import com.linkedin.datahub.graphql.generated.SearchResults;
 import com.linkedin.datahub.graphql.types.dataplatforminstance.mappers.DataPlatformInstanceMapper;
+import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper;
+import com.linkedin.datahub.graphql.types.SearchableEntityType;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.query.AutoCompleteResult;
+import com.linkedin.metadata.query.filter.Filter;
 import graphql.execution.DataFetcherResult;
+import org.apache.commons.lang3.NotImplementedException;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -22,7 +31,10 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-public class DataPlatformInstanceType implements com.linkedin.datahub.graphql.types.EntityType<DataPlatformInstance, String> {
+import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ENTITY_NAME;
+
+public class DataPlatformInstanceType implements SearchableEntityType<DataPlatformInstance, String>,
+        com.linkedin.datahub.graphql.types.EntityType<DataPlatformInstance, String> {
 
     static final Set<String> ASPECTS_TO_FETCH = ImmutableSet.of(
         Constants.DATA_PLATFORM_INSTANCE_KEY_ASPECT_NAME,
@@ -84,4 +96,24 @@ public List<DataFetcherResult<DataPlatformInstance>> batchLoad(@Nonnull List<Str
         }
     }
 
+    @Override
+    public SearchResults search(@Nonnull String query,
+                                @Nullable List<FacetFilterInput> filters,
+                                int start,
+                                int count,
+                                @Nonnull final QueryContext context) throws Exception {
+        throw new NotImplementedException("Searchable type (deprecated) not implemented on DataPlatformInstance entity type");
+    }
+
+    @Override
+    public AutoCompleteResults autoComplete(@Nonnull String query,
+                                            @Nullable String field,
+                                            @Nullable Filter filters,
+                                            int limit,
+                                            @Nonnull final QueryContext context) throws Exception {
+        final AutoCompleteResult result = _entityClient.autoComplete(DATA_PLATFORM_INSTANCE_ENTITY_NAME, query,
+                filters, limit, context.getAuthentication());
+        return AutoCompleteResultsMapper.map(result);
+    }
+
 }
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index 39f86948c77c4..0b15d7b875a9c 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -226,6 +226,11 @@ type Query {
     listOwnershipTypes(
         "Input required for listing custom ownership types"
         input: ListOwnershipTypesInput!): ListOwnershipTypesResult!
+
+    """
+    Fetch a Data Platform Instance by primary key (urn)
+    """
+    dataPlatformInstance(urn: String!): DataPlatformInstance
 }
 
 """

From dfcea2441e75e1eef517c0f9a4765e6e7990f297 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= <sgomezvillamor@gmail.com>
Date: Wed, 11 Oct 2023 03:04:44 +0200
Subject: [PATCH 028/259] feat(config): configurable bootstrap policies file
 (#8812)

Co-authored-by: John Joyce <john@acryl.io>
---
 .../configuration/src/main/resources/application.yml   |  4 ++++
 .../boot/factories/BootstrapManagerFactory.java        |  7 ++++++-
 .../metadata/boot/steps/IngestPoliciesStep.java        | 10 +++++++---
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 4dfd96ac75c6c..d22f92adca8f9 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -276,6 +276,10 @@ bootstrap:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
   backfillBrowsePathsV2:
     enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
+  policies:
+    file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
+    # eg for local file
+    # file: "file:///datahub/datahub-gms/resources/custom-policies.json"
   servlets:
     waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize
 
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
index c490f00021201..3a761bd12647e 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
@@ -31,6 +31,7 @@
 import com.linkedin.metadata.search.EntitySearchService;
 import com.linkedin.metadata.search.SearchService;
 import com.linkedin.metadata.search.transformer.SearchDocumentTransformer;
+
 import java.util.ArrayList;
 import java.util.List;
 import javax.annotation.Nonnull;
@@ -41,6 +42,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Scope;
+import org.springframework.core.io.Resource;
 
 
 @Configuration
@@ -89,13 +91,16 @@ public class BootstrapManagerFactory {
   @Value("${bootstrap.backfillBrowsePathsV2.enabled}")
   private Boolean _backfillBrowsePathsV2Enabled;
 
+  @Value("${bootstrap.policies.file}")
+  private Resource _policiesResource;
+
   @Bean(name = "bootstrapManager")
   @Scope("singleton")
   @Nonnull
   protected BootstrapManager createInstance() {
     final IngestRootUserStep ingestRootUserStep = new IngestRootUserStep(_entityService);
     final IngestPoliciesStep ingestPoliciesStep =
-        new IngestPoliciesStep(_entityRegistry, _entityService, _entitySearchService, _searchDocumentTransformer);
+        new IngestPoliciesStep(_entityRegistry, _entityService, _entitySearchService, _searchDocumentTransformer, _policiesResource);
     final IngestRolesStep ingestRolesStep = new IngestRolesStep(_entityService, _entityRegistry);
     final IngestDataPlatformsStep ingestDataPlatformsStep = new IngestDataPlatformsStep(_entityService);
     final IngestDataPlatformInstancesStep ingestDataPlatformInstancesStep =
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
index 87dcfd736da40..cf29645214466 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
@@ -25,6 +25,7 @@
 import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.policy.DataHubPolicyInfo;
+
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.Collections;
@@ -35,7 +36,8 @@
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
 
 import static com.linkedin.metadata.Constants.*;
 
@@ -52,6 +54,8 @@ public class IngestPoliciesStep implements BootstrapStep {
   private final EntitySearchService _entitySearchService;
   private final SearchDocumentTransformer _searchDocumentTransformer;
 
+  private final Resource _policiesResource;
+
   @Override
   public String name() {
     return "IngestPoliciesStep";
@@ -66,10 +70,10 @@ public void execute() throws IOException, URISyntaxException {
         .maxStringLength(maxSize).build());
 
     // 0. Execute preflight check to see whether we need to ingest policies
-    log.info("Ingesting default access policies...");
+    log.info("Ingesting default access policies from: {}...", _policiesResource);
 
     // 1. Read from the file into JSON.
-    final JsonNode policiesObj = mapper.readTree(new ClassPathResource("./boot/policies.json").getFile());
+    final JsonNode policiesObj = mapper.readTree(_policiesResource.getFile());
 
     if (!policiesObj.isArray()) {
       throw new RuntimeException(

From 10a190470e8c932b6d34cba49de7dbcba687a088 Mon Sep 17 00:00:00 2001
From: siddiquebagwan-gslab <mohdsiddique.bagwan@gslab.com>
Date: Wed, 11 Oct 2023 08:54:08 +0530
Subject: [PATCH 029/259] feat(ingestion/redshift): CLL support in redshift
 (#8921)

---
 .../ingestion/source/redshift/config.py       |   4 +
 .../ingestion/source/redshift/lineage.py      | 215 +++++++++++++-----
 .../ingestion/source/redshift/redshift.py     |   1 +
 .../tests/unit/test_redshift_lineage.py       |  95 ++++++--
 4 files changed, 234 insertions(+), 81 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
index 804a14b0fe1cf..2789b800940db 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -132,6 +132,10 @@ class RedshiftConfig(
         description="Whether `schema_pattern` is matched against fully qualified schema name `<database>.<schema>`.",
     )
 
+    extract_column_level_lineage: bool = Field(
+        default=True, description="Whether to extract column level lineage."
+    )
+
     @root_validator(pre=True)
     def check_email_is_set_on_usage(cls, values):
         if values.get("include_usage_statistics"):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
index bbe52b5d98ba3..c9ddfbe92ab2a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
@@ -9,10 +9,12 @@
 
 import humanfriendly
 import redshift_connector
-from sqllineage.runner import LineageRunner
 
+import datahub.emitter.mce_builder as builder
+import datahub.utilities.sqlglot_lineage as sqlglot_l
 from datahub.emitter import mce_builder
 from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.aws.s3_util import strip_s3_prefix
 from datahub.ingestion.source.redshift.common import get_db_name
 from datahub.ingestion.source.redshift.config import LineageMode, RedshiftConfig
@@ -28,13 +30,19 @@
 from datahub.ingestion.source.state.redundant_run_skip_handler import (
     RedundantLineageRunSkipHandler,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage
+from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
+    FineGrainedLineage,
+    FineGrainedLineageDownstreamType,
+    FineGrainedLineageUpstreamType,
+    UpstreamLineage,
+)
 from datahub.metadata.schema_classes import (
     DatasetLineageTypeClass,
     UpstreamClass,
     UpstreamLineageClass,
 )
 from datahub.utilities import memory_footprint
+from datahub.utilities.urns import dataset_urn
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -56,13 +64,14 @@ class LineageCollectorType(Enum):
 @dataclass(frozen=True, eq=True)
 class LineageDataset:
     platform: LineageDatasetPlatform
-    path: str
+    urn: str
 
 
 @dataclass()
 class LineageItem:
     dataset: LineageDataset
     upstreams: Set[LineageDataset]
+    cll: Optional[List[sqlglot_l.ColumnLineageInfo]]
     collector_type: LineageCollectorType
     dataset_lineage_type: str = field(init=False)
 
@@ -83,10 +92,12 @@ def __init__(
         self,
         config: RedshiftConfig,
         report: RedshiftReport,
+        context: PipelineContext,
         redundant_run_skip_handler: Optional[RedundantLineageRunSkipHandler] = None,
     ):
         self.config = config
         self.report = report
+        self.context = context
         self._lineage_map: Dict[str, LineageItem] = defaultdict()
 
         self.redundant_run_skip_handler = redundant_run_skip_handler
@@ -121,33 +132,37 @@ def _get_s3_path(self, path: str) -> str:
 
         return path
 
-    def _get_sources_from_query(self, db_name: str, query: str) -> List[LineageDataset]:
+    def _get_sources_from_query(
+        self, db_name: str, query: str
+    ) -> Tuple[List[LineageDataset], Optional[List[sqlglot_l.ColumnLineageInfo]]]:
         sources: List[LineageDataset] = list()
 
-        parser = LineageRunner(query)
+        parsed_result: Optional[
+            sqlglot_l.SqlParsingResult
+        ] = sqlglot_l.create_lineage_sql_parsed_result(
+            query=query,
+            platform=LineageDatasetPlatform.REDSHIFT.value,
+            platform_instance=self.config.platform_instance,
+            database=db_name,
+            schema=str(self.config.default_schema),
+            graph=self.context.graph,
+            env=self.config.env,
+        )
 
-        for table in parser.source_tables:
-            split = str(table).split(".")
-            if len(split) == 3:
-                db_name, source_schema, source_table = split
-            elif len(split) == 2:
-                source_schema, source_table = split
-            else:
-                raise ValueError(
-                    f"Invalid table name {table} in query {query}. "
-                    f"Expected format: [db_name].[schema].[table] or [schema].[table] or [table]."
-                )
+        if parsed_result is None:
+            logger.debug(f"native query parsing failed for {query}")
+            return sources, None
 
-            if source_schema == "<default>":
-                source_schema = str(self.config.default_schema)
+        logger.debug(f"parsed_result = {parsed_result}")
 
+        for table_urn in parsed_result.in_tables:
             source = LineageDataset(
                 platform=LineageDatasetPlatform.REDSHIFT,
-                path=f"{db_name}.{source_schema}.{source_table}",
+                urn=table_urn,
             )
             sources.append(source)
 
-        return sources
+        return sources, parsed_result.column_lineage
 
     def _build_s3_path_from_row(self, filename: str) -> str:
         path = filename.strip()
@@ -165,9 +180,11 @@ def _get_sources(
         source_table: Optional[str],
         ddl: Optional[str],
         filename: Optional[str],
-    ) -> List[LineageDataset]:
+    ) -> Tuple[List[LineageDataset], Optional[List[sqlglot_l.ColumnLineageInfo]]]:
         sources: List[LineageDataset] = list()
         # Source
+        cll: Optional[List[sqlglot_l.ColumnLineageInfo]] = None
+
         if (
             lineage_type
             in {
@@ -177,7 +194,7 @@ def _get_sources(
             and ddl is not None
         ):
             try:
-                sources = self._get_sources_from_query(db_name=db_name, query=ddl)
+                sources, cll = self._get_sources_from_query(db_name=db_name, query=ddl)
             except Exception as e:
                 logger.warning(
                     f"Error parsing query {ddl} for getting lineage. Error was {e}."
@@ -192,22 +209,38 @@ def _get_sources(
                         "Only s3 source supported with copy. The source was: {path}."
                     )
                     self.report.num_lineage_dropped_not_support_copy_path += 1
-                    return sources
+                    return sources, cll
                 path = strip_s3_prefix(self._get_s3_path(path))
+                urn = make_dataset_urn_with_platform_instance(
+                    platform=platform.value,
+                    name=path,
+                    env=self.config.env,
+                    platform_instance=self.config.platform_instance_map.get(
+                        platform.value
+                    )
+                    if self.config.platform_instance_map is not None
+                    else None,
+                )
             elif source_schema is not None and source_table is not None:
                 platform = LineageDatasetPlatform.REDSHIFT
                 path = f"{db_name}.{source_schema}.{source_table}"
+                urn = make_dataset_urn_with_platform_instance(
+                    platform=platform.value,
+                    platform_instance=self.config.platform_instance,
+                    name=path,
+                    env=self.config.env,
+                )
             else:
-                return []
+                return [], cll
 
             sources = [
                 LineageDataset(
                     platform=platform,
-                    path=path,
+                    urn=urn,
                 )
             ]
 
-        return sources
+        return sources, cll
 
     def _populate_lineage_map(
         self,
@@ -231,6 +264,7 @@ def _populate_lineage_map(
         :rtype: None
         """
         try:
+            cll: Optional[List[sqlglot_l.ColumnLineageInfo]] = None
             raw_db_name = database
             alias_db_name = get_db_name(self.config)
 
@@ -243,7 +277,7 @@ def _populate_lineage_map(
                 if not target:
                     continue
 
-                sources = self._get_sources(
+                sources, cll = self._get_sources(
                     lineage_type,
                     alias_db_name,
                     source_schema=lineage_row.source_schema,
@@ -251,6 +285,7 @@ def _populate_lineage_map(
                     ddl=lineage_row.ddl,
                     filename=lineage_row.filename,
                 )
+                target.cll = cll
 
                 target.upstreams.update(
                     self._get_upstream_lineages(
@@ -262,20 +297,16 @@ def _populate_lineage_map(
                 )
 
                 # Merging downstreams if dataset already exists and has downstreams
-                if target.dataset.path in self._lineage_map:
-                    self._lineage_map[
-                        target.dataset.path
-                    ].upstreams = self._lineage_map[
-                        target.dataset.path
-                    ].upstreams.union(
-                        target.upstreams
-                    )
+                if target.dataset.urn in self._lineage_map:
+                    self._lineage_map[target.dataset.urn].upstreams = self._lineage_map[
+                        target.dataset.urn
+                    ].upstreams.union(target.upstreams)
 
                 else:
-                    self._lineage_map[target.dataset.path] = target
+                    self._lineage_map[target.dataset.urn] = target
 
                 logger.debug(
-                    f"Lineage[{target}]:{self._lineage_map[target.dataset.path]}"
+                    f"Lineage[{target}]:{self._lineage_map[target.dataset.urn]}"
                 )
         except Exception as e:
             self.warn(
@@ -308,17 +339,34 @@ def _get_target_lineage(
                 target_platform = LineageDatasetPlatform.S3
                 # Following call requires 'filename' key in lineage_row
                 target_path = self._build_s3_path_from_row(lineage_row.filename)
+                urn = make_dataset_urn_with_platform_instance(
+                    platform=target_platform.value,
+                    name=target_path,
+                    env=self.config.env,
+                    platform_instance=self.config.platform_instance_map.get(
+                        target_platform.value
+                    )
+                    if self.config.platform_instance_map is not None
+                    else None,
+                )
             except ValueError as e:
                 self.warn(logger, "non-s3-lineage", str(e))
                 return None
         else:
             target_platform = LineageDatasetPlatform.REDSHIFT
             target_path = f"{alias_db_name}.{lineage_row.target_schema}.{lineage_row.target_table}"
+            urn = make_dataset_urn_with_platform_instance(
+                platform=target_platform.value,
+                platform_instance=self.config.platform_instance,
+                name=target_path,
+                env=self.config.env,
+            )
 
         return LineageItem(
-            dataset=LineageDataset(platform=target_platform, path=target_path),
+            dataset=LineageDataset(platform=target_platform, urn=urn),
             upstreams=set(),
             collector_type=lineage_type,
+            cll=None,
         )
 
     def _get_upstream_lineages(
@@ -331,11 +379,22 @@ def _get_upstream_lineages(
         targe_source = []
         for source in sources:
             if source.platform == LineageDatasetPlatform.REDSHIFT:
-                db, schema, table = source.path.split(".")
+                qualified_table_name = dataset_urn.DatasetUrn.create_from_string(
+                    source.urn
+                ).get_entity_id()[1]
+                db, schema, table = qualified_table_name.split(".")
                 if db == raw_db_name:
                     db = alias_db_name
                     path = f"{db}.{schema}.{table}"
-                    source = LineageDataset(platform=source.platform, path=path)
+                    source = LineageDataset(
+                        platform=source.platform,
+                        urn=make_dataset_urn_with_platform_instance(
+                            platform=LineageDatasetPlatform.REDSHIFT.value,
+                            platform_instance=self.config.platform_instance,
+                            name=path,
+                            env=self.config.env,
+                        ),
+                    )
 
                 # Filtering out tables which does not exist in Redshift
                 # It was deleted in the meantime or query parser did not capture well the table name
@@ -345,7 +404,7 @@ def _get_upstream_lineages(
                     or not any(table == t.name for t in all_tables[db][schema])
                 ):
                     logger.debug(
-                        f"{source.path} missing table, dropping from lineage.",
+                        f"{source.urn} missing table, dropping from lineage.",
                     )
                     self.report.num_lineage_tables_dropped += 1
                     continue
@@ -433,36 +492,73 @@ def populate_lineage(
             memory_footprint.total_size(self._lineage_map)
         )
 
+    def make_fine_grained_lineage_class(
+        self, lineage_item: LineageItem, dataset_urn: str
+    ) -> List[FineGrainedLineage]:
+        fine_grained_lineages: List[FineGrainedLineage] = []
+
+        if (
+            self.config.extract_column_level_lineage is False
+            or lineage_item.cll is None
+        ):
+            logger.debug("CLL extraction is disabled")
+            return fine_grained_lineages
+
+        logger.debug("Extracting column level lineage")
+
+        cll: List[sqlglot_l.ColumnLineageInfo] = lineage_item.cll
+
+        for cll_info in cll:
+            downstream = (
+                [builder.make_schema_field_urn(dataset_urn, cll_info.downstream.column)]
+                if cll_info.downstream is not None
+                and cll_info.downstream.column is not None
+                else []
+            )
+
+            upstreams = [
+                builder.make_schema_field_urn(column_ref.table, column_ref.column)
+                for column_ref in cll_info.upstreams
+            ]
+
+            fine_grained_lineages.append(
+                FineGrainedLineage(
+                    downstreamType=FineGrainedLineageDownstreamType.FIELD,
+                    downstreams=downstream,
+                    upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
+                    upstreams=upstreams,
+                )
+            )
+
+        logger.debug(f"Created fine_grained_lineage for {dataset_urn}")
+
+        return fine_grained_lineages
+
     def get_lineage(
         self,
         table: Union[RedshiftTable, RedshiftView],
         dataset_urn: str,
         schema: RedshiftSchema,
     ) -> Optional[Tuple[UpstreamLineageClass, Dict[str, str]]]:
-        dataset_key = mce_builder.dataset_urn_to_key(dataset_urn)
-        if dataset_key is None:
-            return None
 
         upstream_lineage: List[UpstreamClass] = []
 
-        if dataset_key.name in self._lineage_map:
-            item = self._lineage_map[dataset_key.name]
+        cll_lineage: List[FineGrainedLineage] = []
+
+        if dataset_urn in self._lineage_map:
+            item = self._lineage_map[dataset_urn]
             for upstream in item.upstreams:
                 upstream_table = UpstreamClass(
-                    dataset=make_dataset_urn_with_platform_instance(
-                        upstream.platform.value,
-                        upstream.path,
-                        platform_instance=self.config.platform_instance_map.get(
-                            upstream.platform.value
-                        )
-                        if self.config.platform_instance_map
-                        else None,
-                        env=self.config.env,
-                    ),
+                    dataset=upstream.urn,
                     type=item.dataset_lineage_type,
                 )
                 upstream_lineage.append(upstream_table)
 
+            cll_lineage = self.make_fine_grained_lineage_class(
+                lineage_item=item,
+                dataset_urn=dataset_urn,
+            )
+
         tablename = table.name
         if table.type == "EXTERNAL_TABLE":
             # external_db_params = schema.option
@@ -489,7 +585,12 @@ def get_lineage(
         else:
             return None
 
-        return UpstreamLineage(upstreams=upstream_lineage), {}
+        return (
+            UpstreamLineage(
+                upstreams=upstream_lineage, fineGrainedLineages=cll_lineage or None
+            ),
+            {},
+        )
 
     def report_status(self, step: str, status: bool) -> None:
         if self.redundant_run_skip_handler:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
index e8a8ff976afa6..a1b6333a3775d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -881,6 +881,7 @@ def extract_lineage(
         self.lineage_extractor = RedshiftLineageExtractor(
             config=self.config,
             report=self.report,
+            context=self.ctx,
             redundant_run_skip_handler=self.redundant_lineage_run_skip_handler,
         )
 
diff --git a/metadata-ingestion/tests/unit/test_redshift_lineage.py b/metadata-ingestion/tests/unit/test_redshift_lineage.py
index c7d6ac18e044c..db5af3a71efb9 100644
--- a/metadata-ingestion/tests/unit/test_redshift_lineage.py
+++ b/metadata-ingestion/tests/unit/test_redshift_lineage.py
@@ -1,6 +1,8 @@
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.redshift.config import RedshiftConfig
 from datahub.ingestion.source.redshift.lineage import RedshiftLineageExtractor
 from datahub.ingestion.source.redshift.report import RedshiftReport
+from datahub.utilities.sqlglot_lineage import ColumnLineageInfo, DownstreamColumnRef
 
 
 def test_get_sources_from_query():
@@ -10,14 +12,20 @@ def test_get_sources_from_query():
     test_query = """
         select * from my_schema.my_table
     """
-    lineage_extractor = RedshiftLineageExtractor(config, report)
-    lineage_datasets = lineage_extractor._get_sources_from_query(
+    lineage_extractor = RedshiftLineageExtractor(
+        config, report, PipelineContext(run_id="foo")
+    )
+    lineage_datasets, _ = lineage_extractor._get_sources_from_query(
         db_name="test", query=test_query
     )
     assert len(lineage_datasets) == 1
 
     lineage = lineage_datasets[0]
-    assert lineage.path == "test.my_schema.my_table"
+
+    assert (
+        lineage.urn
+        == "urn:li:dataset:(urn:li:dataPlatform:redshift,test.my_schema.my_table,PROD)"
+    )
 
 
 def test_get_sources_from_query_with_only_table_name():
@@ -27,14 +35,20 @@ def test_get_sources_from_query_with_only_table_name():
     test_query = """
         select * from my_table
     """
-    lineage_extractor = RedshiftLineageExtractor(config, report)
-    lineage_datasets = lineage_extractor._get_sources_from_query(
+    lineage_extractor = RedshiftLineageExtractor(
+        config, report, PipelineContext(run_id="foo")
+    )
+    lineage_datasets, _ = lineage_extractor._get_sources_from_query(
         db_name="test", query=test_query
     )
     assert len(lineage_datasets) == 1
 
     lineage = lineage_datasets[0]
-    assert lineage.path == "test.public.my_table"
+
+    assert (
+        lineage.urn
+        == "urn:li:dataset:(urn:li:dataPlatform:redshift,test.public.my_table,PROD)"
+    )
 
 
 def test_get_sources_from_query_with_database():
@@ -44,14 +58,20 @@ def test_get_sources_from_query_with_database():
     test_query = """
         select * from test.my_schema.my_table
     """
-    lineage_extractor = RedshiftLineageExtractor(config, report)
-    lineage_datasets = lineage_extractor._get_sources_from_query(
+    lineage_extractor = RedshiftLineageExtractor(
+        config, report, PipelineContext(run_id="foo")
+    )
+    lineage_datasets, _ = lineage_extractor._get_sources_from_query(
         db_name="test", query=test_query
     )
     assert len(lineage_datasets) == 1
 
     lineage = lineage_datasets[0]
-    assert lineage.path == "test.my_schema.my_table"
+
+    assert (
+        lineage.urn
+        == "urn:li:dataset:(urn:li:dataPlatform:redshift,test.my_schema.my_table,PROD)"
+    )
 
 
 def test_get_sources_from_query_with_non_default_database():
@@ -61,14 +81,20 @@ def test_get_sources_from_query_with_non_default_database():
     test_query = """
         select * from test2.my_schema.my_table
     """
-    lineage_extractor = RedshiftLineageExtractor(config, report)
-    lineage_datasets = lineage_extractor._get_sources_from_query(
+    lineage_extractor = RedshiftLineageExtractor(
+        config, report, PipelineContext(run_id="foo")
+    )
+    lineage_datasets, _ = lineage_extractor._get_sources_from_query(
         db_name="test", query=test_query
     )
     assert len(lineage_datasets) == 1
 
     lineage = lineage_datasets[0]
-    assert lineage.path == "test2.my_schema.my_table"
+
+    assert (
+        lineage.urn
+        == "urn:li:dataset:(urn:li:dataPlatform:redshift,test2.my_schema.my_table,PROD)"
+    )
 
 
 def test_get_sources_from_query_with_only_table():
@@ -78,27 +104,48 @@ def test_get_sources_from_query_with_only_table():
     test_query = """
         select * from my_table
     """
-    lineage_extractor = RedshiftLineageExtractor(config, report)
-    lineage_datasets = lineage_extractor._get_sources_from_query(
+    lineage_extractor = RedshiftLineageExtractor(
+        config, report, PipelineContext(run_id="foo")
+    )
+    lineage_datasets, _ = lineage_extractor._get_sources_from_query(
         db_name="test", query=test_query
     )
     assert len(lineage_datasets) == 1
 
     lineage = lineage_datasets[0]
-    assert lineage.path == "test.public.my_table"
+
+    assert (
+        lineage.urn
+        == "urn:li:dataset:(urn:li:dataPlatform:redshift,test.public.my_table,PROD)"
+    )
 
 
-def test_get_sources_from_query_with_four_part_table_should_throw_exception():
+def test_cll():
     config = RedshiftConfig(host_port="localhost:5439", database="test")
     report = RedshiftReport()
 
     test_query = """
-        select * from database.schema.my_table.test
+        select a,b,c from db.public.customer inner join db.public.order on db.public.customer.id = db.public.order.customer_id
     """
-    lineage_extractor = RedshiftLineageExtractor(config, report)
-    try:
-        lineage_extractor._get_sources_from_query(db_name="test", query=test_query)
-    except ValueError:
-        pass
-
-    assert f"{test_query} should have thrown a ValueError exception but it didn't"
+    lineage_extractor = RedshiftLineageExtractor(
+        config, report, PipelineContext(run_id="foo")
+    )
+    _, cll = lineage_extractor._get_sources_from_query(db_name="db", query=test_query)
+
+    assert cll == [
+        ColumnLineageInfo(
+            downstream=DownstreamColumnRef(table=None, column="a"),
+            upstreams=[],
+            logic=None,
+        ),
+        ColumnLineageInfo(
+            downstream=DownstreamColumnRef(table=None, column="b"),
+            upstreams=[],
+            logic=None,
+        ),
+        ColumnLineageInfo(
+            downstream=DownstreamColumnRef(table=None, column="c"),
+            upstreams=[],
+            logic=None,
+        ),
+    ]

From 4b6b941a2abf13854511c9af0e88a17d5acfd5e6 Mon Sep 17 00:00:00 2001
From: Harsha Mandadi <115464537+harsha-mandadi-4026@users.noreply.github.com>
Date: Wed, 11 Oct 2023 19:01:46 +0100
Subject: [PATCH 030/259] fix(ingest): Fix postgres lineage within views
 (#8906)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
Co-authored-by: Maggie Hays <maggiem.hays@gmail.com>
---
 .../datahub/ingestion/source/sql/postgres.py   | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
index ba8655b83446d..a6a9d8e2c8597 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
@@ -217,14 +217,15 @@ def _get_view_lineage_elements(
             key = (lineage.dependent_view, lineage.dependent_schema)
             # Append the source table to the list.
             lineage_elements[key].append(
-                mce_builder.make_dataset_urn(
-                    self.platform,
-                    self.get_identifier(
+                mce_builder.make_dataset_urn_with_platform_instance(
+                    platform=self.platform,
+                    name=self.get_identifier(
                         schema=lineage.source_schema,
                         entity=lineage.source_table,
                         inspector=inspector,
                     ),
-                    self.config.env,
+                    platform_instance=self.config.platform_instance,
+                    env=self.config.env,
                 )
             )
 
@@ -244,12 +245,13 @@ def _get_view_lineage_workunits(
             dependent_view, dependent_schema = key
 
             # Construct a lineage object.
-            urn = mce_builder.make_dataset_urn(
-                self.platform,
-                self.get_identifier(
+            urn = mce_builder.make_dataset_urn_with_platform_instance(
+                platform=self.platform,
+                name=self.get_identifier(
                     schema=dependent_schema, entity=dependent_view, inspector=inspector
                 ),
-                self.config.env,
+                platform_instance=self.config.platform_instance,
+                env=self.config.env,
             )
 
             # use the mce_builder to ensure that the change proposal inherits

From 932fbcddbf7c3201898e0918218e80c9246b0cd2 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Wed, 11 Oct 2023 14:17:02 -0400
Subject: [PATCH 031/259] refactor(ingest/dbt): move dbt tests logic to
 dedicated file (#8984)

---
 .../src/datahub/ingestion/api/common.py       |   9 +
 .../datahub/ingestion/source/csv_enricher.py  |   8 +-
 .../datahub/ingestion/source/dbt/dbt_cloud.py |   3 +-
 .../ingestion/source/dbt/dbt_common.py        | 278 +-----------------
 .../datahub/ingestion/source/dbt/dbt_core.py  |   3 +-
 .../datahub/ingestion/source/dbt/dbt_tests.py | 261 ++++++++++++++++
 6 files changed, 288 insertions(+), 274 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py

diff --git a/metadata-ingestion/src/datahub/ingestion/api/common.py b/metadata-ingestion/src/datahub/ingestion/api/common.py
index 778bd119615e2..a6761a3c77d5e 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/common.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/common.py
@@ -2,6 +2,7 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Dict, Generic, Iterable, Optional, Tuple, TypeVar
 
+from datahub.configuration.common import ConfigurationError
 from datahub.emitter.mce_builder import set_dataset_urn_to_lower
 from datahub.ingestion.api.committable import Committable
 from datahub.ingestion.graph.client import DataHubGraph
@@ -75,3 +76,11 @@ def register_checkpointer(self, committable: Committable) -> None:
 
     def get_committables(self) -> Iterable[Tuple[str, Committable]]:
         yield from self.checkpointers.items()
+
+    def require_graph(self, operation: Optional[str] = None) -> DataHubGraph:
+        if not self.graph:
+            raise ConfigurationError(
+                f"{operation or 'This operation'} requires a graph, but none was provided. "
+                "To provide one, either use the datahub-rest sink or set the top-level datahub_api config in the recipe."
+            )
+        return self.graph
diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py
index 7cb487a86d931..611f0c5c52cc6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py
@@ -129,11 +129,9 @@ def __init__(self, config: CSVEnricherConfig, ctx: PipelineContext):
         # Map from entity urn to a list of SubResourceRow.
         self.editable_schema_metadata_map: Dict[str, List[SubResourceRow]] = {}
         self.should_overwrite: bool = self.config.write_semantics == "OVERRIDE"
-        if not self.should_overwrite and not self.ctx.graph:
-            raise ConfigurationError(
-                "With PATCH semantics, the csv-enricher source requires a datahub_api to connect to. "
-                "Consider using the datahub-rest sink or provide a datahub_api: configuration on your ingestion recipe."
-            )
+
+        if not self.should_overwrite:
+            self.ctx.require_graph(operation="The csv-enricher's PATCH semantics flag")
 
     def get_resource_glossary_terms_work_unit(
         self,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
index af9769bc9d94c..da1ea8ecb4678 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
@@ -20,9 +20,8 @@
     DBTCommonConfig,
     DBTNode,
     DBTSourceBase,
-    DBTTest,
-    DBTTestResult,
 )
+from datahub.ingestion.source.dbt.dbt_tests import DBTTest, DBTTestResult
 
 logger = logging.getLogger(__name__)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 0f5c08eb6ac54..48d2118a9b091 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -1,11 +1,10 @@
-import json
 import logging
 import re
 from abc import abstractmethod
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import auto
-from typing import Any, Callable, ClassVar, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 import pydantic
 from pydantic import root_validator, validator
@@ -34,6 +33,12 @@
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import DatasetSubTypes
+from datahub.ingestion.source.dbt.dbt_tests import (
+    DBTTest,
+    DBTTestResult,
+    make_assertion_from_test,
+    make_assertion_result_from_test,
+)
 from datahub.ingestion.source.sql.sql_types import (
     ATHENA_SQL_TYPES_MAP,
     BIGQUERY_TYPES_MAP,
@@ -81,20 +86,7 @@
     TimeTypeClass,
 )
 from datahub.metadata.schema_classes import (
-    AssertionInfoClass,
-    AssertionResultClass,
-    AssertionResultTypeClass,
-    AssertionRunEventClass,
-    AssertionRunStatusClass,
-    AssertionStdAggregationClass,
-    AssertionStdOperatorClass,
-    AssertionStdParameterClass,
-    AssertionStdParametersClass,
-    AssertionStdParameterTypeClass,
-    AssertionTypeClass,
     DataPlatformInstanceClass,
-    DatasetAssertionInfoClass,
-    DatasetAssertionScopeClass,
     DatasetPropertiesClass,
     GlobalTagsClass,
     GlossaryTermsClass,
@@ -551,134 +543,6 @@ def get_column_type(
     return SchemaFieldDataType(type=TypeClass())
 
 
-@dataclass
-class AssertionParams:
-    scope: Union[DatasetAssertionScopeClass, str]
-    operator: Union[AssertionStdOperatorClass, str]
-    aggregation: Union[AssertionStdAggregationClass, str]
-    parameters: Optional[Callable[[Dict[str, str]], AssertionStdParametersClass]] = None
-    logic_fn: Optional[Callable[[Dict[str, str]], Optional[str]]] = None
-
-
-def _get_name_for_relationship_test(kw_args: Dict[str, str]) -> Optional[str]:
-    """
-    Try to produce a useful string for the name of a relationship constraint.
-    Return None if we fail to
-    """
-    destination_ref = kw_args.get("to")
-    source_ref = kw_args.get("model")
-    column_name = kw_args.get("column_name")
-    dest_field_name = kw_args.get("field")
-    if not destination_ref or not source_ref or not column_name or not dest_field_name:
-        # base assertions are violated, bail early
-        return None
-    m = re.match(r"^ref\(\'(.*)\'\)$", destination_ref)
-    if m:
-        destination_table = m.group(1)
-    else:
-        destination_table = destination_ref
-    m = re.search(r"ref\(\'(.*)\'\)", source_ref)
-    if m:
-        source_table = m.group(1)
-    else:
-        source_table = source_ref
-    return f"{source_table}.{column_name} referential integrity to {destination_table}.{dest_field_name}"
-
-
-@dataclass
-class DBTTest:
-    qualified_test_name: str
-    column_name: Optional[str]
-    kw_args: dict
-
-    TEST_NAME_TO_ASSERTION_MAP: ClassVar[Dict[str, AssertionParams]] = {
-        "not_null": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass.NOT_NULL,
-            aggregation=AssertionStdAggregationClass.IDENTITY,
-        ),
-        "unique": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass.EQUAL_TO,
-            aggregation=AssertionStdAggregationClass.UNIQUE_PROPOTION,
-            parameters=lambda _: AssertionStdParametersClass(
-                value=AssertionStdParameterClass(
-                    value="1.0",
-                    type=AssertionStdParameterTypeClass.NUMBER,
-                )
-            ),
-        ),
-        "accepted_values": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass.IN,
-            aggregation=AssertionStdAggregationClass.IDENTITY,
-            parameters=lambda kw_args: AssertionStdParametersClass(
-                value=AssertionStdParameterClass(
-                    value=json.dumps(kw_args.get("values")),
-                    type=AssertionStdParameterTypeClass.SET,
-                ),
-            ),
-        ),
-        "relationships": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass._NATIVE_,
-            aggregation=AssertionStdAggregationClass.IDENTITY,
-            parameters=lambda kw_args: AssertionStdParametersClass(
-                value=AssertionStdParameterClass(
-                    value=json.dumps(kw_args.get("values")),
-                    type=AssertionStdParameterTypeClass.SET,
-                ),
-            ),
-            logic_fn=_get_name_for_relationship_test,
-        ),
-        "dbt_expectations.expect_column_values_to_not_be_null": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass.NOT_NULL,
-            aggregation=AssertionStdAggregationClass.IDENTITY,
-        ),
-        "dbt_expectations.expect_column_values_to_be_between": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass.BETWEEN,
-            aggregation=AssertionStdAggregationClass.IDENTITY,
-            parameters=lambda x: AssertionStdParametersClass(
-                minValue=AssertionStdParameterClass(
-                    value=str(x.get("min_value", "unknown")),
-                    type=AssertionStdParameterTypeClass.NUMBER,
-                ),
-                maxValue=AssertionStdParameterClass(
-                    value=str(x.get("max_value", "unknown")),
-                    type=AssertionStdParameterTypeClass.NUMBER,
-                ),
-            ),
-        ),
-        "dbt_expectations.expect_column_values_to_be_in_set": AssertionParams(
-            scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-            operator=AssertionStdOperatorClass.IN,
-            aggregation=AssertionStdAggregationClass.IDENTITY,
-            parameters=lambda kw_args: AssertionStdParametersClass(
-                value=AssertionStdParameterClass(
-                    value=json.dumps(kw_args.get("value_set")),
-                    type=AssertionStdParameterTypeClass.SET,
-                ),
-            ),
-        ),
-    }
-
-
-@dataclass
-class DBTTestResult:
-    invocation_id: str
-
-    status: str
-    execution_time: datetime
-
-    native_results: Dict[str, str]
-
-
-def string_map(input_map: Dict[str, Any]) -> Dict[str, str]:
-    return {k: str(v) for k, v in input_map.items()}
-
-
 @platform_name("dbt")
 @config_class(DBTCommonConfig)
 @support_status(SupportStatus.CERTIFIED)
@@ -750,7 +614,7 @@ def create_test_entity_mcps(
 
             for upstream_urn in sorted(upstream_urns):
                 if self.config.entities_enabled.can_emit_node_type("test"):
-                    yield self._make_assertion_from_test(
+                    yield make_assertion_from_test(
                         custom_props,
                         node,
                         assertion_urn,
@@ -759,133 +623,17 @@ def create_test_entity_mcps(
 
                 if node.test_result:
                     if self.config.entities_enabled.can_emit_test_results:
-                        yield self._make_assertion_result_from_test(
-                            node, assertion_urn, upstream_urn
+                        yield make_assertion_result_from_test(
+                            node,
+                            assertion_urn,
+                            upstream_urn,
+                            test_warnings_are_errors=self.config.test_warnings_are_errors,
                         )
                     else:
                         logger.debug(
                             f"Skipping test result {node.name} emission since it is turned off."
                         )
 
-    def _make_assertion_from_test(
-        self,
-        extra_custom_props: Dict[str, str],
-        node: DBTNode,
-        assertion_urn: str,
-        upstream_urn: str,
-    ) -> MetadataWorkUnit:
-        assert node.test_info
-        qualified_test_name = node.test_info.qualified_test_name
-        column_name = node.test_info.column_name
-        kw_args = node.test_info.kw_args
-
-        if qualified_test_name in DBTTest.TEST_NAME_TO_ASSERTION_MAP:
-            assertion_params = DBTTest.TEST_NAME_TO_ASSERTION_MAP[qualified_test_name]
-            assertion_info = AssertionInfoClass(
-                type=AssertionTypeClass.DATASET,
-                customProperties=extra_custom_props,
-                datasetAssertion=DatasetAssertionInfoClass(
-                    dataset=upstream_urn,
-                    scope=assertion_params.scope,
-                    operator=assertion_params.operator,
-                    fields=[
-                        mce_builder.make_schema_field_urn(upstream_urn, column_name)
-                    ]
-                    if (
-                        assertion_params.scope
-                        == DatasetAssertionScopeClass.DATASET_COLUMN
-                        and column_name
-                    )
-                    else [],
-                    nativeType=node.name,
-                    aggregation=assertion_params.aggregation,
-                    parameters=assertion_params.parameters(kw_args)
-                    if assertion_params.parameters
-                    else None,
-                    logic=assertion_params.logic_fn(kw_args)
-                    if assertion_params.logic_fn
-                    else None,
-                    nativeParameters=string_map(kw_args),
-                ),
-            )
-        elif column_name:
-            # no match with known test types, column-level test
-            assertion_info = AssertionInfoClass(
-                type=AssertionTypeClass.DATASET,
-                customProperties=extra_custom_props,
-                datasetAssertion=DatasetAssertionInfoClass(
-                    dataset=upstream_urn,
-                    scope=DatasetAssertionScopeClass.DATASET_COLUMN,
-                    operator=AssertionStdOperatorClass._NATIVE_,
-                    fields=[
-                        mce_builder.make_schema_field_urn(upstream_urn, column_name)
-                    ],
-                    nativeType=node.name,
-                    logic=node.compiled_code or node.raw_code,
-                    aggregation=AssertionStdAggregationClass._NATIVE_,
-                    nativeParameters=string_map(kw_args),
-                ),
-            )
-        else:
-            # no match with known test types, default to row-level test
-            assertion_info = AssertionInfoClass(
-                type=AssertionTypeClass.DATASET,
-                customProperties=extra_custom_props,
-                datasetAssertion=DatasetAssertionInfoClass(
-                    dataset=upstream_urn,
-                    scope=DatasetAssertionScopeClass.DATASET_ROWS,
-                    operator=AssertionStdOperatorClass._NATIVE_,
-                    logic=node.compiled_code or node.raw_code,
-                    nativeType=node.name,
-                    aggregation=AssertionStdAggregationClass._NATIVE_,
-                    nativeParameters=string_map(kw_args),
-                ),
-            )
-
-        wu = MetadataChangeProposalWrapper(
-            entityUrn=assertion_urn,
-            aspect=assertion_info,
-        ).as_workunit()
-
-        return wu
-
-    def _make_assertion_result_from_test(
-        self,
-        node: DBTNode,
-        assertion_urn: str,
-        upstream_urn: str,
-    ) -> MetadataWorkUnit:
-        assert node.test_result
-        test_result = node.test_result
-
-        assertionResult = AssertionRunEventClass(
-            timestampMillis=int(test_result.execution_time.timestamp() * 1000.0),
-            assertionUrn=assertion_urn,
-            asserteeUrn=upstream_urn,
-            runId=test_result.invocation_id,
-            result=AssertionResultClass(
-                type=AssertionResultTypeClass.SUCCESS
-                if test_result.status == "pass"
-                or (
-                    not self.config.test_warnings_are_errors
-                    and test_result.status == "warn"
-                )
-                else AssertionResultTypeClass.FAILURE,
-                nativeResults=test_result.native_results,
-            ),
-            status=AssertionRunStatusClass.COMPLETE,
-        )
-
-        event = MetadataChangeProposalWrapper(
-            entityUrn=assertion_urn,
-            aspect=assertionResult,
-        )
-        wu = MetadataWorkUnit(
-            id=f"{assertion_urn}-assertionRunEvent-{upstream_urn}",
-            mcp=event,
-        )
-        return wu
-
     @abstractmethod
     def load_nodes(self) -> Tuple[List[DBTNode], Dict[str, Optional[str]]]:
         # return dbt nodes + global custom properties
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
index c08295ed1dc59..dc3a84847beb2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
@@ -26,9 +26,8 @@
     DBTNode,
     DBTSourceBase,
     DBTSourceReport,
-    DBTTest,
-    DBTTestResult,
 )
+from datahub.ingestion.source.dbt.dbt_tests import DBTTest, DBTTestResult
 
 logger = logging.getLogger(__name__)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py
new file mode 100644
index 0000000000000..721769d214d9e
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py
@@ -0,0 +1,261 @@
+import json
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union
+
+from datahub.emitter import mce_builder
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.metadata.schema_classes import (
+    AssertionInfoClass,
+    AssertionResultClass,
+    AssertionResultTypeClass,
+    AssertionRunEventClass,
+    AssertionRunStatusClass,
+    AssertionStdAggregationClass,
+    AssertionStdOperatorClass,
+    AssertionStdParameterClass,
+    AssertionStdParametersClass,
+    AssertionStdParameterTypeClass,
+    AssertionTypeClass,
+    DatasetAssertionInfoClass,
+    DatasetAssertionScopeClass,
+)
+
+if TYPE_CHECKING:
+    from datahub.ingestion.source.dbt.dbt_common import DBTNode
+
+
+@dataclass
+class DBTTest:
+    qualified_test_name: str
+    column_name: Optional[str]
+    kw_args: dict
+
+
+@dataclass
+class DBTTestResult:
+    invocation_id: str
+
+    status: str
+    execution_time: datetime
+
+    native_results: Dict[str, str]
+
+
+def _get_name_for_relationship_test(kw_args: Dict[str, str]) -> Optional[str]:
+    """
+    Try to produce a useful string for the name of a relationship constraint.
+    Return None if we fail to
+    """
+    destination_ref = kw_args.get("to")
+    source_ref = kw_args.get("model")
+    column_name = kw_args.get("column_name")
+    dest_field_name = kw_args.get("field")
+    if not destination_ref or not source_ref or not column_name or not dest_field_name:
+        # base assertions are violated, bail early
+        return None
+    m = re.match(r"^ref\(\'(.*)\'\)$", destination_ref)
+    if m:
+        destination_table = m.group(1)
+    else:
+        destination_table = destination_ref
+    m = re.search(r"ref\(\'(.*)\'\)", source_ref)
+    if m:
+        source_table = m.group(1)
+    else:
+        source_table = source_ref
+    return f"{source_table}.{column_name} referential integrity to {destination_table}.{dest_field_name}"
+
+
+@dataclass
+class AssertionParams:
+    scope: Union[DatasetAssertionScopeClass, str]
+    operator: Union[AssertionStdOperatorClass, str]
+    aggregation: Union[AssertionStdAggregationClass, str]
+    parameters: Optional[Callable[[Dict[str, str]], AssertionStdParametersClass]] = None
+    logic_fn: Optional[Callable[[Dict[str, str]], Optional[str]]] = None
+
+
+_DBT_TEST_NAME_TO_ASSERTION_MAP: Dict[str, AssertionParams] = {
+    "not_null": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass.NOT_NULL,
+        aggregation=AssertionStdAggregationClass.IDENTITY,
+    ),
+    "unique": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass.EQUAL_TO,
+        aggregation=AssertionStdAggregationClass.UNIQUE_PROPOTION,
+        parameters=lambda _: AssertionStdParametersClass(
+            value=AssertionStdParameterClass(
+                value="1.0",
+                type=AssertionStdParameterTypeClass.NUMBER,
+            )
+        ),
+    ),
+    "accepted_values": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass.IN,
+        aggregation=AssertionStdAggregationClass.IDENTITY,
+        parameters=lambda kw_args: AssertionStdParametersClass(
+            value=AssertionStdParameterClass(
+                value=json.dumps(kw_args.get("values")),
+                type=AssertionStdParameterTypeClass.SET,
+            ),
+        ),
+    ),
+    "relationships": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass._NATIVE_,
+        aggregation=AssertionStdAggregationClass.IDENTITY,
+        parameters=lambda kw_args: AssertionStdParametersClass(
+            value=AssertionStdParameterClass(
+                value=json.dumps(kw_args.get("values")),
+                type=AssertionStdParameterTypeClass.SET,
+            ),
+        ),
+        logic_fn=_get_name_for_relationship_test,
+    ),
+    "dbt_expectations.expect_column_values_to_not_be_null": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass.NOT_NULL,
+        aggregation=AssertionStdAggregationClass.IDENTITY,
+    ),
+    "dbt_expectations.expect_column_values_to_be_between": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass.BETWEEN,
+        aggregation=AssertionStdAggregationClass.IDENTITY,
+        parameters=lambda x: AssertionStdParametersClass(
+            minValue=AssertionStdParameterClass(
+                value=str(x.get("min_value", "unknown")),
+                type=AssertionStdParameterTypeClass.NUMBER,
+            ),
+            maxValue=AssertionStdParameterClass(
+                value=str(x.get("max_value", "unknown")),
+                type=AssertionStdParameterTypeClass.NUMBER,
+            ),
+        ),
+    ),
+    "dbt_expectations.expect_column_values_to_be_in_set": AssertionParams(
+        scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+        operator=AssertionStdOperatorClass.IN,
+        aggregation=AssertionStdAggregationClass.IDENTITY,
+        parameters=lambda kw_args: AssertionStdParametersClass(
+            value=AssertionStdParameterClass(
+                value=json.dumps(kw_args.get("value_set")),
+                type=AssertionStdParameterTypeClass.SET,
+            ),
+        ),
+    ),
+}
+
+
+def _string_map(input_map: Dict[str, Any]) -> Dict[str, str]:
+    return {k: str(v) for k, v in input_map.items()}
+
+
+def make_assertion_from_test(
+    extra_custom_props: Dict[str, str],
+    node: "DBTNode",
+    assertion_urn: str,
+    upstream_urn: str,
+) -> MetadataWorkUnit:
+    assert node.test_info
+    qualified_test_name = node.test_info.qualified_test_name
+    column_name = node.test_info.column_name
+    kw_args = node.test_info.kw_args
+
+    if qualified_test_name in _DBT_TEST_NAME_TO_ASSERTION_MAP:
+        assertion_params = _DBT_TEST_NAME_TO_ASSERTION_MAP[qualified_test_name]
+        assertion_info = AssertionInfoClass(
+            type=AssertionTypeClass.DATASET,
+            customProperties=extra_custom_props,
+            datasetAssertion=DatasetAssertionInfoClass(
+                dataset=upstream_urn,
+                scope=assertion_params.scope,
+                operator=assertion_params.operator,
+                fields=[mce_builder.make_schema_field_urn(upstream_urn, column_name)]
+                if (
+                    assertion_params.scope == DatasetAssertionScopeClass.DATASET_COLUMN
+                    and column_name
+                )
+                else [],
+                nativeType=node.name,
+                aggregation=assertion_params.aggregation,
+                parameters=assertion_params.parameters(kw_args)
+                if assertion_params.parameters
+                else None,
+                logic=assertion_params.logic_fn(kw_args)
+                if assertion_params.logic_fn
+                else None,
+                nativeParameters=_string_map(kw_args),
+            ),
+        )
+    elif column_name:
+        # no match with known test types, column-level test
+        assertion_info = AssertionInfoClass(
+            type=AssertionTypeClass.DATASET,
+            customProperties=extra_custom_props,
+            datasetAssertion=DatasetAssertionInfoClass(
+                dataset=upstream_urn,
+                scope=DatasetAssertionScopeClass.DATASET_COLUMN,
+                operator=AssertionStdOperatorClass._NATIVE_,
+                fields=[mce_builder.make_schema_field_urn(upstream_urn, column_name)],
+                nativeType=node.name,
+                logic=node.compiled_code or node.raw_code,
+                aggregation=AssertionStdAggregationClass._NATIVE_,
+                nativeParameters=_string_map(kw_args),
+            ),
+        )
+    else:
+        # no match with known test types, default to row-level test
+        assertion_info = AssertionInfoClass(
+            type=AssertionTypeClass.DATASET,
+            customProperties=extra_custom_props,
+            datasetAssertion=DatasetAssertionInfoClass(
+                dataset=upstream_urn,
+                scope=DatasetAssertionScopeClass.DATASET_ROWS,
+                operator=AssertionStdOperatorClass._NATIVE_,
+                logic=node.compiled_code or node.raw_code,
+                nativeType=node.name,
+                aggregation=AssertionStdAggregationClass._NATIVE_,
+                nativeParameters=_string_map(kw_args),
+            ),
+        )
+
+    return MetadataChangeProposalWrapper(
+        entityUrn=assertion_urn,
+        aspect=assertion_info,
+    ).as_workunit()
+
+
+def make_assertion_result_from_test(
+    node: "DBTNode",
+    assertion_urn: str,
+    upstream_urn: str,
+    test_warnings_are_errors: bool,
+) -> MetadataWorkUnit:
+    assert node.test_result
+    test_result = node.test_result
+
+    assertionResult = AssertionRunEventClass(
+        timestampMillis=int(test_result.execution_time.timestamp() * 1000.0),
+        assertionUrn=assertion_urn,
+        asserteeUrn=upstream_urn,
+        runId=test_result.invocation_id,
+        result=AssertionResultClass(
+            type=AssertionResultTypeClass.SUCCESS
+            if test_result.status == "pass"
+            or (not test_warnings_are_errors and test_result.status == "warn")
+            else AssertionResultTypeClass.FAILURE,
+            nativeResults=test_result.native_results,
+        ),
+        status=AssertionRunStatusClass.COMPLETE,
+    )
+
+    return MetadataChangeProposalWrapper(
+        entityUrn=assertion_urn,
+        aspect=assertionResult,
+    ).as_workunit()

From 1b06c6a30c8d6c0ee57f75f75ee6a436aa6c13a7 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Thu, 12 Oct 2023 00:31:42 +0530
Subject: [PATCH 032/259] fix(ingest/snowflake): fix sample fraction for very
 large tables (#8988)

---
 .../datahub/ingestion/source/snowflake/snowflake_profiler.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
index 24275dcdff34d..8e18d85d6f3ca 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
@@ -86,7 +86,7 @@ def get_batch_kwargs(
             # Fixed-size sampling can be slower than equivalent fraction-based sampling
             # as per https://docs.snowflake.com/en/sql-reference/constructs/sample#performance-considerations
             sample_pc = 100 * self.config.profiling.sample_size / table.rows_count
-            custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE ({sample_pc:.3f})'
+            custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE ({sample_pc:.8f})'
         return {
             **super().get_batch_kwargs(table, schema_name, db_name),
             # Lowercase/Mixedcase table names in Snowflake do not work by default.

From 245284ec6c6b754b22943ba42d7139ddd5772377 Mon Sep 17 00:00:00 2001
From: jayasimhankv <145704974+jayasimhankv@users.noreply.github.com>
Date: Wed, 11 Oct 2023 17:40:20 -0500
Subject: [PATCH 033/259] fix(): Display generic not found page for corp groups
 that do not exist (#8880)

Co-authored-by: Jay Kadambi <jayasimhan_venkatadri@optum.com>
---
 .../java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java  | 3 ++-
 datahub-graphql-core/src/main/resources/entity.graphql       | 5 +++++
 datahub-web-react/src/app/entity/group/GroupProfile.tsx      | 4 ++++
 datahub-web-react/src/graphql/group.graphql                  | 1 +
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index ebb5c7d62c7d3..b99f712034fe0 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -1292,7 +1292,8 @@ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) {
      */
     private void configureCorpGroupResolvers(final RuntimeWiring.Builder builder) {
         builder.type("CorpGroup", typeWiring -> typeWiring
-            .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)));
+            .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
+            .dataFetcher("exists", new EntityExistsResolver(entityService)));
         builder.type("CorpGroupInfo", typeWiring -> typeWiring
                 .dataFetcher("admins",
                     new LoadableTypeBatchResolver<>(corpUserType,
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index 0b15d7b875a9c..b37a8f34fa056 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -3788,6 +3788,11 @@ type CorpGroup implements Entity {
     Additional read only info about the group
     """
     info: CorpGroupInfo @deprecated
+
+    """
+    Whether or not this entity exists on DataHub
+    """
+    exists: Boolean
 }
 
 """
diff --git a/datahub-web-react/src/app/entity/group/GroupProfile.tsx b/datahub-web-react/src/app/entity/group/GroupProfile.tsx
index d5e284af931df..53d2062277dec 100644
--- a/datahub-web-react/src/app/entity/group/GroupProfile.tsx
+++ b/datahub-web-react/src/app/entity/group/GroupProfile.tsx
@@ -11,6 +11,7 @@ import { RoutedTabs } from '../../shared/RoutedTabs';
 import GroupInfoSidebar from './GroupInfoSideBar';
 import { GroupAssets } from './GroupAssets';
 import { ErrorSection } from '../../shared/error/ErrorSection';
+import NonExistentEntityPage from '../shared/entity/NonExistentEntityPage';
 
 const messageStyle = { marginTop: '10%' };
 
@@ -110,6 +111,9 @@ export default function GroupProfile() {
         urn,
     };
 
+    if (data?.corpGroup?.exists === false) {
+        return <NonExistentEntityPage />;
+    }
     return (
         <>
             {error && <ErrorSection />}
diff --git a/datahub-web-react/src/graphql/group.graphql b/datahub-web-react/src/graphql/group.graphql
index 9aa6e2b005f16..1007721e51a4e 100644
--- a/datahub-web-react/src/graphql/group.graphql
+++ b/datahub-web-react/src/graphql/group.graphql
@@ -3,6 +3,7 @@ query getGroup($urn: String!, $membersCount: Int!) {
         urn
         type
         name
+        exists
         origin {
             type
             externalType

From 245c5c00087116d236acf7a9bbddbdb4dee15949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= <sgomezvillamor@gmail.com>
Date: Thu, 12 Oct 2023 02:06:19 +0200
Subject: [PATCH 034/259] fix(ingest/looker): stop emitting tag owner (#8942)

---
 docs/how/updating-datahub.md                  |  2 +
 .../ingestion/source/looker/looker_common.py  | 13 +-----
 .../looker/golden_looker_mces.json            | 42 -------------------
 .../looker/golden_test_allow_ingest.json      | 42 -------------------
 ...olden_test_external_project_view_mces.json | 42 -------------------
 .../looker/golden_test_file_path_ingest.json  | 42 -------------------
 .../golden_test_independent_look_ingest.json  | 42 -------------------
 .../looker/golden_test_ingest.json            | 42 -------------------
 .../looker/golden_test_ingest_joins.json      | 42 -------------------
 .../golden_test_ingest_unaliased_joins.json   | 42 -------------------
 .../looker_mces_golden_deleted_stateful.json  | 42 -------------------
 .../looker/looker_mces_usage_history.json     | 42 -------------------
 .../lookml/lookml_mces_api_bigquery.json      | 42 -------------------
 .../lookml/lookml_mces_api_hive2.json         | 42 -------------------
 .../lookml/lookml_mces_badsql_parser.json     | 42 -------------------
 .../lookml/lookml_mces_offline.json           | 42 -------------------
 .../lookml_mces_offline_deny_pattern.json     | 42 -------------------
 ...lookml_mces_offline_platform_instance.json | 42 -------------------
 .../lookml_mces_with_external_urls.json       | 42 -------------------
 .../lookml/lookml_reachable_views.json        | 42 -------------------
 20 files changed, 3 insertions(+), 768 deletions(-)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 5d0ad5eaf8f7e..9cd4ad5c6f02d 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -7,6 +7,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 ### Breaking Changes
 
 - #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
+- #8942 - Removed `urn:li:corpuser:datahub` owner for the `Measure`, `Dimension` and `Temporal` tags emitted 
+  by Looker and LookML source connectors.
 - #8853 - The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7. See the docs for more details.
 - #8853 - Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include `pip install 'acryl-datahub-airflow-plugin[plugin-v2]'`. To continue using the v1 plugin, set the `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN` environment variable to `true`.
 - #8943 The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
index 89b1e45695c57..30c38720dd96c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -81,9 +81,6 @@
     EnumTypeClass,
     FineGrainedLineageClass,
     GlobalTagsClass,
-    OwnerClass,
-    OwnershipClass,
-    OwnershipTypeClass,
     SchemaMetadataClass,
     StatusClass,
     SubTypesClass,
@@ -453,17 +450,9 @@ def _get_schema(
     @staticmethod
     def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent:
         assert tag_urn in LookerUtil.tag_definitions
-        ownership = OwnershipClass(
-            owners=[
-                OwnerClass(
-                    owner="urn:li:corpuser:datahub",
-                    type=OwnershipTypeClass.DATAOWNER,
-                )
-            ]
-        )
         return MetadataChangeEvent(
             proposedSnapshot=TagSnapshotClass(
-                urn=tag_urn, aspects=[ownership, LookerUtil.tag_definitions[tag_urn]]
+                urn=tag_urn, aspects=[LookerUtil.tag_definitions[tag_urn]]
             )
         )
 
diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
index dee85b40bb7a8..1da42b94e320c 100644
--- a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
+++ b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json
@@ -533,20 +533,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -566,20 +552,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -599,20 +571,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
index 72db36e63daf7..685a606a57c33 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
@@ -327,20 +327,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -360,20 +346,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -393,20 +365,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
index e5508bdb06b9e..069788cb088ac 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
@@ -327,20 +327,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -360,20 +346,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -393,20 +365,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
index b0f66e7b245c9..f1c932ebd5a70 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
@@ -335,20 +335,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -369,20 +355,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -403,20 +375,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
index 91e13debfa028..9521c9af4bbdc 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
@@ -550,20 +550,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -583,20 +569,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -616,20 +588,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
index e93079119e4f4..dbacd52fe83de 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
@@ -327,20 +327,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -360,20 +346,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -393,20 +365,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
index a9c8efa7cdb98..aaa874d9ff348 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
@@ -351,20 +351,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -384,20 +370,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -417,20 +389,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
index edd15624a14cd..be8db0722aea3 100644
--- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
+++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
@@ -343,20 +343,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -376,20 +362,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -409,20 +381,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
index aebc89b609a08..05b74f163ad45 100644
--- a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
+++ b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
@@ -327,20 +327,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -360,20 +346,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -393,20 +365,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
index 34bded3cf691e..0778aa0050b00 100644
--- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
+++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
@@ -279,20 +279,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -312,20 +298,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -345,20 +317,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json
index 238f4c2580cdf..5a0bd4e12fd3a 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json
@@ -2121,20 +2121,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -2154,20 +2140,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -2187,20 +2159,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json
index 45d5d839e9d21..1b0ee3216383c 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json
@@ -2121,20 +2121,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -2154,20 +2140,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -2187,20 +2159,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json
index 187cedaefb6b2..b960ba581e6b5 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json
@@ -2004,20 +2004,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -2037,20 +2023,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -2070,20 +2042,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json
index c2c879e38f37b..e29292a44c949 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json
@@ -2121,20 +2121,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -2154,20 +2140,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -2187,20 +2159,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json
index c1ac54b0fb588..04ecaecbd4afb 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json
@@ -584,20 +584,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -617,20 +603,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -650,20 +622,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json
index f602ca37b3160..080931ae637bc 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json
@@ -2121,20 +2121,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -2154,20 +2140,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -2187,20 +2159,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json
index 104bd365669e3..5826c4316b539 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json
@@ -2134,20 +2134,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -2167,20 +2153,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -2200,20 +2172,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json
index 37a6c94c6952e..53d1ec0229de1 100644
--- a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json
+++ b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json
@@ -681,20 +681,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Dimension",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Dimension",
@@ -714,20 +700,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Temporal",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Temporal",
@@ -747,20 +719,6 @@
         "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
             "urn": "urn:li:tag:Measure",
             "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.Ownership": {
-                        "owners": [
-                            {
-                                "owner": "urn:li:corpuser:datahub",
-                                "type": "DATAOWNER"
-                            }
-                        ],
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        }
-                    }
-                },
                 {
                     "com.linkedin.pegasus2avro.tag.TagProperties": {
                         "name": "Measure",

From 84bba4dc446ee97f8991689fd17bfa6d14232601 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 12 Oct 2023 01:31:17 -0400
Subject: [PATCH 035/259] feat(ingest): add output schema inference for sql
 parser (#8989)

---
 .../src/datahub/utilities/sqlglot_lineage.py  | 119 ++++++++++++++++--
 .../integration/powerbi/test_m_parser.py      |  93 ++++----------
 .../test_bigquery_create_view_with_cte.json   |  32 ++++-
 ..._bigquery_from_sharded_table_wildcard.json |  16 ++-
 .../test_bigquery_nested_subqueries.json      |  16 ++-
 ..._bigquery_sharded_table_normalization.json |  16 ++-
 .../test_bigquery_star_with_replace.json      |  24 +++-
 .../test_bigquery_view_from_union.json        |  16 ++-
 .../goldens/test_create_view_as_select.json   |  16 ++-
 .../test_expand_select_star_basic.json        |  80 ++++++++++--
 .../goldens/test_insert_as_select.json        |  36 +++++-
 ...est_select_ambiguous_column_no_schema.json |  12 +-
 .../goldens/test_select_count.json            |   8 +-
 .../test_select_from_struct_subfields.json    |  16 ++-
 .../goldens/test_select_from_union.json       |  16 ++-
 .../sql_parsing/goldens/test_select_max.json  |   4 +-
 .../goldens/test_select_with_ctes.json        |   8 +-
 .../test_select_with_full_col_name.json       |  12 +-
 .../test_snowflake_case_statement.json        |  16 ++-
 .../goldens/test_snowflake_column_cast.json   |  63 ++++++++++
 .../test_snowflake_column_normalization.json  |  32 ++++-
 ...t_snowflake_ctas_column_normalization.json |  32 ++++-
 .../test_snowflake_default_normalization.json |  48 ++++++-
 .../unit/sql_parsing/test_sqlglot_lineage.py  |  21 ++++
 24 files changed, 604 insertions(+), 148 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_cast.json

diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 81c43884fdf7d..349eb40a5e865 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -5,12 +5,13 @@
 import logging
 import pathlib
 from collections import defaultdict
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import pydantic.dataclasses
 import sqlglot
 import sqlglot.errors
 import sqlglot.lineage
+import sqlglot.optimizer.annotate_types
 import sqlglot.optimizer.qualify
 import sqlglot.optimizer.qualify_columns
 from pydantic import BaseModel
@@ -23,7 +24,17 @@
 from datahub.ingestion.api.closeable import Closeable
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
-from datahub.metadata.schema_classes import OperationTypeClass, SchemaMetadataClass
+from datahub.metadata.schema_classes import (
+    ArrayTypeClass,
+    BooleanTypeClass,
+    DateTypeClass,
+    NumberTypeClass,
+    OperationTypeClass,
+    SchemaFieldDataTypeClass,
+    SchemaMetadataClass,
+    StringTypeClass,
+    TimeTypeClass,
+)
 from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedDict
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 
@@ -90,8 +101,18 @@ def get_query_type_of_sql(expression: sqlglot.exp.Expression) -> QueryType:
     return QueryType.UNKNOWN
 
 
+class _ParserBaseModel(
+    BaseModel,
+    arbitrary_types_allowed=True,
+    json_encoders={
+        SchemaFieldDataTypeClass: lambda v: v.to_obj(),
+    },
+):
+    pass
+
+
 @functools.total_ordering
-class _FrozenModel(BaseModel, frozen=True):
+class _FrozenModel(_ParserBaseModel, frozen=True):
     def __lt__(self, other: "_FrozenModel") -> bool:
         for field in self.__fields__:
             self_v = getattr(self, field)
@@ -146,29 +167,42 @@ class _ColumnRef(_FrozenModel):
     column: str
 
 
-class ColumnRef(BaseModel):
+class ColumnRef(_ParserBaseModel):
     table: Urn
     column: str
 
 
-class _DownstreamColumnRef(BaseModel):
+class _DownstreamColumnRef(_ParserBaseModel):
     table: Optional[_TableName]
     column: str
+    column_type: Optional[sqlglot.exp.DataType]
 
 
-class DownstreamColumnRef(BaseModel):
+class DownstreamColumnRef(_ParserBaseModel):
     table: Optional[Urn]
     column: str
+    column_type: Optional[SchemaFieldDataTypeClass]
+    native_column_type: Optional[str]
+
+    @pydantic.validator("column_type", pre=True)
+    def _load_column_type(
+        cls, v: Optional[Union[dict, SchemaFieldDataTypeClass]]
+    ) -> Optional[SchemaFieldDataTypeClass]:
+        if v is None:
+            return None
+        if isinstance(v, SchemaFieldDataTypeClass):
+            return v
+        return SchemaFieldDataTypeClass.from_obj(v)
 
 
-class _ColumnLineageInfo(BaseModel):
+class _ColumnLineageInfo(_ParserBaseModel):
     downstream: _DownstreamColumnRef
     upstreams: List[_ColumnRef]
 
     logic: Optional[str]
 
 
-class ColumnLineageInfo(BaseModel):
+class ColumnLineageInfo(_ParserBaseModel):
     downstream: DownstreamColumnRef
     upstreams: List[ColumnRef]
 
@@ -176,7 +210,7 @@ class ColumnLineageInfo(BaseModel):
     logic: Optional[str] = pydantic.Field(default=None, exclude=True)
 
 
-class SqlParsingDebugInfo(BaseModel, arbitrary_types_allowed=True):
+class SqlParsingDebugInfo(_ParserBaseModel):
     confidence: float = 0.0
 
     tables_discovered: int = 0
@@ -190,7 +224,7 @@ def error(self) -> Optional[Exception]:
         return self.table_error or self.column_error
 
 
-class SqlParsingResult(BaseModel):
+class SqlParsingResult(_ParserBaseModel):
     query_type: QueryType = QueryType.UNKNOWN
 
     in_tables: List[Urn]
@@ -541,6 +575,15 @@ def _schema_aware_fuzzy_column_resolve(
         ) from e
     logger.debug("Qualified sql %s", statement.sql(pretty=True, dialect=dialect))
 
+    # Try to figure out the types of the output columns.
+    try:
+        statement = sqlglot.optimizer.annotate_types.annotate_types(
+            statement, schema=sqlglot_db_schema
+        )
+    except sqlglot.errors.OptimizeError as e:
+        # This is not a fatal error, so we can continue.
+        logger.debug("sqlglot failed to annotate types: %s", e)
+
     column_lineage = []
 
     try:
@@ -553,7 +596,6 @@ def _schema_aware_fuzzy_column_resolve(
         logger.debug("output columns: %s", [col[0] for col in output_columns])
         output_col: str
         for output_col, original_col_expression in output_columns:
-            # print(f"output column: {output_col}")
             if output_col == "*":
                 # If schema information is available, the * will be expanded to the actual columns.
                 # Otherwise, we can't process it.
@@ -613,12 +655,19 @@ def _schema_aware_fuzzy_column_resolve(
 
             output_col = _schema_aware_fuzzy_column_resolve(output_table, output_col)
 
+            # Guess the output column type.
+            output_col_type = None
+            if original_col_expression.type:
+                output_col_type = original_col_expression.type
+
             if not direct_col_upstreams:
                 logger.debug(f'  "{output_col}" has no upstreams')
             column_lineage.append(
                 _ColumnLineageInfo(
                     downstream=_DownstreamColumnRef(
-                        table=output_table, column=output_col
+                        table=output_table,
+                        column=output_col,
+                        column_type=output_col_type,
                     ),
                     upstreams=sorted(direct_col_upstreams),
                     # logic=column_logic.sql(pretty=True, dialect=dialect),
@@ -673,6 +722,42 @@ def _try_extract_select(
     return statement
 
 
+def _translate_sqlglot_type(
+    sqlglot_type: sqlglot.exp.DataType.Type,
+) -> Optional[SchemaFieldDataTypeClass]:
+    TypeClass: Any
+    if sqlglot_type in sqlglot.exp.DataType.TEXT_TYPES:
+        TypeClass = StringTypeClass
+    elif sqlglot_type in sqlglot.exp.DataType.NUMERIC_TYPES or sqlglot_type in {
+        sqlglot.exp.DataType.Type.DECIMAL,
+    }:
+        TypeClass = NumberTypeClass
+    elif sqlglot_type in {
+        sqlglot.exp.DataType.Type.BOOLEAN,
+        sqlglot.exp.DataType.Type.BIT,
+    }:
+        TypeClass = BooleanTypeClass
+    elif sqlglot_type in {
+        sqlglot.exp.DataType.Type.DATE,
+    }:
+        TypeClass = DateTypeClass
+    elif sqlglot_type in sqlglot.exp.DataType.TEMPORAL_TYPES:
+        TypeClass = TimeTypeClass
+    elif sqlglot_type in {
+        sqlglot.exp.DataType.Type.ARRAY,
+    }:
+        TypeClass = ArrayTypeClass
+    elif sqlglot_type in {
+        sqlglot.exp.DataType.Type.UNKNOWN,
+    }:
+        return None
+    else:
+        logger.debug("Unknown sqlglot type: %s", sqlglot_type)
+        return None
+
+    return SchemaFieldDataTypeClass(type=TypeClass())
+
+
 def _translate_internal_column_lineage(
     table_name_urn_mapping: Dict[_TableName, str],
     raw_column_lineage: _ColumnLineageInfo,
@@ -684,6 +769,16 @@ def _translate_internal_column_lineage(
         downstream=DownstreamColumnRef(
             table=downstream_urn,
             column=raw_column_lineage.downstream.column,
+            column_type=_translate_sqlglot_type(
+                raw_column_lineage.downstream.column_type.this
+            )
+            if raw_column_lineage.downstream.column_type
+            else None,
+            native_column_type=raw_column_lineage.downstream.column_type.sql()
+            if raw_column_lineage.downstream.column_type
+            and raw_column_lineage.downstream.column_type.this
+            != sqlglot.exp.DataType.Type.UNKNOWN
+            else None,
         ),
         upstreams=[
             ColumnRef(
diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
index e3cc6c8101650..b6cb578217a2c 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py
@@ -17,7 +17,6 @@
 )
 from datahub.ingestion.source.powerbi.m_query import parser, resolver, tree_function
 from datahub.ingestion.source.powerbi.m_query.resolver import DataPlatformTable, Lineage
-from datahub.utilities.sqlglot_lineage import ColumnLineageInfo, DownstreamColumnRef
 
 pytestmark = pytest.mark.integration_batch_2
 
@@ -742,75 +741,25 @@ def test_sqlglot_parser():
         == "urn:li:dataset:(urn:li:dataPlatform:snowflake,sales_deployment.operations_analytics.transformed_prod.v_sme_unit_targets,PROD)"
     )
 
-    assert lineage[0].column_lineage == [
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="client_director"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="tier"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column='upper("manager")'),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="team_type"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="date_target"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="monthid"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="target_team"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="seller_email"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="agent_key"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="sme_quota"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="revenue_quota"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="service_quota"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="bl_target"),
-            upstreams=[],
-            logic=None,
-        ),
-        ColumnLineageInfo(
-            downstream=DownstreamColumnRef(table=None, column="software_quota"),
-            upstreams=[],
-            logic=None,
-        ),
+    # TODO: None of these columns have upstreams?
+    # That doesn't seem right - we probably need to add fake schemas for the two tables above.
+    cols = [
+        "client_director",
+        "tier",
+        'upper("manager")',
+        "team_type",
+        "date_target",
+        "monthid",
+        "target_team",
+        "seller_email",
+        "agent_key",
+        "sme_quota",
+        "revenue_quota",
+        "service_quota",
+        "bl_target",
+        "software_quota",
     ]
+    for i, column in enumerate(cols):
+        assert lineage[0].column_lineage[i].downstream.table is None
+        assert lineage[0].column_lineage[i].downstream.column == column
+        assert lineage[0].column_lineage[i].upstreams == []
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
index e50d944ce72e3..f0175b4dc8892 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
@@ -12,7 +12,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)",
-                "column": "col5"
+                "column": "col5",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -24,7 +30,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)",
-                "column": "col1"
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -36,7 +48,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)",
-                "column": "col2"
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -48,7 +66,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)",
-                "column": "col3"
+                "column": "col3",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
index 78591286feb50..b7df5444987f2 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "col1"
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -20,7 +26,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "col2"
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
index 0e93d31fbb6a6..67e306bebf545 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "col1"
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -20,7 +26,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "col2"
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
index 78591286feb50..b7df5444987f2 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "col1"
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -20,7 +26,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "col2"
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
index 17a801a63e3ff..b393b2445d6c4 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
@@ -10,7 +10,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)",
-                "column": "col1"
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -22,7 +28,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)",
-                "column": "col2"
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -34,7 +46,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)",
-                "column": "something"
+                "column": "something",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
index fd8a586ac74ac..53fb94300e804 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
@@ -11,7 +11,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_view,PROD)",
-                "column": "col1"
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -27,7 +33,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_view,PROD)",
-                "column": "col2"
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
index 1ca56840531e4..ff452467aa5bd 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
@@ -10,7 +10,9 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)",
-                "column": "Department"
+                "column": "Department",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
@@ -22,14 +24,22 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)",
-                "column": "Employees"
+                "column": "Employees",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "BIGINT"
             },
             "upstreams": []
         },
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)",
-                "column": "Salary"
+                "column": "Salary",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json
index e241bdd08e243..eecb2265eaec5 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_agg"
+                "column": "total_agg",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DOUBLE"
             },
             "upstreams": [
                 {
@@ -20,7 +26,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "orderkey"
+                "column": "orderkey",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
@@ -32,7 +44,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "custkey"
+                "column": "custkey",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
@@ -44,7 +62,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "orderstatus"
+                "column": "orderstatus",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -56,7 +80,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "totalprice"
+                "column": "totalprice",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "FLOAT"
             },
             "upstreams": [
                 {
@@ -68,7 +98,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "orderdate"
+                "column": "orderdate",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                    }
+                },
+                "native_column_type": "DATE"
             },
             "upstreams": [
                 {
@@ -80,7 +116,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "orderpriority"
+                "column": "orderpriority",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -92,7 +134,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "clerk"
+                "column": "clerk",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
@@ -104,7 +152,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "shippriority"
+                "column": "shippriority",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
@@ -116,7 +170,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "comment"
+                "column": "comment",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json
index d7264fd2db6b2..326db47e7ab33 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json
@@ -18,21 +18,27 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)",
-                "column": "i_item_desc"
+                "column": "i_item_desc",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": []
         },
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)",
-                "column": "w_warehouse_name"
+                "column": "w_warehouse_name",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": []
         },
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)",
-                "column": "d_week_seq"
+                "column": "d_week_seq",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
@@ -44,7 +50,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)",
-                "column": "no_promo"
+                "column": "no_promo",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "BIGINT"
             },
             "upstreams": [
                 {
@@ -56,7 +68,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)",
-                "column": "promo"
+                "column": "promo",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "BIGINT"
             },
             "upstreams": [
                 {
@@ -68,7 +86,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)",
-                "column": "total_cnt"
+                "column": "total_cnt",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "BIGINT"
             },
             "upstreams": []
         }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_ambiguous_column_no_schema.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_ambiguous_column_no_schema.json
index 10f5ee20b0c1f..b5fd5eebeb1b1 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_ambiguous_column_no_schema.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_ambiguous_column_no_schema.json
@@ -9,21 +9,27 @@
         {
             "downstream": {
                 "table": null,
-                "column": "a"
+                "column": "a",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": []
         },
         {
             "downstream": {
                 "table": null,
-                "column": "b"
+                "column": "b",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": []
         },
         {
             "downstream": {
                 "table": null,
-                "column": "c"
+                "column": "c",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": []
         }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json
index 9f6eeae46c294..a67c944822138 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "COUNT(`fact_complaint_snapshot`.`etl_data_dt_id`)"
+                "column": "COUNT(`fact_complaint_snapshot`.`etl_data_dt_id`)",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "BIGINT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
index 109de96180422..5ad847e252497 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "post_id"
+                "column": "post_id",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
@@ -20,7 +26,9 @@
         {
             "downstream": {
                 "table": null,
-                "column": "id"
+                "column": "id",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
@@ -32,7 +40,9 @@
         {
             "downstream": {
                 "table": null,
-                "column": "min_metric"
+                "column": "min_metric",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json
index 2340b2e95b0d0..902aa010c8afc 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json
@@ -9,14 +9,26 @@
         {
             "downstream": {
                 "table": null,
-                "column": "label"
+                "column": "label",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
             },
             "upstreams": []
         },
         {
             "downstream": {
                 "table": null,
-                "column": "total_agg"
+                "column": "total_agg",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DOUBLE"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json
index 326c07d332c26..6ea88f45847ce 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json
@@ -8,7 +8,9 @@
         {
             "downstream": {
                 "table": null,
-                "column": "max_col"
+                "column": "max_col",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json
index 3e02314d6e8c3..67e9fd2d21a0e 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json
@@ -9,7 +9,9 @@
         {
             "downstream": {
                 "table": null,
-                "column": "COL1"
+                "column": "COL1",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
@@ -21,7 +23,9 @@
         {
             "downstream": {
                 "table": null,
-                "column": "COL3"
+                "column": "COL3",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
index c12ad23b2f03b..6ee3d2e61c39b 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "post_id"
+                "column": "post_id",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
@@ -20,7 +26,9 @@
         {
             "downstream": {
                 "table": null,
-                "column": "id"
+                "column": "id",
+                "column_type": null,
+                "native_column_type": null
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json
index 64cd80e9a2d69..a876824127ec1 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_price_category"
+                "column": "total_price_category",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
             },
             "upstreams": [
                 {
@@ -20,7 +26,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_price_success"
+                "column": "total_price_success",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "FLOAT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_cast.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_cast.json
new file mode 100644
index 0000000000000..7545e2b3269dc
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_cast.json
@@ -0,0 +1,63 @@
+{
+    "query_type": "SELECT",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "orderkey",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL(20, 0)"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                    "column": "o_orderkey"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "total_cast_int",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                    "column": "o_totalprice"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "total_cast_float",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL(16, 4)"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                    "column": "o_totalprice"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json
index 7b22a46757e39..84e6b053000f1 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json
@@ -8,7 +8,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_agg"
+                "column": "total_agg",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DOUBLE"
             },
             "upstreams": [
                 {
@@ -20,7 +26,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_avg"
+                "column": "total_avg",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DOUBLE"
             },
             "upstreams": [
                 {
@@ -32,7 +44,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_min"
+                "column": "total_min",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "FLOAT"
             },
             "upstreams": [
                 {
@@ -44,7 +62,13 @@
         {
             "downstream": {
                 "table": null,
-                "column": "total_max"
+                "column": "total_max",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "FLOAT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_ctas_column_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_ctas_column_normalization.json
index c912d99a3a8a3..39c94cf83c561 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_ctas_column_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_ctas_column_normalization.json
@@ -10,7 +10,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders_normalized,PROD)",
-                "column": "Total_Agg"
+                "column": "Total_Agg",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DOUBLE"
             },
             "upstreams": [
                 {
@@ -22,7 +28,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders_normalized,PROD)",
-                "column": "total_avg"
+                "column": "total_avg",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DOUBLE"
             },
             "upstreams": [
                 {
@@ -34,7 +46,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders_normalized,PROD)",
-                "column": "TOTAL_MIN"
+                "column": "TOTAL_MIN",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "FLOAT"
             },
             "upstreams": [
                 {
@@ -46,7 +64,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders_normalized,PROD)",
-                "column": "total_max"
+                "column": "total_max",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "FLOAT"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json
index 2af308ec60623..dbf5b1b9a4453 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json
@@ -11,7 +11,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)",
-                "column": "user_fk"
+                "column": "user_fk",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL(38, 0)"
             },
             "upstreams": [
                 {
@@ -23,7 +29,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)",
-                "column": "email"
+                "column": "email",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR(16777216)"
             },
             "upstreams": [
                 {
@@ -35,7 +47,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)",
-                "column": "last_purchase_date"
+                "column": "last_purchase_date",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                    }
+                },
+                "native_column_type": "DATE"
             },
             "upstreams": [
                 {
@@ -47,7 +65,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)",
-                "column": "lifetime_purchase_amount"
+                "column": "lifetime_purchase_amount",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
@@ -59,7 +83,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)",
-                "column": "lifetime_purchase_count"
+                "column": "lifetime_purchase_count",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "BIGINT"
             },
             "upstreams": [
                 {
@@ -71,7 +101,13 @@
         {
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)",
-                "column": "average_purchase_amount"
+                "column": "average_purchase_amount",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index 2a965a9bb1e61..bb6e5f1581754 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -608,4 +608,25 @@ def test_snowflake_default_normalization():
     )
 
 
+def test_snowflake_column_cast():
+    assert_sql_result(
+        """
+SELECT
+    o.o_orderkey::NUMBER(20,0) as orderkey,
+    CAST(o.o_totalprice AS INT) as total_cast_int,
+    CAST(o.o_totalprice AS NUMBER(16,4)) as total_cast_float
+FROM snowflake_sample_data.tpch_sf1.orders o
+LIMIT 10
+""",
+        dialect="snowflake",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": {
+                "orderkey": "NUMBER(38,0)",
+                "totalprice": "NUMBER(12,2)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_column_cast.json",
+    )
+
+
 # TODO: Add a test for setting platform_instance or env

From dd418de76d96fb41c9064261cdba37bc2af85309 Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Thu, 12 Oct 2023 13:10:59 +0200
Subject: [PATCH 036/259] fix(ingest/bigquery): Fix shard regexp to match
 without underscore as well (#8934)

---
 .../ingestion/source/bigquery_v2/bigquery.py  |  1 +
 .../source/bigquery_v2/bigquery_audit.py      | 27 ++++++++++++++-----
 .../ingestion/source/bigquery_v2/queries.py   |  8 +++---
 .../ingestion/source_config/bigquery.py       |  8 +++++-
 .../tests/unit/test_bigquery_source.py        | 10 ++++---
 .../unit/test_bigqueryv2_usage_source.py      |  4 +--
 6 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index b4a04d96b532b..e577c2bac8bbd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -1057,6 +1057,7 @@ def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]:
                         ):
                             field.description = col.comment
                             schema_fields[idx] = field
+                            break
             else:
                 tags = []
                 if col.is_partition_column:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
index b0ac77201b415..88060a9cdc91d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
@@ -20,7 +20,13 @@
 
 logger: logging.Logger = logging.getLogger(__name__)
 
-_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX = "((.+)[_$])?(\\d{8})$"
+# Regexp for sharded tables.
+# A sharded table is a table that has a suffix of the form _yyyymmdd or yyyymmdd, where yyyymmdd is a date.
+# The regexp checks for valid dates in the suffix (e.g. 20200101, 20200229, 20201231) and if the date is not valid
+# then it is not a sharded table.
+_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX = (
+    "((.+\\D)[_$]?)?(\\d\\d\\d\\d(?:0[1-9]|1[0-2])(?:0[1-9]|[12][0-9]|3[01]))$"
+)
 
 
 @dataclass(frozen=True, order=True)
@@ -40,7 +46,7 @@ class BigqueryTableIdentifier:
     _BQ_SHARDED_TABLE_SUFFIX: str = "_yyyymmdd"
 
     @staticmethod
-    def get_table_and_shard(table_name: str) -> Tuple[str, Optional[str]]:
+    def get_table_and_shard(table_name: str) -> Tuple[Optional[str], Optional[str]]:
         """
         Args:
             table_name:
@@ -53,16 +59,25 @@ def get_table_and_shard(table_name: str) -> Tuple[str, Optional[str]]:
                 In case of non-sharded tables, returns (<table-id>, None)
                 In case of sharded tables, returns (<table-prefix>, shard)
         """
+        new_table_name = table_name
         match = re.match(
             BigqueryTableIdentifier._BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX,
             table_name,
             re.IGNORECASE,
         )
         if match:
-            table_name = match.group(2)
-            shard = match.group(3)
-            return table_name, shard
-        return table_name, None
+            shard: str = match[3]
+            if shard:
+                if table_name.endswith(shard):
+                    new_table_name = table_name[: -len(shard)]
+
+            new_table_name = (
+                new_table_name.rstrip("_") if new_table_name else new_table_name
+            )
+            if new_table_name.endswith("."):
+                new_table_name = table_name
+            return (new_table_name, shard) if new_table_name else (None, shard)
+        return new_table_name, None
 
     @classmethod
     def from_string_name(cls, table: str) -> "BigqueryTableIdentifier":
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py
index a87cb8c1cbfa5..67fcc33cdf218 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py
@@ -51,8 +51,8 @@ class BigqueryQuery:
   p.max_partition_id,
   p.active_billable_bytes,
   p.long_term_billable_bytes,
-  REGEXP_EXTRACT(t.table_name, r".*_(\\d+)$") as table_suffix,
-  REGEXP_REPLACE(t.table_name, r"_(\\d+)$", "") as table_base
+  REGEXP_EXTRACT(t.table_name, r"(?:(?:.+\\D)[_$]?)(\\d\\d\\d\\d(?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01]))$") as table_suffix,
+  REGEXP_REPLACE(t.table_name, r"(?:[_$]?)(\\d\\d\\d\\d(?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01]))$", "") as table_base
 
 FROM
   `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLES t
@@ -92,8 +92,8 @@ class BigqueryQuery:
   tos.OPTION_VALUE as comment,
   t.is_insertable_into,
   t.ddl,
-  REGEXP_EXTRACT(t.table_name, r".*_(\\d+)$") as table_suffix,
-  REGEXP_REPLACE(t.table_name, r"_(\\d+)$", "") as table_base
+  REGEXP_EXTRACT(t.table_name, r"(?:(?:.+\\D)[_$]?)(\\d\\d\\d\\d(?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01]))$") as table_suffix,
+  REGEXP_REPLACE(t.table_name, r"(?:[_$]?)(\\d\\d\\d\\d(?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01]))$", "") as table_base
 
 FROM
   `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLES t
diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py
index 8ca1296d819c1..0a73bb5203e72 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py
@@ -4,7 +4,13 @@
 
 from datahub.configuration.common import ConfigModel, ConfigurationError
 
-_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX: str = "((.+)[_$])?(\\d{8})$"
+# Regexp for sharded tables.
+# A sharded table is a table that has a suffix of the form _yyyymmdd or yyyymmdd, where yyyymmdd is a date.
+# The regexp checks for valid dates in the suffix (e.g. 20200101, 20200229, 20201231) and if the date is not valid
+# then it is not a sharded table.
+_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX: str = (
+    "((.+\\D)[_$]?)?(\\d\\d\\d\\d(?:0[1-9]|1[0-2])(?:0[1-9]|[12][0-9]|3[01]))$"
+)
 
 
 class BigQueryBaseConfig(ConfigModel):
diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py
index e9e91361f49f4..5a11a933c8595 100644
--- a/metadata-ingestion/tests/unit/test_bigquery_source.py
+++ b/metadata-ingestion/tests/unit/test_bigquery_source.py
@@ -765,11 +765,14 @@ def test_gen_view_dataset_workunits(
         ("project.dataset.table_20231215", "project.dataset.table", "20231215"),
         ("project.dataset.table_2023", "project.dataset.table_2023", None),
         # incorrectly handled special case where dataset itself is a sharded table if full name is specified
-        ("project.dataset.20231215", "project.dataset.20231215", None),
+        ("project.dataset.20231215", "project.dataset.20231215", "20231215"),
+        ("project1.dataset2.20231215", "project1.dataset2.20231215", "20231215"),
         # Cases with Just the table name as input
         ("table", "table", None),
-        ("table20231215", "table20231215", None),
+        ("table20231215", "table", "20231215"),
         ("table_20231215", "table", "20231215"),
+        ("table2_20231215", "table2", "20231215"),
+        ("table220231215", "table220231215", None),
         ("table_1624046611000_name", "table_1624046611000_name", None),
         ("table_1624046611000", "table_1624046611000", None),
         # Special case where dataset itself is a sharded table
@@ -801,7 +804,6 @@ def test_get_table_and_shard_default(
         ("project.dataset.2023", "project.dataset.2023", None),
         # Cases with Just the table name as input
         ("table", "table", None),
-        ("table20231215", "table20231215", None),
         ("table_20231215", "table", "20231215"),
         ("table_2023", "table", "2023"),
         ("table_1624046611000_name", "table_1624046611000_name", None),
@@ -842,7 +844,7 @@ def test_get_table_and_shard_custom_shard_pattern(
             "project.dataset.table_1624046611000_name",
         ),
         ("project.dataset.table_1624046611000", "project.dataset.table_1624046611000"),
-        ("project.dataset.table20231215", "project.dataset.table20231215"),
+        ("project.dataset.table20231215", "project.dataset.table"),
         ("project.dataset.table_*", "project.dataset.table"),
         ("project.dataset.table_2023*", "project.dataset.table"),
         ("project.dataset.table_202301*", "project.dataset.table"),
diff --git a/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py b/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py
index 4cf42da4395f9..44fd840f28d59 100644
--- a/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py
+++ b/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py
@@ -144,10 +144,10 @@ def test_bigquery_table_sanitasitation():
     assert new_table_ref.dataset == "dataset-4567"
 
     table_ref = BigQueryTableRef(
-        BigqueryTableIdentifier("project-1234", "dataset-4567", "foo_20222110")
+        BigqueryTableIdentifier("project-1234", "dataset-4567", "foo_20221210")
     )
     new_table_identifier = table_ref.table_identifier
-    assert new_table_identifier.table == "foo_20222110"
+    assert new_table_identifier.table == "foo_20221210"
     assert new_table_identifier.is_sharded_table()
     assert new_table_identifier.get_table_display_name() == "foo"
     assert new_table_identifier.project_id == "project-1234"

From c381806110ae995dd2164305394ee4e1d131e033 Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Thu, 12 Oct 2023 13:56:30 +0200
Subject: [PATCH 037/259] feat(ingestion): Adding config option to auto
 lowercase dataset urns (#8928)

---
 .../datahub/configuration/source_common.py    |  7 ++
 .../src/datahub/ingestion/api/source.py       | 24 +++++++
 .../datahub/ingestion/api/source_helpers.py   | 20 +++++-
 .../ingestion/source/bigquery_v2/bigquery.py  |  3 -
 .../source/bigquery_v2/bigquery_config.py     |  5 --
 .../src/datahub/ingestion/source/kafka.py     | 11 ++-
 .../ingestion/source/sql/sql_config.py        | 11 ++-
 .../datahub/ingestion/source/unity/config.py  |  6 +-
 .../src/datahub/utilities/urns/urn_iter.py    | 33 +++++++--
 .../api/source_helpers/test_source_helpers.py | 70 +++++++++++++++++++
 10 files changed, 170 insertions(+), 20 deletions(-)

diff --git a/metadata-ingestion/src/datahub/configuration/source_common.py b/metadata-ingestion/src/datahub/configuration/source_common.py
index a9f891ddb7b1e..80b6ceb576c1c 100644
--- a/metadata-ingestion/src/datahub/configuration/source_common.py
+++ b/metadata-ingestion/src/datahub/configuration/source_common.py
@@ -54,6 +54,13 @@ class DatasetSourceConfigMixin(PlatformInstanceConfigMixin, EnvConfigMixin):
     """
 
 
+class LowerCaseDatasetUrnConfigMixin(ConfigModel):
+    convert_urns_to_lowercase: bool = Field(
+        default=False,
+        description="Whether to convert dataset urns to lowercase.",
+    )
+
+
 class DatasetLineageProviderConfigBase(EnvConfigMixin):
     """
     Any non-Dataset source that produces lineage to Datasets should inherit this class.
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py
index 0bcc220cad49b..b86844b1c4c83 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source.py
@@ -29,6 +29,7 @@
 from datahub.ingestion.api.report import Report
 from datahub.ingestion.api.source_helpers import (
     auto_browse_path_v2,
+    auto_lowercase_urns,
     auto_materialize_referenced_tags,
     auto_status_aspect,
     auto_workunit_reporter,
@@ -192,7 +193,30 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
                 self.ctx.pipeline_config.flags.generate_browse_path_v2_dry_run
             )
 
+        auto_lowercase_dataset_urns: Optional[MetadataWorkUnitProcessor] = None
+        if (
+            self.ctx.pipeline_config
+            and self.ctx.pipeline_config.source
+            and self.ctx.pipeline_config.source.config
+            and (
+                (
+                    hasattr(
+                        self.ctx.pipeline_config.source.config,
+                        "convert_urns_to_lowercase",
+                    )
+                    and self.ctx.pipeline_config.source.config.convert_urns_to_lowercase
+                )
+                or (
+                    hasattr(self.ctx.pipeline_config.source.config, "get")
+                    and self.ctx.pipeline_config.source.config.get(
+                        "convert_urns_to_lowercase"
+                    )
+                )
+            )
+        ):
+            auto_lowercase_dataset_urns = auto_lowercase_urns
         return [
+            auto_lowercase_dataset_urns,
             auto_status_aspect,
             auto_materialize_referenced_tags,
             browse_path_processor,
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
index 7fc15cf829678..2ce9e07bc57bc 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
@@ -35,7 +35,7 @@
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.tag_urn import TagUrn
 from datahub.utilities.urns.urn import guess_entity_type
-from datahub.utilities.urns.urn_iter import list_urns
+from datahub.utilities.urns.urn_iter import list_urns, lowercase_dataset_urns
 
 if TYPE_CHECKING:
     from datahub.ingestion.api.source import SourceReport
@@ -70,7 +70,6 @@ def auto_status_aspect(
     for wu in stream:
         urn = wu.get_urn()
         all_urns.add(urn)
-
         if not wu.is_primary_source:
             # If this is a non-primary source, we pretend like we've seen the status
             # aspect so that we don't try to emit a removal for it.
@@ -173,6 +172,23 @@ def auto_materialize_referenced_tags(
         ).as_workunit()
 
 
+def auto_lowercase_urns(
+    stream: Iterable[MetadataWorkUnit],
+) -> Iterable[MetadataWorkUnit]:
+    """Lowercase all dataset urns"""
+
+    for wu in stream:
+        try:
+            old_urn = wu.get_urn()
+            lowercase_dataset_urns(wu.metadata)
+            wu.id = wu.id.replace(old_urn, wu.get_urn())
+
+            yield wu
+        except Exception as e:
+            logger.warning(f"Failed to lowercase urns for {wu}: {e}", exc_info=True)
+            yield wu
+
+
 def auto_browse_path_v2(
     stream: Iterable[MetadataWorkUnit],
     *,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index e577c2bac8bbd..552612f877b9a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -16,7 +16,6 @@
     make_dataplatform_instance_urn,
     make_dataset_urn,
     make_tag_urn,
-    set_dataset_urn_to_lower,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import BigQueryDatasetKey, ContainerKey, ProjectIdKey
@@ -218,8 +217,6 @@ def __init__(self, ctx: PipelineContext, config: BigQueryV2Config):
         if self.config.enable_legacy_sharded_table_support:
             BigqueryTableIdentifier._BQ_SHARDED_TABLE_SUFFIX = ""
 
-        set_dataset_urn_to_lower(self.config.convert_urns_to_lowercase)
-
         self.bigquery_data_dictionary = BigQuerySchemaApi(
             self.report.schema_api_perf, self.config.get_bigquery_client()
         )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index 483355a85ac05..944814b6936a4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -206,11 +206,6 @@ def validate_column_lineage(cls, v: bool, values: Dict[str, Any]) -> bool:
         description="This flag enables the data lineage extraction from Data Lineage API exposed by Google Data Catalog. NOTE: This extractor can't build views lineage. It's recommended to enable the view's DDL parsing. Read the docs to have more information about: https://cloud.google.com/data-catalog/docs/concepts/about-data-lineage",
     )
 
-    convert_urns_to_lowercase: bool = Field(
-        default=False,
-        description="Convert urns to lowercase.",
-    )
-
     enable_legacy_sharded_table_support: bool = Field(
         default=True,
         description="Use the legacy sharded table urn suffix added.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
index 566304e1999b7..d5039360da567 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
@@ -18,7 +18,10 @@
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.kafka import KafkaConsumerConnectionConfig
-from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.source_common import (
+    DatasetSourceConfigMixin,
+    LowerCaseDatasetUrnConfigMixin,
+)
 from datahub.emitter import mce_builder
 from datahub.emitter.mce_builder import (
     make_data_platform_urn,
@@ -76,7 +79,11 @@ class KafkaTopicConfigKeys(str, Enum):
     UNCLEAN_LEADER_ELECTION_CONFIG = "unclean.leader.election.enable"
 
 
-class KafkaSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
+class KafkaSourceConfig(
+    StatefulIngestionConfigBase,
+    DatasetSourceConfigMixin,
+    LowerCaseDatasetUrnConfigMixin,
+):
     connection: KafkaConsumerConnectionConfig = KafkaConsumerConnectionConfig()
 
     topic_patterns: AllowDenyPattern = AllowDenyPattern(allow=[".*"], deny=["^_.*"])
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 677d32c8bac08..08cc74aec3977 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -7,7 +7,10 @@
 from pydantic import Field
 
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.source_common import (
+    DatasetSourceConfigMixin,
+    LowerCaseDatasetUrnConfigMixin,
+)
 from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
@@ -21,7 +24,11 @@
 logger: logging.Logger = logging.getLogger(__name__)
 
 
-class SQLCommonConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
+class SQLCommonConfig(
+    StatefulIngestionConfigBase,
+    DatasetSourceConfigMixin,
+    LowerCaseDatasetUrnConfigMixin,
+):
     options: dict = pydantic.Field(
         default_factory=dict,
         description="Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index 51390873712d3..a57ee39848855 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -7,7 +7,10 @@
 from pydantic import Field
 
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
-from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.source_common import (
+    DatasetSourceConfigMixin,
+    LowerCaseDatasetUrnConfigMixin,
+)
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
@@ -91,6 +94,7 @@ class UnityCatalogSourceConfig(
     BaseUsageConfig,
     DatasetSourceConfigMixin,
     StatefulProfilingConfigMixin,
+    LowerCaseDatasetUrnConfigMixin,
 ):
     token: str = pydantic.Field(description="Databricks personal access token")
     workspace_url: str = pydantic.Field(
diff --git a/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py b/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
index 261f95331af61..e13d439161064 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
@@ -3,7 +3,11 @@
 from avro.schema import Field, RecordSchema
 
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.metadata.schema_classes import DictWrapper
+from datahub.metadata.schema_classes import (
+    DictWrapper,
+    MetadataChangeEventClass,
+    MetadataChangeProposalClass,
+)
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.urn import Urn, guess_entity_type
 
@@ -32,7 +36,7 @@ def list_urns_with_path(
 
     if isinstance(model, MetadataChangeProposalWrapper):
         if model.entityUrn:
-            urns.append((model.entityUrn, ["urn"]))
+            urns.append((model.entityUrn, ["entityUrn"]))
         if model.entityKeyAspect:
             urns.extend(
                 _add_prefix_to_paths(
@@ -83,7 +87,15 @@ def list_urns(model: Union[DictWrapper, MetadataChangeProposalWrapper]) -> List[
     return [urn for urn, _ in list_urns_with_path(model)]
 
 
-def transform_urns(model: DictWrapper, func: Callable[[str], str]) -> None:
+def transform_urns(
+    model: Union[
+        DictWrapper,
+        MetadataChangeEventClass,
+        MetadataChangeProposalClass,
+        MetadataChangeProposalWrapper,
+    ],
+    func: Callable[[str], str],
+) -> None:
     """
     Rewrites all URNs in the given object according to the given function.
     """
@@ -95,7 +107,9 @@ def transform_urns(model: DictWrapper, func: Callable[[str], str]) -> None:
 
 
 def _modify_at_path(
-    model: Union[DictWrapper, list], path: _Path, new_value: str
+    model: Union[DictWrapper, MetadataChangeProposalWrapper, list],
+    path: _Path,
+    new_value: str,
 ) -> None:
     assert len(path) > 0
 
@@ -103,6 +117,8 @@ def _modify_at_path(
         if isinstance(path[0], int):
             assert isinstance(model, list)
             model[path[0]] = new_value
+        elif isinstance(model, MetadataChangeProposalWrapper):
+            setattr(model, path[0], new_value)
         else:
             assert isinstance(model, DictWrapper)
             model._inner_dict[path[0]] = new_value
@@ -120,7 +136,14 @@ def _lowercase_dataset_urn(dataset_urn: str) -> str:
     return str(cur_urn)
 
 
-def lowercase_dataset_urns(model: DictWrapper) -> None:
+def lowercase_dataset_urns(
+    model: Union[
+        DictWrapper,
+        MetadataChangeEventClass,
+        MetadataChangeProposalClass,
+        MetadataChangeProposalWrapper,
+    ]
+) -> None:
     def modify_urn(urn: str) -> str:
         if guess_entity_type(urn) == "dataset":
             return _lowercase_dataset_urn(urn)
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py b/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
index b6ec6ebce240c..b667af8bb41e9 100644
--- a/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_source_helpers.py
@@ -16,6 +16,7 @@
 from datahub.ingestion.api.source_helpers import (
     auto_browse_path_v2,
     auto_empty_dataset_usage_statistics,
+    auto_lowercase_urns,
     auto_status_aspect,
     auto_workunit,
 )
@@ -275,6 +276,75 @@ def test_auto_browse_path_v2_legacy_browse_path(telemetry_ping_mock):
     assert paths["platform,dataset-2,PROD)"] == _make_browse_path_entries(["something"])
 
 
+def test_auto_lowercase_aspects():
+    mcws = auto_workunit(
+        [
+            MetadataChangeProposalWrapper(
+                entityUrn=make_dataset_urn(
+                    "bigquery", "myProject.mySchema.myTable", "PROD"
+                ),
+                aspect=models.DatasetKeyClass(
+                    "urn:li:dataPlatform:bigquery", "myProject.mySchema.myTable", "PROD"
+                ),
+            ),
+            MetadataChangeProposalWrapper(
+                entityUrn="urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                aspect=models.ContainerPropertiesClass(
+                    name="test",
+                ),
+            ),
+            models.MetadataChangeEventClass(
+                proposedSnapshot=models.DatasetSnapshotClass(
+                    urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-Public-Data.Covid19_Aha.staffing,PROD)",
+                    aspects=[
+                        models.DatasetPropertiesClass(
+                            customProperties={
+                                "key": "value",
+                            },
+                        ),
+                    ],
+                ),
+            ),
+        ]
+    )
+
+    expected = [
+        *list(
+            auto_workunit(
+                [
+                    MetadataChangeProposalWrapper(
+                        entityUrn="urn:li:dataset:(urn:li:dataPlatform:bigquery,myproject.myschema.mytable,PROD)",
+                        aspect=models.DatasetKeyClass(
+                            "urn:li:dataPlatform:bigquery",
+                            "myProject.mySchema.myTable",
+                            "PROD",
+                        ),
+                    ),
+                    MetadataChangeProposalWrapper(
+                        entityUrn="urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a",
+                        aspect=models.ContainerPropertiesClass(
+                            name="test",
+                        ),
+                    ),
+                    models.MetadataChangeEventClass(
+                        proposedSnapshot=models.DatasetSnapshotClass(
+                            urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_aha.staffing,PROD)",
+                            aspects=[
+                                models.DatasetPropertiesClass(
+                                    customProperties={
+                                        "key": "value",
+                                    },
+                                ),
+                            ],
+                        ),
+                    ),
+                ]
+            )
+        ),
+    ]
+    assert list(auto_lowercase_urns(mcws)) == expected
+
+
 @patch("datahub.ingestion.api.source_helpers.telemetry.telemetry_instance.ping")
 def test_auto_browse_path_v2_container_over_legacy_browse_path(telemetry_ping_mock):
     structure = {"a": {"b": ["c"]}}

From 8813ae2fb15a1f80d5f0ef433fce1f84e1a240b5 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 12 Oct 2023 07:58:10 -0400
Subject: [PATCH 038/259] feat(ingest/s3): support .gzip and fix decompression
 bug (#8990)

---
 .../ingestion/source/data_lake_common/path_spec.py       | 9 ++++++++-
 .../src/datahub/ingestion/source/s3/source.py            | 8 +++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
index d1c949f48e2cd..a35fb94614f72 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
@@ -18,7 +18,14 @@
 logger: logging.Logger = logging.getLogger(__name__)
 
 SUPPORTED_FILE_TYPES: List[str] = ["csv", "tsv", "json", "parquet", "avro"]
-SUPPORTED_COMPRESSIONS: List[str] = ["gz", "bz2"]
+
+# These come from the smart_open library.
+SUPPORTED_COMPRESSIONS: List[str] = [
+    "gz",
+    "bz2",
+    # We have a monkeypatch on smart_open that aliases .gzip to .gz.
+    "gzip",
+]
 
 
 class PathSpec(ConfigModel):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
index ac4433b7eb1f0..eb49fcbb268c0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
@@ -10,6 +10,7 @@
 from pathlib import PurePath
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
+import smart_open.compression as so_compression
 from more_itertools import peekable
 from pyspark.conf import SparkConf
 from pyspark.sql import SparkSession
@@ -120,6 +121,9 @@
 }
 PAGE_SIZE = 1000
 
+# Hack to support the .gzip extension with smart_open.
+so_compression.register_compressor(".gzip", so_compression._COMPRESSOR_REGISTRY[".gz"])
+
 
 def get_column_type(
     report: SourceReport, dataset_name: str, column_type: str
@@ -407,7 +411,9 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List:
                 table_data.full_path, "rb", transport_params={"client": s3_client}
             )
         else:
-            file = open(table_data.full_path, "rb")
+            # We still use smart_open here to take advantage of the compression
+            # capabilities of smart_open.
+            file = smart_open(table_data.full_path, "rb")
 
         fields = []
 

From f6e131206394e1f56e4f966689c8abd1e8641919 Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Thu, 12 Oct 2023 18:43:14 +0100
Subject: [PATCH 039/259] feat(ingestion): Adds support for  memory profiling
 (#8856)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 docs-website/sidebars.js                      |   1 +
 .../docs/dev_guides/profiling_ingestions.md   |  55 +++++++
 metadata-ingestion/setup.py                   |   5 +
 .../src/datahub/ingestion/run/pipeline.py     | 148 ++++++++++--------
 .../datahub/ingestion/run/pipeline_config.py  |   7 +
 5 files changed, 148 insertions(+), 68 deletions(-)
 create mode 100644 metadata-ingestion/docs/dev_guides/profiling_ingestions.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index bdf3926c17e0d..21b3a1d3fe4d3 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -140,6 +140,7 @@ module.exports = {
             "metadata-ingestion/docs/dev_guides/classification",
             "metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source",
             "metadata-ingestion/docs/dev_guides/sql_profiles",
+            "metadata-ingestion/docs/dev_guides/profiling_ingestions",
           ],
         },
       ],
diff --git a/metadata-ingestion/docs/dev_guides/profiling_ingestions.md b/metadata-ingestion/docs/dev_guides/profiling_ingestions.md
new file mode 100644
index 0000000000000..d876d99b494f8
--- /dev/null
+++ b/metadata-ingestion/docs/dev_guides/profiling_ingestions.md
@@ -0,0 +1,55 @@
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Profiling ingestions
+
+<FeatureAvailability/>
+
+**🤝 Version compatibility**
+> Open Source DataHub: **0.11.1** | Acryl: **0.2.12**
+
+This page documents how to perform memory profiles of ingestion runs. 
+It is useful when trying to size the amount of resources necessary to ingest some source or when developing new features or sources.
+
+## How to use
+Install the `debug` plugin for DataHub's CLI wherever the ingestion runs:
+
+```bash
+pip install 'acryl-datahub[debug]'
+```
+
+This will install [memray](https://github.com/bloomberg/memray) in your python environment.
+
+Add a flag to your ingestion recipe to generate a memray memory dump of your ingestion:
+```yaml
+source:
+  ...
+
+sink:
+  ...
+
+flags:
+  generate_memory_profiles: "<path to folder where dumps will be written to>"
+```
+
+Once the ingestion run starts a binary file will be created and appended to during the execution of the ingestion. 
+
+These files follow the pattern `file-<ingestion-run-urn>.bin` for a unique identification.
+Once the ingestion has finished you can use `memray` to analyze the memory dump in a flamegraph view using:
+
+```$ memray flamegraph file-None-file-2023_09_18-21_38_43.bin```
+
+This will generate an interactive HTML file for analysis:
+
+<p align="center">
+    <img width="70%" src="https://github.com/datahub-project/static-assets/blob/main/imgs/metadata-ingestion/memray-example.png?raw=true"/>
+</p>
+
+
+`memray` has an extensive set of features for memory investigation. Take a look at their [documentation](https://bloomberg.github.io/memray/overview.html) to see the full feature set.
+
+
+## Questions
+
+If you've got any questions on configuring profiling, feel free to ping us on [our Slack](https://slack.datahubproject.io/)!
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index fe8e3be4632c4..61e7b684682a4 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -431,6 +431,10 @@
 deepdiff_dep = "deepdiff"
 test_api_requirements = {pytest_dep, deepdiff_dep, "PyYAML"}
 
+debug_requirements = {
+    "memray"
+}
+
 base_dev_requirements = {
     *base_requirements,
     *framework_common,
@@ -723,5 +727,6 @@
         "dev": list(dev_requirements),
         "testing-utils": list(test_api_requirements),  # To import `datahub.testing`
         "integration-tests": list(full_test_dev_requirements),
+        "debug": list(debug_requirements),
     },
 )
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
index 79d959965e0dd..07b55e0e25a89 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
@@ -353,77 +353,89 @@ def _time_to_print(self) -> bool:
         return False
 
     def run(self) -> None:
-        self.final_status = "unknown"
-        self._notify_reporters_on_ingestion_start()
-        callback = None
-        try:
-            callback = (
-                LoggingCallback()
-                if not self.config.failure_log.enabled
-                else DeadLetterQueueCallback(
-                    self.ctx, self.config.failure_log.log_config
-                )
-            )
-            for wu in itertools.islice(
-                self.source.get_workunits(),
-                self.preview_workunits if self.preview_mode else None,
-            ):
-                try:
-                    if self._time_to_print():
-                        self.pretty_print_summary(currently_running=True)
-                except Exception as e:
-                    logger.warning(f"Failed to print summary {e}")
-
-                if not self.dry_run:
-                    self.sink.handle_work_unit_start(wu)
-                try:
-                    record_envelopes = self.extractor.get_records(wu)
-                    for record_envelope in self.transform(record_envelopes):
-                        if not self.dry_run:
-                            self.sink.write_record_async(record_envelope, callback)
-
-                except RuntimeError:
-                    raise
-                except SystemExit:
-                    raise
-                except Exception as e:
-                    logger.error(
-                        "Failed to process some records. Continuing.", exc_info=e
+        with contextlib.ExitStack() as stack:
+            if self.config.flags.generate_memory_profiles:
+                import memray
+
+                stack.enter_context(
+                    memray.Tracker(
+                        f"{self.config.flags.generate_memory_profiles}/{self.config.run_id}.bin"
                     )
-                    # TODO: Transformer errors should cause the pipeline to fail.
-
-                self.extractor.close()
-                if not self.dry_run:
-                    self.sink.handle_work_unit_end(wu)
-            self.source.close()
-            # no more data is coming, we need to let the transformers produce any additional records if they are holding on to state
-            for record_envelope in self.transform(
-                [
-                    RecordEnvelope(
-                        record=EndOfStream(), metadata={"workunit_id": "end-of-stream"}
+                )
+
+            self.final_status = "unknown"
+            self._notify_reporters_on_ingestion_start()
+            callback = None
+            try:
+                callback = (
+                    LoggingCallback()
+                    if not self.config.failure_log.enabled
+                    else DeadLetterQueueCallback(
+                        self.ctx, self.config.failure_log.log_config
                     )
-                ]
-            ):
-                if not self.dry_run and not isinstance(
-                    record_envelope.record, EndOfStream
+                )
+                for wu in itertools.islice(
+                    self.source.get_workunits(),
+                    self.preview_workunits if self.preview_mode else None,
+                ):
+                    try:
+                        if self._time_to_print():
+                            self.pretty_print_summary(currently_running=True)
+                    except Exception as e:
+                        logger.warning(f"Failed to print summary {e}")
+
+                    if not self.dry_run:
+                        self.sink.handle_work_unit_start(wu)
+                    try:
+                        record_envelopes = self.extractor.get_records(wu)
+                        for record_envelope in self.transform(record_envelopes):
+                            if not self.dry_run:
+                                self.sink.write_record_async(record_envelope, callback)
+
+                    except RuntimeError:
+                        raise
+                    except SystemExit:
+                        raise
+                    except Exception as e:
+                        logger.error(
+                            "Failed to process some records. Continuing.",
+                            exc_info=e,
+                        )
+                        # TODO: Transformer errors should cause the pipeline to fail.
+
+                    self.extractor.close()
+                    if not self.dry_run:
+                        self.sink.handle_work_unit_end(wu)
+                self.source.close()
+                # no more data is coming, we need to let the transformers produce any additional records if they are holding on to state
+                for record_envelope in self.transform(
+                    [
+                        RecordEnvelope(
+                            record=EndOfStream(),
+                            metadata={"workunit_id": "end-of-stream"},
+                        )
+                    ]
                 ):
-                    # TODO: propagate EndOfStream and other control events to sinks, to allow them to flush etc.
-                    self.sink.write_record_async(record_envelope, callback)
-
-            self.sink.close()
-            self.process_commits()
-            self.final_status = "completed"
-        except (SystemExit, RuntimeError, KeyboardInterrupt) as e:
-            self.final_status = "cancelled"
-            logger.error("Caught error", exc_info=e)
-            raise
-        finally:
-            clear_global_warnings()
-
-            if callback and hasattr(callback, "close"):
-                callback.close()  # type: ignore
-
-            self._notify_reporters_on_ingestion_completion()
+                    if not self.dry_run and not isinstance(
+                        record_envelope.record, EndOfStream
+                    ):
+                        # TODO: propagate EndOfStream and other control events to sinks, to allow them to flush etc.
+                        self.sink.write_record_async(record_envelope, callback)
+
+                self.sink.close()
+                self.process_commits()
+                self.final_status = "completed"
+            except (SystemExit, RuntimeError, KeyboardInterrupt) as e:
+                self.final_status = "cancelled"
+                logger.error("Caught error", exc_info=e)
+                raise
+            finally:
+                clear_global_warnings()
+
+                if callback and hasattr(callback, "close"):
+                    callback.close()  # type: ignore
+
+                self._notify_reporters_on_ingestion_completion()
 
     def transform(self, records: Iterable[RecordEnvelope]) -> Iterable[RecordEnvelope]:
         """
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
index ff9a7a6f3d146..da3cee8ad9c1b 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
@@ -57,6 +57,13 @@ class FlagsConfig(ConfigModel):
         ),
     )
 
+    generate_memory_profiles: Optional[str] = Field(
+        default=None,
+        description=(
+            "Generate memray memory dumps for ingestion process by providing a path to write the dump file in."
+        ),
+    )
+
 
 class PipelineConfig(ConfigModel):
     # Once support for discriminated unions gets merged into Pydantic, we can

From c564abcbf049e5251f9cc25bf0e339956279649d Mon Sep 17 00:00:00 2001
From: Amanda Hernando <110099762+amanda-her@users.noreply.github.com>
Date: Thu, 12 Oct 2023 20:38:42 +0200
Subject: [PATCH 040/259] feat(auth): add group membership field resolver
 provider (#8846)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Pertíñez <khurzak92@gmail.com>
Co-authored-by: Adrián Pertíñez <adrian.pertinez@thoughtworks.com>
---
 .../authorization/AuthorizationUtils.java     |   8 +-
 .../dataset/DatasetStatsSummaryResolver.java  |   4 +-
 .../dataset/DatasetUsageStatsResolver.java    |   4 +-
 .../load/TimeSeriesAspectResolver.java        |   4 +-
 .../policy/GetGrantedPrivilegesResolver.java  |   6 +-
 .../resolvers/glossary/GlossaryUtilsTest.java |  36 +--
 .../query/CreateQueryResolverTest.java        |   6 +-
 .../query/DeleteQueryResolverTest.java        |   6 +-
 .../query/UpdateQueryResolverTest.java        |  10 +-
 .../com/datahub/authorization/AuthUtil.java   |  10 +-
 .../authorization/AuthorizationRequest.java   |   2 +-
 .../authorization/AuthorizerContext.java      |   4 +-
 .../authorization/EntityFieldType.java        |  31 ++
 .../com/datahub/authorization/EntitySpec.java |  23 ++
 .../authorization/EntitySpecResolver.java     |  11 +
 .../datahub/authorization/FieldResolver.java  |   6 +-
 .../authorization/ResolvedEntitySpec.java     |  66 ++++
 .../authorization/ResolvedResourceSpec.java   |  55 ----
 .../authorization/ResourceFieldType.java      |  27 --
 .../datahub/authorization/ResourceSpec.java   |  23 --
 .../authorization/ResourceSpecResolver.java   |  11 -
 .../auth/authorization/Authorizer.java        |   4 +-
 .../authorization/AuthorizerChain.java        |   2 +-
 .../authorization/DataHubAuthorizer.java      |  42 ++-
 ...er.java => DefaultEntitySpecResolver.java} |  33 +-
 .../datahub/authorization/FilterUtils.java    |   8 +-
 .../datahub/authorization/PolicyEngine.java   | 206 +++++-------
 ...PlatformInstanceFieldResolverProvider.java |  28 +-
 .../DomainFieldResolverProvider.java          |  20 +-
 .../EntityFieldResolverProvider.java          |  22 ++
 .../EntityTypeFieldResolverProvider.java      |  16 +-
 .../EntityUrnFieldResolverProvider.java       |  16 +-
 .../GroupMembershipFieldResolverProvider.java |  78 +++++
 .../OwnerFieldResolverProvider.java           |  20 +-
 .../ResourceFieldResolverProvider.java        |  22 --
 .../authorization/DataHubAuthorizerTest.java  |  22 +-
 .../authorization/PolicyEngineTest.java       | 304 ++++++++----------
 ...formInstanceFieldResolverProviderTest.java |  37 ++-
 ...upMembershipFieldResolverProviderTest.java | 212 ++++++++++++
 .../factory/auth/AuthorizerChainFactory.java  |  14 +-
 .../delegates/EntityApiDelegateImpl.java      |   9 +-
 .../openapi/entities/EntitiesController.java  |  10 +-
 .../RelationshipsController.java              |   6 +-
 .../openapi/timeline/TimelineController.java  |   4 +-
 .../openapi/util/MappingUtil.java             |  11 +-
 .../datahub/plugins/test/TestAuthorizer.java  |   4 +-
 .../resources/entity/AspectResource.java      |  13 +-
 .../entity/BatchIngestionRunResource.java     |   6 +-
 .../resources/entity/EntityResource.java      |  54 ++--
 .../resources/entity/EntityV2Resource.java    |   8 +-
 .../entity/EntityVersionedV2Resource.java     |   6 +-
 .../resources/lineage/Relationships.java      |   8 +-
 .../metadata/resources/operations/Utils.java  |   6 +-
 .../resources/platform/PlatformResource.java  |   4 +-
 .../resources/restli/RestliUtils.java         |   6 +-
 .../metadata/resources/usage/UsageStats.java  |   8 +-
 56 files changed, 937 insertions(+), 685 deletions(-)
 create mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
 create mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpec.java
 create mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpecResolver.java
 create mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedEntitySpec.java
 delete mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java
 delete mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java
 delete mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java
 delete mode 100644 metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpecResolver.java
 rename metadata-service/auth-impl/src/main/java/com/datahub/authorization/{DefaultResourceSpecResolver.java => DefaultEntitySpecResolver.java} (51%)
 create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
 create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
 delete mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java
 create mode 100644 metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
index 3089b8c8fc2db..03e63c7fb472f 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
@@ -4,7 +4,7 @@
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
@@ -90,7 +90,7 @@ public static boolean canManageTags(@Nonnull QueryContext context) {
   }
 
   public static boolean canDeleteEntity(@Nonnull Urn entityUrn, @Nonnull QueryContext context) {
-    return isAuthorized(context, Optional.of(new ResourceSpec(entityUrn.getEntityType(), entityUrn.toString())), PoliciesConfig.DELETE_ENTITY_PRIVILEGE);
+    return isAuthorized(context, Optional.of(new EntitySpec(entityUrn.getEntityType(), entityUrn.toString())), PoliciesConfig.DELETE_ENTITY_PRIVILEGE);
   }
 
   public static boolean canManageUserCredentials(@Nonnull QueryContext context) {
@@ -173,7 +173,7 @@ public static boolean canDeleteQuery(@Nonnull Urn entityUrn, @Nonnull List<Urn>
 
   public static boolean isAuthorized(
       @Nonnull QueryContext context,
-      @Nonnull Optional<ResourceSpec> resourceSpec,
+      @Nonnull Optional<EntitySpec> resourceSpec,
       @Nonnull PoliciesConfig.Privilege privilege) {
     final Authorizer authorizer = context.getAuthorizer();
     final String actor = context.getActorUrn();
@@ -196,7 +196,7 @@ public static boolean isAuthorized(
       @Nonnull String resource,
       @Nonnull DisjunctivePrivilegeGroup privilegeGroup
   ) {
-    final ResourceSpec resourceSpec = new ResourceSpec(resourceType, resource);
+    final EntitySpec resourceSpec = new EntitySpec(resourceType, resource);
     return AuthUtil.isAuthorized(authorizer, actor, Optional.of(resourceSpec), privilegeGroup);
   }
 
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java
index 23be49c7e7140..2873866bb34f7 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java
@@ -1,6 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.dataset;
 
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.linkedin.common.urn.Urn;
@@ -104,7 +104,7 @@ private CorpUser createPartialUser(final Urn userUrn) {
 
   private boolean isAuthorized(final Urn resourceUrn, final QueryContext context) {
     return AuthorizationUtils.isAuthorized(context,
-            Optional.of(new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString())),
+            Optional.of(new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString())),
             PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE);
   }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java
index 20361830ad5a5..e4bec8e896fdf 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java
@@ -1,6 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.dataset;
 
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -52,7 +52,7 @@ public CompletableFuture<UsageQueryResult> get(DataFetchingEnvironment environme
 
   private boolean isAuthorized(final Urn resourceUrn, final QueryContext context) {
     return AuthorizationUtils.isAuthorized(context,
-        Optional.of(new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString())),
+        Optional.of(new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString())),
         PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE);
   }
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java
index 197ca8640559d..f13ebf8373e91 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java
@@ -1,6 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.load;
 
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
 import com.linkedin.datahub.graphql.generated.Entity;
@@ -79,7 +79,7 @@ public TimeSeriesAspectResolver(
   private boolean isAuthorized(QueryContext context, String urn) {
     if (_entityName.equals(Constants.DATASET_ENTITY_NAME) && _aspectName.equals(
         Constants.DATASET_PROFILE_ASPECT_NAME)) {
-      return AuthorizationUtils.isAuthorized(context, Optional.of(new ResourceSpec(_entityName, urn)),
+      return AuthorizationUtils.isAuthorized(context, Optional.of(new EntitySpec(_entityName, urn)),
           PoliciesConfig.VIEW_DATASET_PROFILE_PRIVILEGE);
     }
     return true;
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java
index 2f20fdaf1e9b1..11f7793db82c8 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java
@@ -2,7 +2,7 @@
 
 import com.datahub.authorization.AuthorizerChain;
 import com.datahub.authorization.DataHubAuthorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.GetGrantedPrivilegesInput;
@@ -33,8 +33,8 @@ public CompletableFuture<Privileges> get(final DataFetchingEnvironment environme
     if (!isAuthorized(context, actor)) {
       throw new AuthorizationException("Unauthorized to get privileges for the given author.");
     }
-    final Optional<ResourceSpec> resourceSpec = Optional.ofNullable(input.getResourceSpec())
-        .map(spec -> new ResourceSpec(EntityTypeMapper.getName(spec.getResourceType()), spec.getResourceUrn()));
+    final Optional<EntitySpec> resourceSpec = Optional.ofNullable(input.getResourceSpec())
+        .map(spec -> new EntitySpec(EntityTypeMapper.getName(spec.getResourceType()), spec.getResourceUrn()));
 
     if (context.getAuthorizer() instanceof AuthorizerChain) {
       DataHubAuthorizer dataHubAuthorizer = ((AuthorizerChain) context.getAuthorizer()).getDefaultAuthorizer();
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java
index ccaab44f60dd4..8bfc32e1999ae 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/GlossaryUtilsTest.java
@@ -5,7 +5,7 @@
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.common.urn.GlossaryNodeUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
@@ -89,17 +89,17 @@ private void setUpTests() throws Exception {
       Mockito.any(Authentication.class)
     )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentNode3Aspects)));
 
-    final ResourceSpec resourceSpec3 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
+    final EntitySpec resourceSpec3 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
     mockAuthRequest("MANAGE_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec3);
 
-    final ResourceSpec resourceSpec2 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
+    final EntitySpec resourceSpec2 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
     mockAuthRequest("MANAGE_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec2);
 
-    final ResourceSpec resourceSpec1 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
+    final EntitySpec resourceSpec1 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
     mockAuthRequest("MANAGE_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec1);
   }
 
-  private void mockAuthRequest(String privilege, AuthorizationResult.Type allowOrDeny, ResourceSpec resourceSpec) {
+  private void mockAuthRequest(String privilege, AuthorizationResult.Type allowOrDeny, EntitySpec resourceSpec) {
     final AuthorizationRequest authorizationRequest = new AuthorizationRequest(
         userUrn,
         privilege,
@@ -150,7 +150,7 @@ public void testCanManageChildrenEntitiesAuthorized() throws Exception {
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn.toString());
+    final EntitySpec resourceSpec = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn.toString());
     mockAuthRequest("MANAGE_GLOSSARY_CHILDREN", AuthorizationResult.Type.ALLOW, resourceSpec);
 
     assertTrue(GlossaryUtils.canManageChildrenEntities(mockContext, parentNodeUrn, mockClient));
@@ -162,7 +162,7 @@ public void testCanManageChildrenEntitiesUnauthorized() throws Exception {
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn.toString());
+    final EntitySpec resourceSpec = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn.toString());
     mockAuthRequest("MANAGE_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec);
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec);
 
@@ -175,13 +175,13 @@ public void testCanManageChildrenRecursivelyEntitiesAuthorized() throws Exceptio
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec3 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
+    final EntitySpec resourceSpec3 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.ALLOW, resourceSpec3);
 
-    final ResourceSpec resourceSpec2 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
+    final EntitySpec resourceSpec2 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec2);
 
-    final ResourceSpec resourceSpec1 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
+    final EntitySpec resourceSpec1 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec1);
 
     assertTrue(GlossaryUtils.canManageChildrenEntities(mockContext, parentNodeUrn1, mockClient));
@@ -193,13 +193,13 @@ public void testCanManageChildrenRecursivelyEntitiesUnauthorized() throws Except
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec3 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
+    final EntitySpec resourceSpec3 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec3);
 
-    final ResourceSpec resourceSpec2 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
+    final EntitySpec resourceSpec2 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec2);
 
-    final ResourceSpec resourceSpec1 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
+    final EntitySpec resourceSpec1 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec1);
 
     assertFalse(GlossaryUtils.canManageChildrenEntities(mockContext, parentNodeUrn1, mockClient));
@@ -211,10 +211,10 @@ public void testCanManageChildrenRecursivelyEntitiesAuthorizedLevel2() throws Ex
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec2 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
+    final EntitySpec resourceSpec2 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.ALLOW, resourceSpec2);
 
-    final ResourceSpec resourceSpec1 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
+    final EntitySpec resourceSpec1 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn1.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec1);
 
     assertTrue(GlossaryUtils.canManageChildrenEntities(mockContext, parentNodeUrn1, mockClient));
@@ -226,10 +226,10 @@ public void testCanManageChildrenRecursivelyEntitiesUnauthorizedLevel2() throws
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec3 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
+    final EntitySpec resourceSpec3 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec3);
 
-    final ResourceSpec resourceSpec2 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
+    final EntitySpec resourceSpec2 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn2.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec2);
 
     assertFalse(GlossaryUtils.canManageChildrenEntities(mockContext, parentNodeUrn2, mockClient));
@@ -241,7 +241,7 @@ public void testCanManageChildrenRecursivelyEntitiesNoLevel2() throws Exception
     // they do NOT have the MANAGE_GLOSSARIES platform privilege
     mockAuthRequest("MANAGE_GLOSSARIES", AuthorizationResult.Type.DENY, null);
 
-    final ResourceSpec resourceSpec3 = new ResourceSpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
+    final EntitySpec resourceSpec3 = new EntitySpec(parentNodeUrn.getEntityType(), parentNodeUrn3.toString());
     mockAuthRequest("MANAGE_ALL_GLOSSARY_CHILDREN", AuthorizationResult.Type.DENY, resourceSpec3);
 
     assertFalse(GlossaryUtils.canManageChildrenEntities(mockContext, parentNodeUrn3, mockClient));
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java
index 196eb24b52bf8..9c04c67dd3a3b 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/CreateQueryResolverTest.java
@@ -5,7 +5,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -201,7 +201,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN.getEntityType(),
                 TEST_DATASET_URN.toString()))
     );
@@ -210,7 +210,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN.getEntityType(),
                 TEST_DATASET_URN.toString()))
     );
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java
index a6b4887b0e882..78c894f27cbc3 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/DeleteQueryResolverTest.java
@@ -5,7 +5,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
@@ -134,7 +134,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         DeleteQueryResolverTest.TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 DeleteQueryResolverTest.TEST_DATASET_URN.getEntityType(),
                 DeleteQueryResolverTest.TEST_DATASET_URN.toString()))
     );
@@ -143,7 +143,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN.getEntityType(),
                 TEST_DATASET_URN.toString()))
     );
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java
index 7a76b6d6be5a4..9b500b5fb3936 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/UpdateQueryResolverTest.java
@@ -5,7 +5,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -206,7 +206,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN.getEntityType(),
                 TEST_DATASET_URN.toString()))
     );
@@ -215,7 +215,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN.getEntityType(),
                 TEST_DATASET_URN.toString()))
     );
@@ -224,7 +224,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_QUERIES_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN_2.getEntityType(),
                 TEST_DATASET_URN_2.toString()))
     );
@@ -233,7 +233,7 @@ private QueryContext getMockQueryContext(boolean allowEditEntityQueries) {
         TEST_ACTOR_URN.toString(),
         PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType(),
         Optional.of(
-            new ResourceSpec(
+            new EntitySpec(
                 TEST_DATASET_URN_2.getEntityType(),
                 TEST_DATASET_URN_2.toString()))
     );
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthUtil.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthUtil.java
index dfb936c61ee0c..e159993a8a243 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthUtil.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthUtil.java
@@ -11,7 +11,7 @@ public class AuthUtil {
   public static boolean isAuthorized(
       @Nonnull Authorizer authorizer,
       @Nonnull String actor,
-      @Nonnull Optional<ResourceSpec> maybeResourceSpec,
+      @Nonnull Optional<EntitySpec> maybeResourceSpec,
       @Nonnull DisjunctivePrivilegeGroup privilegeGroup
   ) {
     for (ConjunctivePrivilegeGroup andPrivilegeGroup : privilegeGroup.getAuthorizedPrivilegeGroups()) {
@@ -27,7 +27,7 @@ public static boolean isAuthorized(
   public static boolean isAuthorizedForResources(
       @Nonnull Authorizer authorizer,
       @Nonnull String actor,
-      @Nonnull List<Optional<ResourceSpec>> resourceSpecs,
+      @Nonnull List<Optional<EntitySpec>> resourceSpecs,
       @Nonnull DisjunctivePrivilegeGroup privilegeGroup
   ) {
     for (ConjunctivePrivilegeGroup andPrivilegeGroup : privilegeGroup.getAuthorizedPrivilegeGroups()) {
@@ -44,7 +44,7 @@ private static boolean isAuthorized(
       @Nonnull Authorizer authorizer,
       @Nonnull String actor,
       @Nonnull ConjunctivePrivilegeGroup requiredPrivileges,
-      @Nonnull Optional<ResourceSpec> resourceSpec) {
+      @Nonnull Optional<EntitySpec> resourceSpec) {
     // Each privilege in a group _must_ all be true to permit the operation.
     for (final String privilege : requiredPrivileges.getRequiredPrivileges()) {
       // Create and evaluate an Authorization request.
@@ -62,11 +62,11 @@ private static boolean isAuthorizedForResources(
       @Nonnull Authorizer authorizer,
       @Nonnull String actor,
       @Nonnull ConjunctivePrivilegeGroup requiredPrivileges,
-      @Nonnull List<Optional<ResourceSpec>> resourceSpecs) {
+      @Nonnull List<Optional<EntitySpec>> resourceSpecs) {
     // Each privilege in a group _must_ all be true to permit the operation.
     for (final String privilege : requiredPrivileges.getRequiredPrivileges()) {
       // Create and evaluate an Authorization request.
-      for (Optional<ResourceSpec> resourceSpec : resourceSpecs) {
+      for (Optional<EntitySpec> resourceSpec : resourceSpecs) {
         final AuthorizationRequest request = new AuthorizationRequest(actor, privilege, resourceSpec);
         final AuthorizationResult result = authorizer.authorize(request);
         if (AuthorizationResult.Type.DENY.equals(result.getType())) {
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java
index 084a455495551..9e75de3cbf44d 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java
@@ -21,5 +21,5 @@ public class AuthorizationRequest {
    * The resource that the user is requesting for, if applicable. If the privilege is a platform privilege
    * this optional will be empty.
    */
-  Optional<ResourceSpec> resourceSpec;
+  Optional<EntitySpec> resourceSpec;
 }
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizerContext.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizerContext.java
index f9940d171d5d4..b79a4fa20c7ea 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizerContext.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizerContext.java
@@ -18,9 +18,9 @@ public class AuthorizerContext {
   private final Map<String, Object> contextMap;
 
   /**
-   * A utility for resolving a {@link ResourceSpec} to resolved resource field values.
+   * A utility for resolving an {@link EntitySpec} to resolved entity field values.
    */
-  private ResourceSpecResolver resourceSpecResolver;
+  private EntitySpecResolver entitySpecResolver;
 
   /**
    *
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
new file mode 100644
index 0000000000000..46763f29a7040
--- /dev/null
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
@@ -0,0 +1,31 @@
+package com.datahub.authorization;
+
+/**
+ * List of entity field types to fetch for a given entity
+ */
+public enum EntityFieldType {
+  /**
+   * Type of the entity (e.g. dataset, chart)
+   */
+  TYPE,
+  /**
+   * Urn of the entity
+   */
+  URN,
+  /**
+   * Owners of the entity
+   */
+  OWNER,
+  /**
+   * Domains of the entity
+   */
+  DOMAIN,
+  /**
+   * Groups of which the entity (only applies to corpUser) is a member
+   */
+  GROUP_MEMBERSHIP,
+  /**
+   * Data platform instance of resource
+   */
+  DATA_PLATFORM_INSTANCE
+}
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpec.java
new file mode 100644
index 0000000000000..656bec0f44fc2
--- /dev/null
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpec.java
@@ -0,0 +1,23 @@
+package com.datahub.authorization;
+
+import javax.annotation.Nonnull;
+import lombok.Value;
+
+
+/**
+ * Details about the entities involved in the authorization process. It models the actor and the resource being acted
+ * upon. Resource types currently supported can be found inside of {@link com.linkedin.metadata.authorization.PoliciesConfig}
+ */
+@Value
+public class EntitySpec {
+  /**
+   * The entity type. (dataset, chart, dashboard, corpGroup, etc).
+   */
+  @Nonnull
+  String type;
+  /**
+   * The entity identity. Most often, this corresponds to the raw entity urn. (urn:li:corpGroup:groupId)
+   */
+  @Nonnull
+  String entity;
+}
\ No newline at end of file
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpecResolver.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpecResolver.java
new file mode 100644
index 0000000000000..67347fbf87a87
--- /dev/null
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntitySpecResolver.java
@@ -0,0 +1,11 @@
+package com.datahub.authorization;
+
+/**
+ * An Entity Spec Resolver is responsible for resolving a {@link EntitySpec} to a {@link ResolvedEntitySpec}.
+ */
+public interface EntitySpecResolver {
+  /**
+   Resolve a {@link EntitySpec} to a resolved entity spec.
+   **/
+  ResolvedEntitySpec resolve(EntitySpec entitySpec);
+}
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/FieldResolver.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/FieldResolver.java
index 9318f5f8e7b96..955a06fd54cb9 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/FieldResolver.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/FieldResolver.java
@@ -33,9 +33,9 @@ public static FieldResolver getResolverFromValues(Set<String> values) {
   /**
    * Helper function that returns FieldResolver given a fetchFieldValue function
    */
-  public static FieldResolver getResolverFromFunction(ResourceSpec resourceSpec,
-      Function<ResourceSpec, FieldValue> fetchFieldValue) {
-    return new FieldResolver(() -> CompletableFuture.supplyAsync(() -> fetchFieldValue.apply(resourceSpec)));
+  public static FieldResolver getResolverFromFunction(EntitySpec entitySpec,
+      Function<EntitySpec, FieldValue> fetchFieldValue) {
+    return new FieldResolver(() -> CompletableFuture.supplyAsync(() -> fetchFieldValue.apply(entitySpec)));
   }
 
   public static FieldValue emptyFieldValue() {
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedEntitySpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedEntitySpec.java
new file mode 100644
index 0000000000000..7948766df5715
--- /dev/null
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedEntitySpec.java
@@ -0,0 +1,66 @@
+package com.datahub.authorization;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+
+
+/**
+ * Wrapper around authorization request with field resolvers for lazily fetching the field values for each field type
+ */
+@RequiredArgsConstructor
+@ToString
+public class ResolvedEntitySpec {
+  @Getter
+  private final EntitySpec spec;
+  private final Map<EntityFieldType, FieldResolver> fieldResolvers;
+
+  public Set<String> getFieldValues(EntityFieldType entityFieldType) {
+    if (!fieldResolvers.containsKey(entityFieldType)) {
+      return Collections.emptySet();
+    }
+    return fieldResolvers.get(entityFieldType).getFieldValuesFuture().join().getValues();
+  }
+
+  /**
+   * Fetch the owners for an entity.
+   * @return a set of owner urns, or empty set if none exist.
+   */
+  public Set<String> getOwners() {
+    if (!fieldResolvers.containsKey(EntityFieldType.OWNER)) {
+      return Collections.emptySet();
+    }
+    return fieldResolvers.get(EntityFieldType.OWNER).getFieldValuesFuture().join().getValues();
+  }
+
+  /**
+   * Fetch the platform instance for a Resolved Resource Spec
+   * @return a Platform Instance or null if one does not exist.
+   */
+  @Nullable
+  public String getDataPlatformInstance() {
+    if (!fieldResolvers.containsKey(EntityFieldType.DATA_PLATFORM_INSTANCE)) {
+      return null;
+    }
+    Set<String> dataPlatformInstance = fieldResolvers.get(EntityFieldType.DATA_PLATFORM_INSTANCE).getFieldValuesFuture().join().getValues();
+    if (dataPlatformInstance.size() > 0) {
+      return dataPlatformInstance.stream().findFirst().get();
+    }
+    return null;
+  }
+
+  /**
+   * Fetch the group membership for an entity.
+   * @return a set of groups urns, or empty set if none exist.
+   */
+  public Set<String> getGroupMembership() {
+    if (!fieldResolvers.containsKey(EntityFieldType.GROUP_MEMBERSHIP)) {
+      return Collections.emptySet();
+    }
+    return fieldResolvers.get(EntityFieldType.GROUP_MEMBERSHIP).getFieldValuesFuture().join().getValues();
+  }
+}
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java
deleted file mode 100644
index 8e429a8ca1b94..0000000000000
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package com.datahub.authorization;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import javax.annotation.Nullable;
-import lombok.Getter;
-import lombok.RequiredArgsConstructor;
-import lombok.ToString;
-
-
-/**
- * Wrapper around authorization request with field resolvers for lazily fetching the field values for each field type
- */
-@RequiredArgsConstructor
-@ToString
-public class ResolvedResourceSpec {
-  @Getter
-  private final ResourceSpec spec;
-  private final Map<ResourceFieldType, FieldResolver> fieldResolvers;
-
-  public Set<String> getFieldValues(ResourceFieldType resourceFieldType) {
-    if (!fieldResolvers.containsKey(resourceFieldType)) {
-      return Collections.emptySet();
-    }
-    return fieldResolvers.get(resourceFieldType).getFieldValuesFuture().join().getValues();
-  }
-
-  /**
-   * Fetch the owners for a resource.
-   * @return a set of owner urns, or empty set if none exist.
-   */
-  public Set<String> getOwners() {
-    if (!fieldResolvers.containsKey(ResourceFieldType.OWNER)) {
-      return Collections.emptySet();
-    }
-    return fieldResolvers.get(ResourceFieldType.OWNER).getFieldValuesFuture().join().getValues();
-  }
-
-  /**
-   * Fetch the platform instance for a Resolved Resource Spec
-   * @return a Platform Instance or null if one does not exist.
-   */
-  @Nullable
-  public String getDataPlatformInstance() {
-    if (!fieldResolvers.containsKey(ResourceFieldType.DATA_PLATFORM_INSTANCE)) {
-      return null;
-    }
-    Set<String> dataPlatformInstance = fieldResolvers.get(ResourceFieldType.DATA_PLATFORM_INSTANCE).getFieldValuesFuture().join().getValues();
-    if (dataPlatformInstance.size() > 0) {
-      return dataPlatformInstance.stream().findFirst().get();
-    }
-    return null;
-  }
-}
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java
deleted file mode 100644
index 478522dc7c331..0000000000000
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceFieldType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.datahub.authorization;
-
-/**
- * List of resource field types to fetch for a given resource
- */
-public enum ResourceFieldType {
-  /**
-   * Type of resource (e.g. dataset, chart)
-   */
-  RESOURCE_TYPE,
-  /**
-   * Urn of resource
-   */
-  RESOURCE_URN,
-  /**
-   * Owners of resource
-   */
-  OWNER,
-  /**
-   * Domains of resource
-   */
-  DOMAIN,
-  /**
-   * Data platform instance of resource
-   */
-  DATA_PLATFORM_INSTANCE
-}
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java
deleted file mode 100644
index c1bd53e31fe29..0000000000000
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.datahub.authorization;
-
-import javax.annotation.Nonnull;
-import lombok.Value;
-
-
-/**
- * Details about a specific resource being acted upon. Resource types currently supported
- * can be found inside of {@link com.linkedin.metadata.authorization.PoliciesConfig}
- */
-@Value
-public class ResourceSpec {
-  /**
-   * The resource type. Most often, this corresponds to the entity type. (dataset, chart, dashboard, corpGroup, etc).
-   */
-  @Nonnull
-  String type;
-  /**
-   * The resource identity. Most often, this corresponds to the raw entity urn. (urn:li:corpGroup:groupId)
-   */
-  @Nonnull
-  String resource;
-}
\ No newline at end of file
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpecResolver.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpecResolver.java
deleted file mode 100644
index 05c35f377b9a9..0000000000000
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResourceSpecResolver.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package com.datahub.authorization;
-
-/**
- * A Resource Spec Resolver is responsible for resolving a {@link ResourceSpec} to a {@link ResolvedResourceSpec}.
- */
-public interface ResourceSpecResolver {
-  /**
-   Resolve a {@link ResourceSpec} to a resolved resource spec.
-   **/
-  ResolvedResourceSpec resolve(ResourceSpec resourceSpec);
-}
diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/plugins/auth/authorization/Authorizer.java b/metadata-auth/auth-api/src/main/java/com/datahub/plugins/auth/authorization/Authorizer.java
index ce7a3f22b3147..c731a3ec987c1 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/plugins/auth/authorization/Authorizer.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/plugins/auth/authorization/Authorizer.java
@@ -4,7 +4,7 @@
 import com.datahub.authorization.AuthorizationResult;
 import com.datahub.authorization.AuthorizedActors;
 import com.datahub.authorization.AuthorizerContext;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.Plugin;
 import java.util.Map;
 import java.util.Optional;
@@ -32,5 +32,5 @@ public interface Authorizer extends Plugin {
    * Retrieves the current list of actors authorized to for a particular privilege against
    * an optional resource
    */
-  AuthorizedActors authorizedActors(final String privilege, final Optional<ResourceSpec> resourceSpec);
+  AuthorizedActors authorizedActors(final String privilege, final Optional<EntitySpec> resourceSpec);
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java
index d62c37160f816..f8eca541e1efb 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java
@@ -82,7 +82,7 @@ public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request
   }
 
   @Override
-  public AuthorizedActors authorizedActors(String privilege, Optional<ResourceSpec> resourceSpec) {
+  public AuthorizedActors authorizedActors(String privilege, Optional<EntitySpec> resourceSpec) {
     if (this.authorizers.isEmpty()) {
       return null;
     }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
index f653ccf72cf54..4553139e3ca54 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
@@ -8,6 +8,8 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.policy.DataHubPolicyInfo;
+
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -55,7 +57,7 @@ public enum AuthorizationMode {
   private final ScheduledExecutorService _refreshExecutorService = Executors.newScheduledThreadPool(1);
   private final PolicyRefreshRunnable _policyRefreshRunnable;
   private final PolicyEngine _policyEngine;
-  private ResourceSpecResolver _resourceSpecResolver;
+  private EntitySpecResolver _entitySpecResolver;
   private AuthorizationMode _mode;
 
   public static final String ALL = "ALL";
@@ -76,7 +78,7 @@ public DataHubAuthorizer(
   @Override
   public void init(@Nonnull Map<String, Object> authorizerConfig, @Nonnull AuthorizerContext ctx) {
     // Pass. No static config.
-    _resourceSpecResolver = Objects.requireNonNull(ctx.getResourceSpecResolver());
+    _entitySpecResolver = Objects.requireNonNull(ctx.getEntitySpecResolver());
   }
 
   public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request) {
@@ -86,7 +88,7 @@ public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request
       return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW, null);
     }
 
-    Optional<ResolvedResourceSpec> resolvedResourceSpec = request.getResourceSpec().map(_resourceSpecResolver::resolve);
+    Optional<ResolvedEntitySpec> resolvedResourceSpec = request.getResourceSpec().map(_entitySpecResolver::resolve);
 
     // 1. Fetch the policies relevant to the requested privilege.
     final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>());
@@ -102,14 +104,17 @@ public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request
     return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
   }
 
-  public List<String> getGrantedPrivileges(final String actorUrn, final Optional<ResourceSpec> resourceSpec) {
+  public List<String> getGrantedPrivileges(final String actor, final Optional<EntitySpec> resourceSpec) {
 
     // 1. Fetch all policies
     final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>());
 
-    Optional<ResolvedResourceSpec> resolvedResourceSpec = resourceSpec.map(_resourceSpecResolver::resolve);
+    Urn actorUrn = UrnUtils.getUrn(actor);
+    final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
+
+    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-    return _policyEngine.getGrantedPrivileges(policiesToEvaluate, UrnUtils.getUrn(actorUrn), resolvedResourceSpec);
+    return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
   }
 
   /**
@@ -118,11 +123,11 @@ public List<String> getGrantedPrivileges(final String actorUrn, final Optional<R
    */
   public AuthorizedActors authorizedActors(
       final String privilege,
-      final Optional<ResourceSpec> resourceSpec) {
+      final Optional<EntitySpec> resourceSpec) {
     // Step 1: Find policies granting the privilege.
     final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>());
 
-    Optional<ResolvedResourceSpec> resolvedResourceSpec = resourceSpec.map(_resourceSpecResolver::resolve);
+    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
     final List<Urn> authorizedUsers = new ArrayList<>();
     final List<Urn> authorizedGroups = new ArrayList<>();
@@ -180,19 +185,36 @@ private boolean isSystemRequest(final AuthorizationRequest request, final Authen
   /**
    * Returns true if a policy grants the requested privilege for a given actor and resource.
    */
-  private boolean isRequestGranted(final DataHubPolicyInfo policy, final AuthorizationRequest request, final Optional<ResolvedResourceSpec> resourceSpec) {
+  private boolean isRequestGranted(final DataHubPolicyInfo policy, final AuthorizationRequest request, final Optional<ResolvedEntitySpec> resourceSpec) {
     if (AuthorizationMode.ALLOW_ALL.equals(mode())) {
       return true;
     }
+
+    Optional<Urn> actorUrn = getUrnFromRequestActor(request.getActorUrn());
+    if (actorUrn.isEmpty()) {
+      return false;
+    }
+
+    final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(
+            new EntitySpec(actorUrn.get().getEntityType(), request.getActorUrn()));
     final PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy(
         policy,
-        request.getActorUrn(),
+        resolvedActorSpec,
         request.getPrivilege(),
         resourceSpec
     );
     return result.isGranted();
   }
 
+  private Optional<Urn> getUrnFromRequestActor(String actor) {
+    try {
+      return Optional.of(Urn.createFromString(actor));
+    } catch (URISyntaxException e) {
+      log.error(String.format("Failed to bind actor %s to an URN. Actors must be URNs. Denying the authorization request", actor));
+      return Optional.empty();
+    }
+  }
+
   /**
    * A {@link Runnable} used to periodically fetch a new instance of the policies Cache.
    *
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
similarity index 51%
rename from metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java
rename to metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
index 64c43dc8aa591..4ad14ed59c9c0 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultResourceSpecResolver.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
@@ -1,39 +1,40 @@
 package com.datahub.authorization;
 
-import com.datahub.authentication.Authentication;
 import com.datahub.authorization.fieldresolverprovider.DataPlatformInstanceFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.DomainFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.ResourceFieldResolverProvider;
+import com.datahub.authentication.Authentication;
+import com.datahub.authorization.fieldresolverprovider.DomainFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.GroupMembershipFieldResolverProvider;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.entity.client.EntityClient;
-
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 
-public class DefaultResourceSpecResolver implements ResourceSpecResolver {
-  private final List<ResourceFieldResolverProvider> _resourceFieldResolverProviders;
+public class DefaultEntitySpecResolver implements EntitySpecResolver {
+  private final List<EntityFieldResolverProvider> _entityFieldResolverProviders;
 
-  public DefaultResourceSpecResolver(Authentication systemAuthentication, EntityClient entityClient) {
-    _resourceFieldResolverProviders =
+  public DefaultEntitySpecResolver(Authentication systemAuthentication, EntityClient entityClient) {
+    _entityFieldResolverProviders =
         ImmutableList.of(new EntityTypeFieldResolverProvider(), new EntityUrnFieldResolverProvider(),
             new DomainFieldResolverProvider(entityClient, systemAuthentication),
             new OwnerFieldResolverProvider(entityClient, systemAuthentication),
-            new DataPlatformInstanceFieldResolverProvider(entityClient, systemAuthentication));
+            new DataPlatformInstanceFieldResolverProvider(entityClient, systemAuthentication),
+            new GroupMembershipFieldResolverProvider(entityClient, systemAuthentication));
   }
 
   @Override
-  public ResolvedResourceSpec resolve(ResourceSpec resourceSpec) {
-    return new ResolvedResourceSpec(resourceSpec, getFieldResolvers(resourceSpec));
+  public ResolvedEntitySpec resolve(EntitySpec entitySpec) {
+    return new ResolvedEntitySpec(entitySpec, getFieldResolvers(entitySpec));
   }
 
-  private Map<ResourceFieldType, FieldResolver> getFieldResolvers(ResourceSpec resourceSpec) {
-    return _resourceFieldResolverProviders.stream()
-        .collect(Collectors.toMap(ResourceFieldResolverProvider::getFieldType,
-            hydrator -> hydrator.getFieldResolver(resourceSpec)));
+  private Map<EntityFieldType, FieldResolver> getFieldResolvers(EntitySpec entitySpec) {
+    return _entityFieldResolverProviders.stream()
+        .collect(Collectors.toMap(EntityFieldResolverProvider::getFieldType,
+            hydrator -> hydrator.getFieldResolver(entitySpec)));
   }
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java
index 76ed18e2baf78..0dbb9cd132f8a 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java
@@ -26,7 +26,7 @@ private FilterUtils() {
    * Creates new PolicyMatchCriterion with field and value, using EQUAL PolicyMatchCondition.
    */
   @Nonnull
-  public static PolicyMatchCriterion newCriterion(@Nonnull ResourceFieldType field, @Nonnull List<String> values) {
+  public static PolicyMatchCriterion newCriterion(@Nonnull EntityFieldType field, @Nonnull List<String> values) {
     return newCriterion(field, values, PolicyMatchCondition.EQUALS);
   }
 
@@ -34,7 +34,7 @@ public static PolicyMatchCriterion newCriterion(@Nonnull ResourceFieldType field
    * Creates new PolicyMatchCriterion with field, value and PolicyMatchCondition.
    */
   @Nonnull
-  public static PolicyMatchCriterion newCriterion(@Nonnull ResourceFieldType field, @Nonnull List<String> values,
+  public static PolicyMatchCriterion newCriterion(@Nonnull EntityFieldType field, @Nonnull List<String> values,
       @Nonnull PolicyMatchCondition policyMatchCondition) {
     return new PolicyMatchCriterion().setField(field.name())
         .setValues(new StringArray(values))
@@ -45,7 +45,7 @@ public static PolicyMatchCriterion newCriterion(@Nonnull ResourceFieldType field
    * Creates new PolicyMatchFilter from a map of Criteria by removing null-valued Criteria and using EQUAL PolicyMatchCondition (default).
    */
   @Nonnull
-  public static PolicyMatchFilter newFilter(@Nullable Map<ResourceFieldType, List<String>> params) {
+  public static PolicyMatchFilter newFilter(@Nullable Map<EntityFieldType, List<String>> params) {
     if (params == null) {
       return EMPTY_FILTER;
     }
@@ -61,7 +61,7 @@ public static PolicyMatchFilter newFilter(@Nullable Map<ResourceFieldType, List<
    * Creates new PolicyMatchFilter from a single PolicyMatchCriterion with EQUAL PolicyMatchCondition (default).
    */
   @Nonnull
-  public static PolicyMatchFilter newFilter(@Nonnull ResourceFieldType field, @Nonnull List<String> values) {
+  public static PolicyMatchFilter newFilter(@Nonnull EntityFieldType field, @Nonnull List<String> values) {
     return newFilter(Collections.singletonMap(field, values));
   }
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java
index 6a36fac7de4e0..f8c017ea74e1f 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java
@@ -1,7 +1,6 @@
 package com.datahub.authorization;
 
 import com.datahub.authentication.Authentication;
-import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.Owner;
 import com.linkedin.common.Ownership;
 import com.linkedin.common.urn.Urn;
@@ -11,8 +10,6 @@
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
-import com.linkedin.identity.GroupMembership;
-import com.linkedin.identity.NativeGroupMembership;
 import com.linkedin.identity.RoleMembership;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.authorization.PoliciesConfig;
@@ -23,7 +20,7 @@
 import com.linkedin.policy.PolicyMatchCriterion;
 import com.linkedin.policy.PolicyMatchCriterionArray;
 import com.linkedin.policy.PolicyMatchFilter;
-import java.net.URISyntaxException;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -34,6 +31,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -49,37 +47,22 @@ public class PolicyEngine {
 
   public PolicyEvaluationResult evaluatePolicy(
       final DataHubPolicyInfo policy,
-      final String actorStr,
+      final ResolvedEntitySpec resolvedActorSpec,
       final String privilege,
-      final Optional<ResolvedResourceSpec> resource) {
-    try {
-      // Currently Actor must be an urn. Consider whether this contract should be pushed up.
-      final Urn actor = Urn.createFromString(actorStr);
-      return evaluatePolicy(policy, actor, privilege, resource);
-    } catch (URISyntaxException e) {
-      log.error(String.format("Failed to bind actor %s to an URN. Actors must be URNs. Denying the authorization request", actorStr));
-      return PolicyEvaluationResult.DENIED;
-    }
-  }
-
-  public PolicyEvaluationResult evaluatePolicy(
-      final DataHubPolicyInfo policy,
-      final Urn actor,
-      final String privilege,
-      final Optional<ResolvedResourceSpec> resource) {
+      final Optional<ResolvedEntitySpec> resource) {
 
     final PolicyEvaluationContext context = new PolicyEvaluationContext();
     log.debug("Evaluating policy {}", policy.getDisplayName());
 
     // If the privilege is not in scope, deny the request.
-    if (!isPrivilegeMatch(privilege, policy.getPrivileges(), context)) {
+    if (!isPrivilegeMatch(privilege, policy.getPrivileges())) {
       log.debug("Policy denied based on irrelevant privileges {} for {}", policy.getPrivileges(), privilege);
       return PolicyEvaluationResult.DENIED;
     }
 
     // If policy is not applicable, deny the request
-    if (!isPolicyApplicable(policy, actor, resource, context)) {
-      log.debug("Policy does not applicable for actor {} and resource {}", actor, resource);
+    if (!isPolicyApplicable(policy, resolvedActorSpec, resource, context)) {
+      log.debug("Policy does not applicable for actor {} and resource {}", resolvedActorSpec.getSpec().getEntity(), resource);
       return PolicyEvaluationResult.DENIED;
     }
 
@@ -89,7 +72,7 @@ public PolicyEvaluationResult evaluatePolicy(
 
   public PolicyActors getMatchingActors(
       final DataHubPolicyInfo policy,
-      final Optional<ResolvedResourceSpec> resource) {
+      final Optional<ResolvedEntitySpec> resource) {
     final List<Urn> users = new ArrayList<>();
     final List<Urn> groups = new ArrayList<>();
     boolean allUsers = false;
@@ -126,8 +109,8 @@ public PolicyActors getMatchingActors(
 
   private boolean isPolicyApplicable(
       final DataHubPolicyInfo policy,
-      final Urn actor,
-      final Optional<ResolvedResourceSpec> resource,
+      final ResolvedEntitySpec resolvedActorSpec,
+      final Optional<ResolvedEntitySpec> resource,
       final PolicyEvaluationContext context
   ) {
 
@@ -137,25 +120,21 @@ private boolean isPolicyApplicable(
     }
 
     // If the resource is not in scope, deny the request.
-    if (!isResourceMatch(policy.getType(), policy.getResources(), resource, context)) {
+    if (!isResourceMatch(policy.getType(), policy.getResources(), resource)) {
       return false;
     }
 
     // If the actor does not match, deny the request.
-    if (!isActorMatch(actor, policy.getActors(), resource, context)) {
-      return false;
-    }
-
-    return true;
+    return isActorMatch(resolvedActorSpec, policy.getActors(), resource, context);
   }
 
   public List<String> getGrantedPrivileges(
       final List<DataHubPolicyInfo> policies,
-      final Urn actor,
-      final Optional<ResolvedResourceSpec> resource) {
+      final ResolvedEntitySpec resolvedActorSpec,
+      final Optional<ResolvedEntitySpec> resource) {
     PolicyEvaluationContext context = new PolicyEvaluationContext();
     return policies.stream()
-        .filter(policy -> isPolicyApplicable(policy, actor, resource, context))
+        .filter(policy -> isPolicyApplicable(policy, resolvedActorSpec, resource, context))
         .flatMap(policy -> policy.getPrivileges().stream())
         .distinct()
         .collect(Collectors.toList());
@@ -168,9 +147,8 @@ public List<String> getGrantedPrivileges(
    * If the policy is of type "METADATA", the resourceSpec parameter will be matched against the
    * resource filter defined on the policy.
    */
-  public Boolean policyMatchesResource(final DataHubPolicyInfo policy, final Optional<ResolvedResourceSpec> resourceSpec) {
-    return isResourceMatch(policy.getType(), policy.getResources(), resourceSpec,
-        new PolicyEvaluationContext());
+  public Boolean policyMatchesResource(final DataHubPolicyInfo policy, final Optional<ResolvedEntitySpec> resourceSpec) {
+    return isResourceMatch(policy.getType(), policy.getResources(), resourceSpec);
   }
 
   /**
@@ -178,8 +156,7 @@ public Boolean policyMatchesResource(final DataHubPolicyInfo policy, final Optio
    */
   private boolean isPrivilegeMatch(
       final String requestPrivilege,
-      final List<String> policyPrivileges,
-      final PolicyEvaluationContext context) {
+      final List<String> policyPrivileges) {
     return policyPrivileges.contains(requestPrivilege);
   }
 
@@ -189,8 +166,7 @@ private boolean isPrivilegeMatch(
   private boolean isResourceMatch(
       final String policyType,
       final @Nullable DataHubResourceFilter policyResourceFilter,
-      final Optional<ResolvedResourceSpec> requestResource,
-      final PolicyEvaluationContext context) {
+      final Optional<ResolvedEntitySpec> requestResource) {
     if (PoliciesConfig.PLATFORM_POLICY_TYPE.equals(policyType)) {
       // Currently, platform policies have no associated resource.
       return true;
@@ -199,7 +175,7 @@ private boolean isResourceMatch(
       // No resource defined on the policy.
       return true;
     }
-    if (!requestResource.isPresent()) {
+    if (requestResource.isEmpty()) {
       // Resource filter present in policy, but no resource spec provided.
       log.debug("Resource filter present in policy, but no resource spec provided.");
       return false;
@@ -218,31 +194,31 @@ private PolicyMatchFilter getFilter(DataHubResourceFilter policyResourceFilter)
     }
     PolicyMatchCriterionArray criteria = new PolicyMatchCriterionArray();
     if (policyResourceFilter.hasType()) {
-      criteria.add(new PolicyMatchCriterion().setField(ResourceFieldType.RESOURCE_TYPE.name())
+      criteria.add(new PolicyMatchCriterion().setField(EntityFieldType.TYPE.name())
           .setValues(new StringArray(Collections.singletonList(policyResourceFilter.getType()))));
     }
     if (policyResourceFilter.hasType() && policyResourceFilter.hasResources()
         && !policyResourceFilter.isAllResources()) {
       criteria.add(
-          new PolicyMatchCriterion().setField(ResourceFieldType.RESOURCE_URN.name()).setValues(policyResourceFilter.getResources()));
+          new PolicyMatchCriterion().setField(EntityFieldType.URN.name()).setValues(policyResourceFilter.getResources()));
     }
     return new PolicyMatchFilter().setCriteria(criteria);
   }
 
-  private boolean checkFilter(final PolicyMatchFilter filter, final ResolvedResourceSpec resource) {
+  private boolean checkFilter(final PolicyMatchFilter filter, final ResolvedEntitySpec resource) {
     return filter.getCriteria().stream().allMatch(criterion -> checkCriterion(criterion, resource));
   }
 
-  private boolean checkCriterion(final PolicyMatchCriterion criterion, final ResolvedResourceSpec resource) {
-    ResourceFieldType resourceFieldType;
+  private boolean checkCriterion(final PolicyMatchCriterion criterion, final ResolvedEntitySpec resource) {
+    EntityFieldType entityFieldType;
     try {
-      resourceFieldType = ResourceFieldType.valueOf(criterion.getField().toUpperCase());
+      entityFieldType = EntityFieldType.valueOf(criterion.getField().toUpperCase());
     } catch (IllegalArgumentException e) {
       log.error("Unsupported field type {}", criterion.getField());
       return false;
     }
 
-    Set<String> fieldValues = resource.getFieldValues(resourceFieldType);
+    Set<String> fieldValues = resource.getFieldValues(entityFieldType);
     return criterion.getValues()
         .stream()
         .anyMatch(filterValue -> checkCondition(fieldValues, filterValue, criterion.getCondition()));
@@ -257,46 +233,51 @@ private boolean checkCondition(Set<String> fieldValues, String filterValue, Poli
   }
 
   /**
+   * Returns true if the actor portion of a DataHub policy matches a the actor being evaluated, false otherwise.
    * Returns true if the actor portion of a DataHub policy matches a the actor being evaluated, false otherwise.
    */
   private boolean isActorMatch(
-      final Urn actor,
+      final ResolvedEntitySpec resolvedActorSpec,
       final DataHubActorFilter actorFilter,
-      final Optional<ResolvedResourceSpec> resourceSpec,
+      final Optional<ResolvedEntitySpec> resourceSpec,
       final PolicyEvaluationContext context) {
 
     // 1. If the actor is a matching "User" in the actor filter, return true immediately.
-    if (isUserMatch(actor, actorFilter)) {
+    if (isUserMatch(resolvedActorSpec, actorFilter)) {
       return true;
     }
 
     // 2. If the actor is in a matching "Group" in the actor filter, return true immediately.
-    if (isGroupMatch(actor, actorFilter, context)) {
+    if (isGroupMatch(resolvedActorSpec, actorFilter, context)) {
       return true;
     }
 
     // 3. If the actor is the owner, either directly or indirectly via a group, return true immediately.
-    if (isOwnerMatch(actor, actorFilter, resourceSpec, context)) {
+    if (isOwnerMatch(resolvedActorSpec, actorFilter, resourceSpec, context)) {
       return true;
     }
 
     // 4. If the actor is in a matching "Role" in the actor filter, return true immediately.
-    return isRoleMatch(actor, actorFilter, context);
+    return isRoleMatch(resolvedActorSpec, actorFilter, context);
   }
 
-  private boolean isUserMatch(final Urn actor, final DataHubActorFilter actorFilter) {
+  private boolean isUserMatch(final ResolvedEntitySpec resolvedActorSpec, final DataHubActorFilter actorFilter) {
     // If the actor is a matching "User" in the actor filter, return true immediately.
     return actorFilter.isAllUsers() || (actorFilter.hasUsers() && Objects.requireNonNull(actorFilter.getUsers())
-        .stream()
-        .anyMatch(user -> user.equals(actor)));
+        .stream().map(Urn::toString)
+        .anyMatch(user -> user.equals(resolvedActorSpec.getSpec().getEntity())));
   }
 
-  private boolean isGroupMatch(final Urn actor, final DataHubActorFilter actorFilter, final PolicyEvaluationContext context) {
+  private boolean isGroupMatch(
+      final ResolvedEntitySpec resolvedActorSpec,
+      final DataHubActorFilter actorFilter,
+      final PolicyEvaluationContext context) {
     // If the actor is in a matching "Group" in the actor filter, return true immediately.
     if (actorFilter.isAllGroups() || actorFilter.hasGroups()) {
-      final Set<Urn> groups = resolveGroups(actor, context);
-      return actorFilter.isAllGroups() || (actorFilter.hasGroups() && Objects.requireNonNull(actorFilter.getGroups())
-          .stream()
+      final Set<String> groups = resolveGroups(resolvedActorSpec, context);
+      return (actorFilter.isAllGroups() && !groups.isEmpty())
+          || (actorFilter.hasGroups() && Objects.requireNonNull(actorFilter.getGroups())
+          .stream().map(Urn::toString)
           .anyMatch(groups::contains));
     }
     // If there are no groups on the policy, return false for the group match.
@@ -304,24 +285,24 @@ private boolean isGroupMatch(final Urn actor, final DataHubActorFilter actorFilt
   }
 
   private boolean isOwnerMatch(
-      final Urn actor,
+      final ResolvedEntitySpec resolvedActorSpec,
       final DataHubActorFilter actorFilter,
-      final Optional<ResolvedResourceSpec> requestResource,
+      final Optional<ResolvedEntitySpec> requestResource,
       final PolicyEvaluationContext context) {
     // If the policy does not apply to owners, or there is no resource to own, return false immediately.
-    if (!actorFilter.isResourceOwners() || !requestResource.isPresent()) {
+    if (!actorFilter.isResourceOwners() || requestResource.isEmpty()) {
       return false;
     }
     List<Urn> ownershipTypes = actorFilter.getResourceOwnersTypes();
-    return isActorOwner(actor, requestResource.get(), ownershipTypes, context);
+    return isActorOwner(resolvedActorSpec, requestResource.get(), ownershipTypes, context);
   }
 
-  private Set<String> getOwnersForType(ResourceSpec resourceSpec, List<Urn> ownershipTypes) {
-    Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource());
+  private Set<String> getOwnersForType(EntitySpec resourceSpec, List<Urn> ownershipTypes) {
+    Urn entityUrn = UrnUtils.getUrn(resourceSpec.getEntity());
     EnvelopedAspect ownershipAspect;
     try {
       EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn,
-              Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME), _systemAuthentication);
+          Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME), _systemAuthentication);
       if (response == null || !response.getAspects().containsKey(Constants.OWNERSHIP_ASPECT_NAME)) {
         return Collections.emptySet();
       }
@@ -338,50 +319,56 @@ private Set<String> getOwnersForType(ResourceSpec resourceSpec, List<Urn> owners
     return ownersStream.map(owner -> owner.getOwner().toString()).collect(Collectors.toSet());
   }
 
-  private boolean isActorOwner(Urn actor, ResolvedResourceSpec resourceSpec, List<Urn> ownershipTypes, PolicyEvaluationContext context) {
+  private boolean isActorOwner(
+      final ResolvedEntitySpec resolvedActorSpec,
+      ResolvedEntitySpec resourceSpec, List<Urn> ownershipTypes,
+      PolicyEvaluationContext context) {
     Set<String> owners = this.getOwnersForType(resourceSpec.getSpec(), ownershipTypes);
-    if (isUserOwner(actor, owners)) {
-      return true;
-    }
-    final Set<Urn> groups = resolveGroups(actor, context);
-    if (isGroupOwner(groups, owners)) {
+    if (isUserOwner(resolvedActorSpec, owners)) {
       return true;
     }
-    return false;
+    final Set<String> groups = resolveGroups(resolvedActorSpec, context);
+
+    return isGroupOwner(groups, owners);
   }
 
-  private boolean isUserOwner(Urn actor, Set<String> owners) {
-    return owners.contains(actor.toString());
+  private boolean isUserOwner(final ResolvedEntitySpec resolvedActorSpec, Set<String> owners) {
+    return owners.contains(resolvedActorSpec.getSpec().getEntity());
   }
 
-  private boolean isGroupOwner(Set<Urn> groups, Set<String> owners) {
-    return groups.stream().anyMatch(group -> owners.contains(group.toString()));
+  private boolean isGroupOwner(Set<String> groups, Set<String> owners) {
+    return groups.stream().anyMatch(owners::contains);
   }
 
-  private boolean isRoleMatch(final Urn actor, final DataHubActorFilter actorFilter,
+  private boolean isRoleMatch(
+      final ResolvedEntitySpec resolvedActorSpec,
+      final DataHubActorFilter actorFilter,
       final PolicyEvaluationContext context) {
     // Can immediately return false if the actor filter does not have any roles
     if (!actorFilter.hasRoles()) {
       return false;
     }
     // If the actor has a matching "Role" in the actor filter, return true immediately.
-    Set<Urn> actorRoles = resolveRoles(actor, context);
+    Set<Urn> actorRoles = resolveRoles(resolvedActorSpec, context);
     return Objects.requireNonNull(actorFilter.getRoles())
         .stream()
         .anyMatch(actorRoles::contains);
   }
 
-  private Set<Urn> resolveRoles(Urn actor, PolicyEvaluationContext context) {
+  private Set<Urn> resolveRoles(final ResolvedEntitySpec resolvedActorSpec, PolicyEvaluationContext context) {
     if (context.roles != null) {
       return context.roles;
     }
 
+    String actor = resolvedActorSpec.getSpec().getEntity();
+
     Set<Urn> roles = new HashSet<>();
     final EnvelopedAspectMap aspectMap;
 
     try {
-      final EntityResponse corpUser = _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, Collections.singleton(actor),
-          Collections.singleton(ROLE_MEMBERSHIP_ASPECT_NAME), _systemAuthentication).get(actor);
+      Urn actorUrn = Urn.createFromString(actor);
+      final EntityResponse corpUser = _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, Collections.singleton(actorUrn),
+          Collections.singleton(ROLE_MEMBERSHIP_ASPECT_NAME), _systemAuthentication).get(actorUrn);
       if (corpUser == null || !corpUser.hasAspects()) {
         return roles;
       }
@@ -403,62 +390,25 @@ private Set<Urn> resolveRoles(Urn actor, PolicyEvaluationContext context) {
     return roles;
   }
 
-  private Set<Urn> resolveGroups(Urn actor, PolicyEvaluationContext context) {
+  private Set<String> resolveGroups(ResolvedEntitySpec resolvedActorSpec, PolicyEvaluationContext context) {
     if (context.groups != null) {
       return context.groups;
     }
 
-    Set<Urn> groups = new HashSet<>();
-    final EnvelopedAspectMap aspectMap;
-
-    try {
-      final EntityResponse corpUser = _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, Collections.singleton(actor),
-              ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME), _systemAuthentication)
-          .get(actor);
-      if (corpUser == null || !corpUser.hasAspects()) {
-        return groups;
-      }
-      aspectMap = corpUser.getAspects();
-    } catch (Exception e) {
-      throw new RuntimeException(String.format("Failed to fetch %s and %s for urn %s", GROUP_MEMBERSHIP_ASPECT_NAME,
-          NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME, actor), e);
-    }
-
-    Optional<GroupMembership> maybeGroupMembership = resolveGroupMembership(aspectMap);
-    maybeGroupMembership.ifPresent(groupMembership -> groups.addAll(groupMembership.getGroups()));
-
-    Optional<NativeGroupMembership> maybeNativeGroupMembership = resolveNativeGroupMembership(aspectMap);
-    maybeNativeGroupMembership.ifPresent(
-        nativeGroupMembership -> groups.addAll(nativeGroupMembership.getNativeGroups()));
+    Set<String> groups = resolvedActorSpec.getGroupMembership();
 
     context.setGroups(groups); // Cache the groups.
     return groups;
   }
 
-  // TODO: Optimization - Cache the group membership. Refresh periodically.
-  private Optional<GroupMembership> resolveGroupMembership(final EnvelopedAspectMap aspectMap) {
-    if (aspectMap.containsKey(GROUP_MEMBERSHIP_ASPECT_NAME)) {
-      return Optional.of(new GroupMembership(aspectMap.get(GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()));
-    }
-    return Optional.empty();
-  }
-
-  private Optional<NativeGroupMembership> resolveNativeGroupMembership(final EnvelopedAspectMap aspectMap) {
-    if (aspectMap.containsKey(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)) {
-      return Optional.of(
-          new NativeGroupMembership(aspectMap.get(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()));
-    }
-    return Optional.empty();
-  }
-
   /**
    * Class used to store state across a single Policy evaluation.
    */
   static class PolicyEvaluationContext {
-    private Set<Urn> groups;
+    private Set<String> groups;
     private Set<Urn> roles;
 
-    public void setGroups(Set<Urn> groups) {
+    public void setGroups(Set<String> groups) {
       this.groups = groups;
     }
 
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
index cd838625c2ca1..27cb8fcee8138 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
@@ -1,45 +1,45 @@
 package com.datahub.authorization.fieldresolverprovider;
 
+import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ENTITY_NAME;
+
 import com.datahub.authentication.Authentication;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.authorization.FieldResolver;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
 import com.linkedin.common.DataPlatformInstance;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.client.EntityClient;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
 import java.util.Collections;
 import java.util.Objects;
-
-import static com.linkedin.metadata.Constants.*;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Provides field resolver for domain given resourceSpec
  */
 @Slf4j
 @RequiredArgsConstructor
-public class DataPlatformInstanceFieldResolverProvider implements ResourceFieldResolverProvider {
+public class DataPlatformInstanceFieldResolverProvider implements EntityFieldResolverProvider {
 
   private final EntityClient _entityClient;
   private final Authentication _systemAuthentication;
 
   @Override
-  public ResourceFieldType getFieldType() {
-    return ResourceFieldType.DATA_PLATFORM_INSTANCE;
+  public EntityFieldType getFieldType() {
+    return EntityFieldType.DATA_PLATFORM_INSTANCE;
   }
 
   @Override
-  public FieldResolver getFieldResolver(ResourceSpec resourceSpec) {
-    return FieldResolver.getResolverFromFunction(resourceSpec, this::getDataPlatformInstance);
+  public FieldResolver getFieldResolver(EntitySpec entitySpec) {
+    return FieldResolver.getResolverFromFunction(entitySpec, this::getDataPlatformInstance);
   }
 
-  private FieldResolver.FieldValue getDataPlatformInstance(ResourceSpec resourceSpec) {
-    Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource());
+  private FieldResolver.FieldValue getDataPlatformInstance(EntitySpec entitySpec) {
+    Urn entityUrn = UrnUtils.getUrn(entitySpec.getEntity());
     // In the case that the entity is a platform instance, the associated platform instance entity is the instance itself
     if (entityUrn.getEntityType().equals(DATA_PLATFORM_INSTANCE_ENTITY_NAME)) {
       return FieldResolver.FieldValue.builder()
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
index 68c1dd4f644e5..25c2165f02b94 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
@@ -2,8 +2,8 @@
 
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.FieldResolver;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.domain.DomainProperties;
@@ -27,23 +27,23 @@
 
 
 /**
- * Provides field resolver for domain given resourceSpec
+ * Provides field resolver for domain given entitySpec
  */
 @Slf4j
 @RequiredArgsConstructor
-public class DomainFieldResolverProvider implements ResourceFieldResolverProvider {
+public class DomainFieldResolverProvider implements EntityFieldResolverProvider {
 
   private final EntityClient _entityClient;
   private final Authentication _systemAuthentication;
 
   @Override
-  public ResourceFieldType getFieldType() {
-    return ResourceFieldType.DOMAIN;
+  public EntityFieldType getFieldType() {
+    return EntityFieldType.DOMAIN;
   }
 
   @Override
-  public FieldResolver getFieldResolver(ResourceSpec resourceSpec) {
-    return FieldResolver.getResolverFromFunction(resourceSpec, this::getDomains);
+  public FieldResolver getFieldResolver(EntitySpec entitySpec) {
+    return FieldResolver.getResolverFromFunction(entitySpec, this::getDomains);
   }
 
   private Set<Urn> getBatchedParentDomains(@Nonnull final Set<Urn> urns) {
@@ -78,8 +78,8 @@ private Set<Urn> getBatchedParentDomains(@Nonnull final Set<Urn> urns) {
     return parentUrns;
   }
 
-  private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) {
-    final Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource());
+  private FieldResolver.FieldValue getDomains(EntitySpec entitySpec) {
+    final Urn entityUrn = UrnUtils.getUrn(entitySpec.getEntity());
     // In the case that the entity is a domain, the associated domain is the domain itself
     if (entityUrn.getEntityType().equals(DOMAIN_ENTITY_NAME)) {
       return FieldResolver.FieldValue.builder()
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
new file mode 100644
index 0000000000000..a76db0ecb5102
--- /dev/null
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
@@ -0,0 +1,22 @@
+package com.datahub.authorization.fieldresolverprovider;
+
+import com.datahub.authorization.FieldResolver;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
+
+
+/**
+ * Base class for defining a class that provides the field resolver for the given field type
+ */
+public interface EntityFieldResolverProvider {
+
+  /**
+   * Field that this hydrator is hydrating
+   */
+  EntityFieldType getFieldType();
+
+  /**
+   * Return resolver for fetching the field values given the entity
+   */
+  FieldResolver getFieldResolver(EntitySpec entitySpec);
+}
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
index 58e3d78ce8c3b..187f696904947 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
@@ -1,22 +1,22 @@
 package com.datahub.authorization.fieldresolverprovider;
 
 import com.datahub.authorization.FieldResolver;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
 import java.util.Collections;
 
 
 /**
- * Provides field resolver for entity type given resourceSpec
+ * Provides field resolver for entity type given entitySpec
  */
-public class EntityTypeFieldResolverProvider implements ResourceFieldResolverProvider {
+public class EntityTypeFieldResolverProvider implements EntityFieldResolverProvider {
   @Override
-  public ResourceFieldType getFieldType() {
-    return ResourceFieldType.RESOURCE_TYPE;
+  public EntityFieldType getFieldType() {
+    return EntityFieldType.TYPE;
   }
 
   @Override
-  public FieldResolver getFieldResolver(ResourceSpec resourceSpec) {
-    return FieldResolver.getResolverFromValues(Collections.singleton(resourceSpec.getType()));
+  public FieldResolver getFieldResolver(EntitySpec entitySpec) {
+    return FieldResolver.getResolverFromValues(Collections.singleton(entitySpec.getType()));
   }
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
index b9d98f1dcbac0..2f5c4a7c6c961 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
@@ -1,22 +1,22 @@
 package com.datahub.authorization.fieldresolverprovider;
 
 import com.datahub.authorization.FieldResolver;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
 import java.util.Collections;
 
 
 /**
- * Provides field resolver for entity urn given resourceSpec
+ * Provides field resolver for entity urn given entitySpec
  */
-public class EntityUrnFieldResolverProvider implements ResourceFieldResolverProvider {
+public class EntityUrnFieldResolverProvider implements EntityFieldResolverProvider {
   @Override
-  public ResourceFieldType getFieldType() {
-    return ResourceFieldType.RESOURCE_URN;
+  public EntityFieldType getFieldType() {
+    return EntityFieldType.URN;
   }
 
   @Override
-  public FieldResolver getFieldResolver(ResourceSpec resourceSpec) {
-    return FieldResolver.getResolverFromValues(Collections.singleton(resourceSpec.getResource()));
+  public FieldResolver getFieldResolver(EntitySpec entitySpec) {
+    return FieldResolver.getResolverFromValues(Collections.singleton(entitySpec.getEntity()));
   }
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
new file mode 100644
index 0000000000000..8db029632d7e2
--- /dev/null
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
@@ -0,0 +1,78 @@
+package com.datahub.authorization.fieldresolverprovider;
+
+import com.datahub.authentication.Authentication;
+import com.datahub.authorization.FieldResolver;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
+import com.google.common.collect.ImmutableSet;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.identity.NativeGroupMembership;
+import com.linkedin.metadata.Constants;
+import com.linkedin.identity.GroupMembership;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME;
+
+
+/**
+ * Provides field resolver for owners given entitySpec
+ */
+@Slf4j
+@RequiredArgsConstructor
+public class GroupMembershipFieldResolverProvider implements EntityFieldResolverProvider {
+
+  private final EntityClient _entityClient;
+  private final Authentication _systemAuthentication;
+
+  @Override
+  public EntityFieldType getFieldType() {
+    return EntityFieldType.GROUP_MEMBERSHIP;
+  }
+
+  @Override
+  public FieldResolver getFieldResolver(EntitySpec entitySpec) {
+    return FieldResolver.getResolverFromFunction(entitySpec, this::getGroupMembership);
+  }
+
+  private FieldResolver.FieldValue getGroupMembership(EntitySpec entitySpec) {
+    Urn entityUrn = UrnUtils.getUrn(entitySpec.getEntity());
+    EnvelopedAspect groupMembershipAspect;
+    EnvelopedAspect nativeGroupMembershipAspect;
+    List<Urn> groups = new ArrayList<>();
+    try {
+      EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn,
+              ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME), _systemAuthentication);
+      if (response == null
+              || !(response.getAspects().containsKey(Constants.GROUP_MEMBERSHIP_ASPECT_NAME)
+              || response.getAspects().containsKey(Constants.NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME))) {
+        return FieldResolver.emptyFieldValue();
+      }
+      if (response.getAspects().containsKey(Constants.GROUP_MEMBERSHIP_ASPECT_NAME)) {
+        groupMembershipAspect = response.getAspects().get(Constants.GROUP_MEMBERSHIP_ASPECT_NAME);
+        GroupMembership groupMembership = new GroupMembership(groupMembershipAspect.getValue().data());
+        groups.addAll(groupMembership.getGroups());
+      }
+      if (response.getAspects().containsKey(Constants.NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)) {
+        nativeGroupMembershipAspect = response.getAspects().get(Constants.NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME);
+        NativeGroupMembership nativeGroupMembership = new NativeGroupMembership(nativeGroupMembershipAspect.getValue().data());
+        groups.addAll(nativeGroupMembership.getNativeGroups());
+      }
+    } catch (Exception e) {
+      log.error("Error while retrieving group membership aspect for urn {}", entityUrn, e);
+      return FieldResolver.emptyFieldValue();
+    }
+    return FieldResolver.FieldValue.builder()
+        .values(groups.stream().map(Urn::toString).collect(Collectors.toSet()))
+        .build();
+  }
+}
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
index 20ec6a09377c8..bdd652d1d3871 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
@@ -2,8 +2,8 @@
 
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.FieldResolver;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.common.Ownership;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
@@ -18,27 +18,27 @@
 
 
 /**
- * Provides field resolver for owners given resourceSpec
+ * Provides field resolver for owners given entitySpec
  */
 @Slf4j
 @RequiredArgsConstructor
-public class OwnerFieldResolverProvider implements ResourceFieldResolverProvider {
+public class OwnerFieldResolverProvider implements EntityFieldResolverProvider {
 
   private final EntityClient _entityClient;
   private final Authentication _systemAuthentication;
 
   @Override
-  public ResourceFieldType getFieldType() {
-    return ResourceFieldType.OWNER;
+  public EntityFieldType getFieldType() {
+    return EntityFieldType.OWNER;
   }
 
   @Override
-  public FieldResolver getFieldResolver(ResourceSpec resourceSpec) {
-    return FieldResolver.getResolverFromFunction(resourceSpec, this::getOwners);
+  public FieldResolver getFieldResolver(EntitySpec entitySpec) {
+    return FieldResolver.getResolverFromFunction(entitySpec, this::getOwners);
   }
 
-  private FieldResolver.FieldValue getOwners(ResourceSpec resourceSpec) {
-    Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource());
+  private FieldResolver.FieldValue getOwners(EntitySpec entitySpec) {
+    Urn entityUrn = UrnUtils.getUrn(entitySpec.getEntity());
     EnvelopedAspect ownershipAspect;
     try {
       EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn,
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java
deleted file mode 100644
index 4ba4200f8035e..0000000000000
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package com.datahub.authorization.fieldresolverprovider;
-
-import com.datahub.authorization.FieldResolver;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
-
-
-/**
- * Base class for defining a class that provides the field resolver for the given field type
- */
-public interface ResourceFieldResolverProvider {
-
-  /**
-   * Field that this hydrator is hydrating
-   */
-  ResourceFieldType getFieldType();
-
-  /**
-   * Return resolver for fetching the field values given the resource
-   */
-  FieldResolver getFieldResolver(ResourceSpec resourceSpec);
-}
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
index 2e48123fb1813..24ecfa6fefc85 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
@@ -158,7 +158,7 @@ public void testSystemAuthentication() throws Exception {
 
     // Validate that the System Actor is authorized, even if there is no policy.
 
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     AuthorizationRequest request = new AuthorizationRequest(
         new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID).toUrnStr(),
@@ -172,7 +172,7 @@ public void testSystemAuthentication() throws Exception {
   @Test
   public void testAuthorizeGranted() throws Exception {
 
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     AuthorizationRequest request = new AuthorizationRequest(
         "urn:li:corpuser:test",
@@ -186,7 +186,7 @@ public void testAuthorizeGranted() throws Exception {
   @Test
   public void testAuthorizeNotGranted() throws Exception {
 
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     // Policy for this privilege is inactive.
     AuthorizationRequest request = new AuthorizationRequest(
@@ -203,7 +203,7 @@ public void testAllowAllMode() throws Exception {
 
     _dataHubAuthorizer.setMode(DataHubAuthorizer.AuthorizationMode.ALLOW_ALL);
 
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     // Policy for this privilege is inactive.
     AuthorizationRequest request = new AuthorizationRequest(
@@ -219,7 +219,7 @@ public void testAllowAllMode() throws Exception {
   public void testInvalidateCache() throws Exception {
 
     // First make sure that the default policies are as expected.
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     AuthorizationRequest request = new AuthorizationRequest(
         "urn:li:corpuser:test",
@@ -250,7 +250,7 @@ public void testInvalidateCache() throws Exception {
   public void testAuthorizedActorsActivePolicy() throws Exception {
     final AuthorizedActors actors =
         _dataHubAuthorizer.authorizedActors("EDIT_ENTITY_TAGS", // Should be inside the active policy.
-            Optional.of(new ResourceSpec("dataset", "urn:li:dataset:1")));
+            Optional.of(new EntitySpec("dataset", "urn:li:dataset:1")));
 
     assertTrue(actors.isAllUsers());
     assertTrue(actors.isAllGroups());
@@ -272,7 +272,7 @@ public void testAuthorizedActorsActivePolicy() throws Exception {
 
   @Test
   public void testAuthorizationOnDomainWithPrivilegeIsAllowed() {
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     AuthorizationRequest request = new AuthorizationRequest(
         "urn:li:corpuser:test",
@@ -285,7 +285,7 @@ public void testAuthorizationOnDomainWithPrivilegeIsAllowed() {
 
   @Test
   public void testAuthorizationOnDomainWithParentPrivilegeIsAllowed() {
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     AuthorizationRequest request = new AuthorizationRequest(
         "urn:li:corpuser:test",
@@ -298,7 +298,7 @@ public void testAuthorizationOnDomainWithParentPrivilegeIsAllowed() {
 
   @Test
   public void testAuthorizationOnDomainWithoutPrivilegeIsDenied() {
-    ResourceSpec resourceSpec = new ResourceSpec("dataset", "urn:li:dataset:test");
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
 
     AuthorizationRequest request = new AuthorizationRequest(
         "urn:li:corpuser:test",
@@ -334,7 +334,7 @@ private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List<String> p
     resourceFilter.setType("dataset");
 
     if (domain != null) {
-      resourceFilter.setFilter(FilterUtils.newFilter(ImmutableMap.of(ResourceFieldType.DOMAIN, Collections.singletonList(domain.toString()))));
+      resourceFilter.setFilter(FilterUtils.newFilter(ImmutableMap.of(EntityFieldType.DOMAIN, Collections.singletonList(domain.toString()))));
     }
 
     dataHubPolicyInfo.setResources(resourceFilter);
@@ -398,6 +398,6 @@ private Map<Urn, EntityResponse> createDomainPropertiesBatchResponse(@Nullable f
   }
 
   private AuthorizerContext createAuthorizerContext(final Authentication systemAuthentication, final EntityClient entityClient) {
-    return new AuthorizerContext(Collections.emptyMap(), new DefaultResourceSpecResolver(systemAuthentication, entityClient));
+    return new AuthorizerContext(Collections.emptyMap(), new DefaultEntitySpecResolver(systemAuthentication, entityClient));
   }
 }
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java
index 99d8fee309d91..be8c948f8ef89 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java
@@ -11,15 +11,12 @@
 import com.linkedin.common.OwnershipType;
 import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.Urn;
-import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.StringArray;
 import com.linkedin.entity.Aspect;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
-import com.linkedin.identity.CorpUserInfo;
-import com.linkedin.identity.GroupMembership;
 import com.linkedin.identity.RoleMembership;
 import com.linkedin.metadata.Constants;
 import com.linkedin.policy.DataHubActorFilter;
@@ -45,22 +42,19 @@ public class PolicyEngineTest {
 
   private static final String AUTHORIZED_PRINCIPAL = "urn:li:corpuser:datahub";
   private static final String UNAUTHORIZED_PRINCIPAL = "urn:li:corpuser:unauthorized";
-
   private static final String AUTHORIZED_GROUP = "urn:li:corpGroup:authorizedGroup";
-
   private static final String RESOURCE_URN = "urn:li:dataset:test";
-
   private static final String DOMAIN_URN = "urn:li:domain:domain1";
-
   private static final String OWNERSHIP_TYPE_URN = "urn:li:ownershipType:__system__technical_owner";
-
   private static final String OTHER_OWNERSHIP_TYPE_URN = "urn:li:ownershipType:__system__data_steward";
 
   private EntityClient _entityClient;
   private PolicyEngine _policyEngine;
 
   private Urn authorizedUserUrn;
+  private ResolvedEntitySpec resolvedAuthorizedUserSpec;
   private Urn unauthorizedUserUrn;
+  private ResolvedEntitySpec resolvedUnauthorizedUserSpec;
   private Urn resourceUrn;
 
   @BeforeMethod
@@ -68,29 +62,34 @@ public void setupTest() throws Exception {
     _entityClient = Mockito.mock(EntityClient.class);
     _policyEngine = new PolicyEngine(Mockito.mock(Authentication.class), _entityClient);
 
-    // Init mocks.
-    EntityResponse authorizedEntityResponse = createAuthorizedEntityResponse();
     authorizedUserUrn = Urn.createFromString(AUTHORIZED_PRINCIPAL);
+    resolvedAuthorizedUserSpec = buildEntityResolvers(CORP_USER_ENTITY_NAME, AUTHORIZED_PRINCIPAL,
+        Collections.emptySet(), Collections.emptySet(), Collections.singleton(AUTHORIZED_GROUP));
+    unauthorizedUserUrn = Urn.createFromString(UNAUTHORIZED_PRINCIPAL);
+    resolvedUnauthorizedUserSpec = buildEntityResolvers(CORP_USER_ENTITY_NAME, UNAUTHORIZED_PRINCIPAL);
+    resourceUrn = Urn.createFromString(RESOURCE_URN);
+
+    // Init role membership mocks.
+    EntityResponse authorizedEntityResponse = createAuthorizedEntityResponse();
     authorizedEntityResponse.setUrn(authorizedUserUrn);
     Map<Urn, EntityResponse> authorizedEntityResponseMap =
         Collections.singletonMap(authorizedUserUrn, authorizedEntityResponse);
-    when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), any(),
-        any())).thenReturn(authorizedEntityResponseMap);
+    when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
+        eq(Collections.singleton(ROLE_MEMBERSHIP_ASPECT_NAME)), any())).thenReturn(authorizedEntityResponseMap);
 
     EntityResponse unauthorizedEntityResponse = createUnauthorizedEntityResponse();
-    unauthorizedUserUrn = Urn.createFromString(UNAUTHORIZED_PRINCIPAL);
     unauthorizedEntityResponse.setUrn(unauthorizedUserUrn);
     Map<Urn, EntityResponse> unauthorizedEntityResponseMap =
         Collections.singletonMap(unauthorizedUserUrn, unauthorizedEntityResponse);
-    when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), any(),
-        any())).thenReturn(unauthorizedEntityResponseMap);
+    when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)),
+        eq(Collections.singleton(ROLE_MEMBERSHIP_ASPECT_NAME)), any())).thenReturn(unauthorizedEntityResponseMap);
 
+    // Init ownership type mocks.
     EntityResponse entityResponse = new EntityResponse();
     EnvelopedAspectMap envelopedAspectMap = new EnvelopedAspectMap();
     envelopedAspectMap.put(OWNERSHIP_ASPECT_NAME,
         new EnvelopedAspect().setValue(new com.linkedin.entity.Aspect(createOwnershipAspect(true, true).data())));
     entityResponse.setAspects(envelopedAspectMap);
-    resourceUrn = Urn.createFromString(RESOURCE_URN);
     Map<Urn, EntityResponse> mockMap = mock(Map.class);
     when(_entityClient.batchGetV2(any(), eq(Collections.singleton(resourceUrn)),
         eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), any())).thenReturn(mockMap);
@@ -120,9 +119,9 @@ public void testEvaluatePolicyInactivePolicyState() {
     resourceFilter.setAllResources(true);
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
 
     assertFalse(result.isGranted());
@@ -149,9 +148,9 @@ public void testEvaluatePolicyPrivilegeFilterNoMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_OWNERS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_OWNERS",
             Optional.of(resourceSpec));
     assertFalse(result.isGranted());
 
@@ -176,7 +175,8 @@ public void testEvaluatePlatformPolicyPrivilegeFilterMatch() throws Exception {
     dataHubPolicyInfo.setActors(actorFilter);
 
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "MANAGE_POLICIES", Optional.empty());
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "MANAGE_POLICIES",
+            Optional.empty());
     assertTrue(result.isGranted());
 
     // Verify no network calls
@@ -208,10 +208,10 @@ public void testEvaluatePolicyActorFilterUserMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert Authorized user can edit entity tags.
     PolicyEngine.PolicyEvaluationResult result1 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
 
     assertTrue(result1.isGranted());
@@ -245,10 +245,10 @@ public void testEvaluatePolicyActorFilterUserNoMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert unauthorized user cannot edit entity tags.
     PolicyEngine.PolicyEvaluationResult result2 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, "urn:li:corpuser:test", "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, buildEntityResolvers(CORP_USER_ENTITY_NAME, "urn:li:corpuser:test"), "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
 
     assertFalse(result2.isGranted());
@@ -270,7 +270,7 @@ public void testEvaluatePolicyActorFilterGroupMatch() throws Exception {
 
     final DataHubActorFilter actorFilter = new DataHubActorFilter();
     final UrnArray groupsUrnArray = new UrnArray();
-    groupsUrnArray.add(Urn.createFromString("urn:li:corpGroup:authorizedGroup"));
+    groupsUrnArray.add(Urn.createFromString(AUTHORIZED_GROUP));
     actorFilter.setGroups(groupsUrnArray);
     actorFilter.setResourceOwners(false);
     actorFilter.setAllUsers(false);
@@ -282,16 +282,15 @@ public void testEvaluatePolicyActorFilterGroupMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert authorized user can edit entity tags, because of group membership.
     PolicyEngine.PolicyEvaluationResult result1 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result1.isGranted());
 
-    // Verify we are only calling for group during these requests.
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
-        any(), any());
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -307,7 +306,7 @@ public void testEvaluatePolicyActorFilterGroupNoMatch() throws Exception {
 
     final DataHubActorFilter actorFilter = new DataHubActorFilter();
     final UrnArray groupsUrnArray = new UrnArray();
-    groupsUrnArray.add(Urn.createFromString("urn:li:corpGroup:authorizedGroup"));
+    groupsUrnArray.add(Urn.createFromString(AUTHORIZED_GROUP));
     actorFilter.setGroups(groupsUrnArray);
     actorFilter.setResourceOwners(false);
     actorFilter.setAllUsers(false);
@@ -319,16 +318,15 @@ public void testEvaluatePolicyActorFilterGroupNoMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert unauthorized user cannot edit entity tags.
     PolicyEngine.PolicyEvaluationResult result2 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedUnauthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertFalse(result2.isGranted());
 
-    // Verify we are only calling for group during these requests.
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME),
-        eq(Collections.singleton(unauthorizedUserUrn)), any(), any());
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -357,17 +355,17 @@ public void testEvaluatePolicyActorFilterRoleMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert authorized user can edit entity tags.
     PolicyEngine.PolicyEvaluationResult authorizedResult =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
 
     assertTrue(authorizedResult.isGranted());
 
     // Verify we are only calling for roles during these requests.
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
-        any(), any());
+    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME),
+        eq(Collections.singleton(authorizedUserUrn)), any(), any());
   }
 
   @Test
@@ -396,10 +394,10 @@ public void testEvaluatePolicyActorFilterNoRoleMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert authorized user can edit entity tags.
     PolicyEngine.PolicyEvaluationResult unauthorizedResult =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedUnauthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
 
     assertFalse(unauthorizedResult.isGranted());
@@ -431,16 +429,16 @@ public void testEvaluatePolicyActorFilterAllUsersMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert authorized user can edit entity tags, because of group membership.
     PolicyEngine.PolicyEvaluationResult result1 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result1.isGranted());
 
     // Assert unauthorized user cannot edit entity tags.
     PolicyEngine.PolicyEvaluationResult result2 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedUnauthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result2.isGranted());
 
@@ -470,24 +468,21 @@ public void testEvaluatePolicyActorFilterAllGroupsMatch() throws Exception {
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert authorized user can edit entity tags, because of group membership.
     PolicyEngine.PolicyEvaluationResult result1 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result1.isGranted());
 
     // Assert unauthorized user cannot edit entity tags.
     PolicyEngine.PolicyEvaluationResult result2 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedUnauthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
-    assertTrue(result2.isGranted());
+    assertFalse(result2.isGranted());
 
-    // Verify we are only calling for group during these requests.
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
-        any(), any());
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME),
-        eq(Collections.singleton(unauthorizedUserUrn)), any(), any());
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -519,17 +514,17 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersMatch() throws Except
     when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)),
             any())).thenReturn(entityResponse);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet());
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet(),
+            Collections.emptySet());
     // Assert authorized user can edit entity tags, because he is a user owner.
     PolicyEngine.PolicyEvaluationResult result1 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result1.isGranted());
 
-    // Ensure no calls for group membership.
-    verify(_entityClient, times(0)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
-        eq(null), any());
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -562,13 +557,17 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersTypeMatch() throws Ex
     when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)),
             any())).thenReturn(entityResponse);
 
-    ResolvedResourceSpec resourceSpec =
-            buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet());
+    ResolvedEntitySpec resourceSpec =
+            buildEntityResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet(),
+                Collections.emptySet());
     
     PolicyEngine.PolicyEvaluationResult result1 =
-            _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+            _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
                     Optional.of(resourceSpec));
     assertTrue(result1.isGranted());
+
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -601,13 +600,16 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersTypeNoMatch() throws
     when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)),
             any())).thenReturn(entityResponse);
 
-    ResolvedResourceSpec resourceSpec =
-            buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet());
+    ResolvedEntitySpec resourceSpec =
+            buildEntityResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet(), Collections.emptySet());
 
     PolicyEngine.PolicyEvaluationResult result1 =
-            _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+            _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
                     Optional.of(resourceSpec));
     assertFalse(result1.isGranted());
+
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -639,17 +641,17 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersMatch() throws Excep
     when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)),
             any())).thenReturn(entityResponse);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_GROUP), Collections.emptySet());
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_GROUP), Collections.emptySet(),
+            Collections.emptySet());
     // Assert authorized user can edit entity tags, because he is a user owner.
     PolicyEngine.PolicyEvaluationResult result1 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result1.isGranted());
 
-    // Ensure that caching of groups is working with 1 call to entity client for each principal.
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
-        any(), any());
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -673,16 +675,15 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersNoMatch() throws Exc
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     // Assert unauthorized user cannot edit entity tags.
     PolicyEngine.PolicyEvaluationResult result2 =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedUnauthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertFalse(result2.isGranted());
 
-    // Ensure that caching of groups is working with 1 call to entity client for each principal.
-    verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME),
-        eq(Collections.singleton(unauthorizedUserUrn)), any(), any());
+    // Verify no network calls
+    verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
   @Test
@@ -706,10 +707,10 @@ public void testEvaluatePolicyResourceFilterAllResourcesMatch() throws Exception
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", "urn:li:dataset:random"); // A dataset Authorized principal _does not own_.
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", "urn:li:dataset:random"); // A dataset Authorized principal _does not own_.
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result.isGranted());
 
@@ -738,9 +739,9 @@ public void testEvaluatePolicyResourceFilterAllResourcesNoMatch() throws Excepti
     resourceFilter.setType("dataset");
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("chart", RESOURCE_URN); // Notice: Not a dataset.
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("chart", RESOURCE_URN); // Notice: Not a dataset.
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertFalse(result.isGranted());
 
@@ -773,9 +774,9 @@ public void testEvaluatePolicyResourceFilterSpecificResourceMatchLegacy() throws
     resourceFilter.setResources(resourceUrns);
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result.isGranted());
 
@@ -801,13 +802,13 @@ public void testEvaluatePolicyResourceFilterSpecificResourceMatch() throws Excep
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
     resourceFilter.setFilter(FilterUtils.newFilter(
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"),
-            ResourceFieldType.RESOURCE_URN, Collections.singletonList(RESOURCE_URN))));
+        ImmutableMap.of(EntityFieldType.TYPE, Collections.singletonList("dataset"),
+            EntityFieldType.URN, Collections.singletonList(RESOURCE_URN))));
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN);
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN);
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result.isGranted());
 
@@ -833,14 +834,14 @@ public void testEvaluatePolicyResourceFilterSpecificResourceNoMatch() throws Exc
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
     resourceFilter.setFilter(FilterUtils.newFilter(
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"),
-            ResourceFieldType.RESOURCE_URN, Collections.singletonList(RESOURCE_URN))));
+        ImmutableMap.of(EntityFieldType.TYPE, Collections.singletonList("dataset"),
+            EntityFieldType.URN, Collections.singletonList(RESOURCE_URN))));
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy.
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy.
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertFalse(result.isGranted());
 
@@ -866,14 +867,14 @@ public void testEvaluatePolicyResourceFilterSpecificResourceMatchDomain() throws
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
     resourceFilter.setFilter(FilterUtils.newFilter(
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.DOMAIN,
+        ImmutableMap.of(EntityFieldType.TYPE, Collections.singletonList("dataset"), EntityFieldType.DOMAIN,
             Collections.singletonList(DOMAIN_URN))));
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(), Collections.singleton(DOMAIN_URN));
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", RESOURCE_URN, Collections.emptySet(), Collections.singleton(DOMAIN_URN), Collections.emptySet());
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertTrue(result.isGranted());
 
@@ -899,14 +900,14 @@ public void testEvaluatePolicyResourceFilterSpecificResourceNoMatchDomain() thro
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
     resourceFilter.setFilter(FilterUtils.newFilter(
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.DOMAIN,
+        ImmutableMap.of(EntityFieldType.TYPE, Collections.singletonList("dataset"), EntityFieldType.DOMAIN,
             Collections.singletonList(DOMAIN_URN))));
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(),
-        Collections.singleton("urn:li:domain:domain2")); // Domain doesn't match
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN, Collections.emptySet(),
+        Collections.singleton("urn:li:domain:domain2"), Collections.emptySet()); // Domain doesn't match
     PolicyEngine.PolicyEvaluationResult result =
-        _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS",
+        _policyEngine.evaluatePolicy(dataHubPolicyInfo, resolvedAuthorizedUserSpec, "EDIT_ENTITY_TAGS",
             Optional.of(resourceSpec));
     assertFalse(result.isGranted());
 
@@ -933,7 +934,7 @@ public void testGetGrantedPrivileges() throws Exception {
 
     final DataHubResourceFilter resourceFilter1 = new DataHubResourceFilter();
     resourceFilter1.setFilter(FilterUtils.newFilter(
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.DOMAIN,
+        ImmutableMap.of(EntityFieldType.TYPE, Collections.singletonList("dataset"), EntityFieldType.DOMAIN,
             Collections.singletonList(DOMAIN_URN))));
     dataHubPolicyInfo1.setResources(resourceFilter1);
 
@@ -954,8 +955,8 @@ public void testGetGrantedPrivileges() throws Exception {
 
     final DataHubResourceFilter resourceFilter2 = new DataHubResourceFilter();
     resourceFilter2.setFilter(FilterUtils.newFilter(
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"),
-            ResourceFieldType.RESOURCE_URN, Collections.singletonList(RESOURCE_URN))));
+        ImmutableMap.of(EntityFieldType.TYPE, Collections.singletonList("dataset"),
+            EntityFieldType.URN, Collections.singletonList(RESOURCE_URN))));
     dataHubPolicyInfo2.setResources(resourceFilter2);
 
     // Policy 3, match dataset type and owner (legacy resource filter)
@@ -981,25 +982,25 @@ public void testGetGrantedPrivileges() throws Exception {
     final List<DataHubPolicyInfo> policies =
         ImmutableList.of(dataHubPolicyInfo1, dataHubPolicyInfo2, dataHubPolicyInfo3);
 
-    assertEquals(_policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.empty()),
+    assertEquals(_policyEngine.getGrantedPrivileges(policies, resolvedAuthorizedUserSpec, Optional.empty()),
         Collections.emptyList());
 
-    ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(),
-        Collections.singleton(DOMAIN_URN)); // Everything matches
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN, Collections.emptySet(),
+        Collections.singleton(DOMAIN_URN), Collections.emptySet()); // Everything matches
     assertEquals(
-        _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)),
+        _policyEngine.getGrantedPrivileges(policies, resolvedAuthorizedUserSpec, Optional.of(resourceSpec)),
         ImmutableList.of("PRIVILEGE_1", "PRIVILEGE_2_1", "PRIVILEGE_2_2"));
 
-    resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(),
-        Collections.singleton("urn:li:domain:domain2")); // Domain doesn't match
+    resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN, Collections.emptySet(),
+        Collections.singleton("urn:li:domain:domain2"), Collections.emptySet()); // Domain doesn't match
     assertEquals(
-        _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)),
+        _policyEngine.getGrantedPrivileges(policies, resolvedAuthorizedUserSpec, Optional.of(resourceSpec)),
         ImmutableList.of("PRIVILEGE_2_1", "PRIVILEGE_2_2"));
 
-    resourceSpec = buildResourceResolvers("dataset", "urn:li:dataset:random", Collections.emptySet(),
-        Collections.singleton(DOMAIN_URN)); // Resource doesn't match
+    resourceSpec = buildEntityResolvers("dataset", "urn:li:dataset:random", Collections.emptySet(),
+        Collections.singleton(DOMAIN_URN), Collections.emptySet()); // Resource doesn't match
     assertEquals(
-        _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)),
+        _policyEngine.getGrantedPrivileges(policies, resolvedAuthorizedUserSpec, Optional.of(resourceSpec)),
         ImmutableList.of("PRIVILEGE_1"));
 
     final EntityResponse entityResponse = new EntityResponse();
@@ -1008,16 +1009,16 @@ public void testGetGrantedPrivileges() throws Exception {
     entityResponse.setAspects(aspectMap);
     when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)),
             any())).thenReturn(entityResponse);
-    resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL),
-        Collections.singleton(DOMAIN_URN)); // Is owner
+    resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL),
+        Collections.singleton(DOMAIN_URN), Collections.emptySet()); // Is owner
     assertEquals(
-        _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)),
+        _policyEngine.getGrantedPrivileges(policies, resolvedAuthorizedUserSpec, Optional.of(resourceSpec)),
         ImmutableList.of("PRIVILEGE_1", "PRIVILEGE_2_1", "PRIVILEGE_2_2", "PRIVILEGE_3"));
 
-    resourceSpec = buildResourceResolvers("chart", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL),
-        Collections.singleton(DOMAIN_URN)); // Resource type doesn't match
+    resourceSpec = buildEntityResolvers("chart", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL),
+        Collections.singleton(DOMAIN_URN), Collections.emptySet()); // Resource type doesn't match
     assertEquals(
-        _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)),
+        _policyEngine.getGrantedPrivileges(policies, resolvedAuthorizedUserSpec, Optional.of(resourceSpec)),
         Collections.emptyList());
   }
 
@@ -1050,9 +1051,9 @@ public void testGetMatchingActorsResourceMatch() throws Exception {
     resourceFilter.setResources(resourceUrns);
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL, AUTHORIZED_GROUP),
-            Collections.emptySet());
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL, AUTHORIZED_GROUP),
+            Collections.emptySet(), Collections.emptySet());
     PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec));
 
     assertTrue(actors.allUsers());
@@ -1101,8 +1102,8 @@ public void testGetMatchingActorsNoResourceMatch() throws Exception {
     resourceFilter.setResources(resourceUrns);
     dataHubPolicyInfo.setResources(resourceFilter);
 
-    ResolvedResourceSpec resourceSpec =
-        buildResourceResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy.
+    ResolvedEntitySpec resourceSpec =
+        buildEntityResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy.
     PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec));
 
     assertFalse(actors.allUsers());
@@ -1155,21 +1156,6 @@ private EntityResponse createAuthorizedEntityResponse() throws URISyntaxExceptio
     final EntityResponse entityResponse = new EntityResponse();
     final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
 
-    final CorpUserInfo userInfo = new CorpUserInfo();
-    userInfo.setActive(true);
-    userInfo.setFullName("Data Hub");
-    userInfo.setFirstName("Data");
-    userInfo.setLastName("Hub");
-    userInfo.setEmail("datahub@gmail.com");
-    userInfo.setTitle("Admin");
-    aspectMap.put(CORP_USER_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(userInfo.data())));
-
-    final GroupMembership groupsAspect = new GroupMembership();
-    final UrnArray groups = new UrnArray();
-    groups.add(Urn.createFromString("urn:li:corpGroup:authorizedGroup"));
-    groupsAspect.setGroups(groups);
-    aspectMap.put(GROUP_MEMBERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(groupsAspect.data())));
-
     final RoleMembership rolesAspect = new RoleMembership();
     final UrnArray roles = new UrnArray();
     roles.add(Urn.createFromString("urn:li:dataHubRole:admin"));
@@ -1184,21 +1170,6 @@ private EntityResponse createUnauthorizedEntityResponse() throws URISyntaxExcept
     final EntityResponse entityResponse = new EntityResponse();
     final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
 
-    final CorpUserInfo userInfo = new CorpUserInfo();
-    userInfo.setActive(true);
-    userInfo.setFullName("Unauthorized User");
-    userInfo.setFirstName("Unauthorized");
-    userInfo.setLastName("User");
-    userInfo.setEmail("Unauth");
-    userInfo.setTitle("Engineer");
-    aspectMap.put(CORP_USER_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(userInfo.data())));
-
-    final GroupMembership groupsAspect = new GroupMembership();
-    final UrnArray groups = new UrnArray();
-    groups.add(Urn.createFromString("urn:li:corpGroup:unauthorizedGroup"));
-    groupsAspect.setGroups(groups);
-    aspectMap.put(GROUP_MEMBERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(groupsAspect.data())));
-
     final RoleMembership rolesAspect = new RoleMembership();
     final UrnArray roles = new UrnArray();
     roles.add(Urn.createFromString("urn:li:dataHubRole:reader"));
@@ -1209,17 +1180,18 @@ private EntityResponse createUnauthorizedEntityResponse() throws URISyntaxExcept
     return entityResponse;
   }
 
-  public static ResolvedResourceSpec buildResourceResolvers(String entityType, String entityUrn) {
-    return buildResourceResolvers(entityType, entityUrn, Collections.emptySet(), Collections.emptySet());
+  public static ResolvedEntitySpec buildEntityResolvers(String entityType, String entityUrn) {
+    return buildEntityResolvers(entityType, entityUrn, Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
   }
 
-  public static ResolvedResourceSpec buildResourceResolvers(String entityType, String entityUrn, Set<String> owners,
-      Set<String> domains) {
-    return new ResolvedResourceSpec(new ResourceSpec(entityType, entityUrn),
-        ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE,
-            FieldResolver.getResolverFromValues(Collections.singleton(entityType)), ResourceFieldType.RESOURCE_URN,
-            FieldResolver.getResolverFromValues(Collections.singleton(entityUrn)), ResourceFieldType.OWNER,
-            FieldResolver.getResolverFromValues(owners), ResourceFieldType.DOMAIN,
-            FieldResolver.getResolverFromValues(domains)));
+  public static ResolvedEntitySpec buildEntityResolvers(String entityType, String entityUrn, Set<String> owners,
+      Set<String> domains, Set<String> groups) {
+    return new ResolvedEntitySpec(new EntitySpec(entityType, entityUrn),
+        ImmutableMap.of(EntityFieldType.TYPE,
+            FieldResolver.getResolverFromValues(Collections.singleton(entityType)), EntityFieldType.URN,
+            FieldResolver.getResolverFromValues(Collections.singleton(entityUrn)), EntityFieldType.OWNER,
+            FieldResolver.getResolverFromValues(owners), EntityFieldType.DOMAIN,
+            FieldResolver.getResolverFromValues(domains), EntityFieldType.GROUP_MEMBERSHIP,
+            FieldResolver.getResolverFromValues(groups)));
   }
 }
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
index e525c602c2620..b2343bbb01509 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
@@ -1,8 +1,21 @@
 package com.datahub.authorization.fieldresolverprovider;
 
+import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME;
+import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME;
+import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ENTITY_NAME;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
 import com.datahub.authentication.Authentication;
-import com.datahub.authorization.ResourceFieldType;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.common.DataPlatformInstance;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.Aspect;
@@ -11,29 +24,21 @@
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.r2.RemoteInvocationException;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.Set;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.Set;
-
-import static com.linkedin.metadata.Constants.*;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.*;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
 public class DataPlatformInstanceFieldResolverProviderTest {
 
   private static final String DATA_PLATFORM_INSTANCE_URN =
       "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)";
   private static final String RESOURCE_URN =
       "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.testDataset,PROD)";
-  private static final ResourceSpec RESOURCE_SPEC = new ResourceSpec(DATASET_ENTITY_NAME, RESOURCE_URN);
+  private static final EntitySpec RESOURCE_SPEC = new EntitySpec(DATASET_ENTITY_NAME, RESOURCE_URN);
 
   @Mock
   private EntityClient entityClientMock;
@@ -51,12 +56,12 @@ public void setup() {
 
   @Test
   public void shouldReturnDataPlatformInstanceType() {
-    assertEquals(ResourceFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldType());
+    assertEquals(EntityFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldType());
   }
 
   @Test
   public void shouldReturnFieldValueWithResourceSpecIfTypeIsDataPlatformInstance() {
-    var resourceSpec = new ResourceSpec(DATA_PLATFORM_INSTANCE_ENTITY_NAME, DATA_PLATFORM_INSTANCE_URN);
+    var resourceSpec = new EntitySpec(DATA_PLATFORM_INSTANCE_ENTITY_NAME, DATA_PLATFORM_INSTANCE_URN);
 
     var result = dataPlatformInstanceFieldResolverProvider.getFieldResolver(resourceSpec);
 
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
new file mode 100644
index 0000000000000..54675045b4413
--- /dev/null
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
@@ -0,0 +1,212 @@
+package com.datahub.authorization.fieldresolverprovider;
+
+import com.datahub.authentication.Authentication;
+import com.datahub.authorization.EntityFieldType;
+import com.datahub.authorization.EntitySpec;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.linkedin.common.UrnArray;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.entity.Aspect;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.identity.GroupMembership;
+import com.linkedin.identity.NativeGroupMembership;
+import com.linkedin.r2.RemoteInvocationException;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.net.URISyntaxException;
+import java.util.Set;
+
+import static com.linkedin.metadata.Constants.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class GroupMembershipFieldResolverProviderTest {
+
+  private static final String CORPGROUP_URN = "urn:li:corpGroup:groupname";
+  private static final String NATIVE_CORPGROUP_URN = "urn:li:corpGroup:nativegroupname";
+  private static final String RESOURCE_URN = "urn:li:dataset:(urn:li:dataPlatform:testPlatform,testDataset,PROD)";
+  private static final EntitySpec RESOURCE_SPEC = new EntitySpec(DATASET_ENTITY_NAME, RESOURCE_URN);
+
+  @Mock
+  private EntityClient entityClientMock;
+  @Mock
+  private Authentication systemAuthenticationMock;
+
+  private GroupMembershipFieldResolverProvider groupMembershipFieldResolverProvider;
+
+  @BeforeMethod
+  public void setup() {
+    MockitoAnnotations.initMocks(this);
+    groupMembershipFieldResolverProvider =
+        new GroupMembershipFieldResolverProvider(entityClientMock, systemAuthenticationMock);
+  }
+
+  @Test
+  public void shouldReturnGroupsMembershipType() {
+    assertEquals(EntityFieldType.GROUP_MEMBERSHIP, groupMembershipFieldResolverProvider.getFieldType());
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenResponseIsNull() throws RemoteInvocationException, URISyntaxException {
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(null);
+
+    var result = groupMembershipFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenResourceDoesNotBelongToAnyGroup()
+      throws RemoteInvocationException, URISyntaxException {
+    var entityResponseMock = mock(EntityResponse.class);
+    when(entityResponseMock.getAspects()).thenReturn(new EnvelopedAspectMap());
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = groupMembershipFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnEmptyFieldValueWhenThereIsAnException() throws RemoteInvocationException, URISyntaxException {
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenThrow(new RemoteInvocationException());
+
+    var result = groupMembershipFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnFieldValueWithOnlyGroupsOfTheResource()
+      throws RemoteInvocationException, URISyntaxException {
+
+    var groupMembership = new GroupMembership().setGroups(
+        new UrnArray(ImmutableList.of(Urn.createFromString(CORPGROUP_URN))));
+    var entityResponseMock = mock(EntityResponse.class);
+    var envelopedAspectMap = new EnvelopedAspectMap();
+    envelopedAspectMap.put(GROUP_MEMBERSHIP_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(groupMembership.data())));
+    when(entityResponseMock.getAspects()).thenReturn(envelopedAspectMap);
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = groupMembershipFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertEquals(Set.of(CORPGROUP_URN), result.getFieldValuesFuture().join().getValues());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnFieldValueWithOnlyNativeGroupsOfTheResource()
+      throws RemoteInvocationException, URISyntaxException {
+
+    var nativeGroupMembership = new NativeGroupMembership().setNativeGroups(
+        new UrnArray(ImmutableList.of(Urn.createFromString(NATIVE_CORPGROUP_URN))));
+    var entityResponseMock = mock(EntityResponse.class);
+    var envelopedAspectMap = new EnvelopedAspectMap();
+    envelopedAspectMap.put(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(nativeGroupMembership.data())));
+    when(entityResponseMock.getAspects()).thenReturn(envelopedAspectMap);
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = groupMembershipFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertEquals(Set.of(NATIVE_CORPGROUP_URN), result.getFieldValuesFuture().join().getValues());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+
+  @Test
+  public void shouldReturnFieldValueWithGroupsAndNativeGroupsOfTheResource()
+      throws RemoteInvocationException, URISyntaxException {
+
+    var groupMembership = new GroupMembership().setGroups(
+        new UrnArray(ImmutableList.of(Urn.createFromString(CORPGROUP_URN))));
+    var nativeGroupMembership = new NativeGroupMembership().setNativeGroups(
+        new UrnArray(ImmutableList.of(Urn.createFromString(NATIVE_CORPGROUP_URN))));
+    var entityResponseMock = mock(EntityResponse.class);
+    var envelopedAspectMap = new EnvelopedAspectMap();
+    envelopedAspectMap.put(GROUP_MEMBERSHIP_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(groupMembership.data())));
+    envelopedAspectMap.put(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME,
+        new EnvelopedAspect().setValue(new Aspect(nativeGroupMembership.data())));
+    when(entityResponseMock.getAspects()).thenReturn(envelopedAspectMap);
+    when(entityClientMock.getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    )).thenReturn(entityResponseMock);
+
+    var result = groupMembershipFieldResolverProvider.getFieldResolver(RESOURCE_SPEC);
+
+    assertEquals(Set.of(CORPGROUP_URN, NATIVE_CORPGROUP_URN), result.getFieldValuesFuture().join().getValues());
+    verify(entityClientMock, times(1)).getV2(
+        eq(DATASET_ENTITY_NAME),
+        any(Urn.class),
+        eq(ImmutableSet.of(GROUP_MEMBERSHIP_ASPECT_NAME, NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)),
+        eq(systemAuthenticationMock)
+    );
+  }
+}
\ No newline at end of file
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java
index bf50a0c7b6473..b90257870a8b2 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java
@@ -2,12 +2,12 @@
 
 import com.datahub.authorization.AuthorizerChain;
 import com.datahub.authorization.DataHubAuthorizer;
-import com.datahub.authorization.DefaultResourceSpecResolver;
+import com.datahub.authorization.DefaultEntitySpecResolver;
 import com.datahub.plugins.PluginConstant;
 import com.datahub.authentication.Authentication;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.datahub.authorization.AuthorizerContext;
-import com.datahub.authorization.ResourceSpecResolver;
+import com.datahub.authorization.EntitySpecResolver;
 import com.datahub.plugins.common.PluginConfig;
 import com.datahub.plugins.common.PluginPermissionManager;
 import com.datahub.plugins.common.PluginType;
@@ -64,7 +64,7 @@ public class AuthorizerChainFactory {
   @Scope("singleton")
   @Nonnull
   protected AuthorizerChain getInstance() {
-    final ResourceSpecResolver resolver = initResolver();
+    final EntitySpecResolver resolver = initResolver();
 
     // Extract + initialize customer authorizers from application configs.
     final List<Authorizer> authorizers = new ArrayList<>(initCustomAuthorizers(resolver));
@@ -79,11 +79,11 @@ protected AuthorizerChain getInstance() {
     return new AuthorizerChain(authorizers, dataHubAuthorizer);
   }
 
-  private ResourceSpecResolver initResolver() {
-    return new DefaultResourceSpecResolver(systemAuthentication, entityClient);
+  private EntitySpecResolver initResolver() {
+    return new DefaultEntitySpecResolver(systemAuthentication, entityClient);
   }
 
-  private List<Authorizer> initCustomAuthorizers(ResourceSpecResolver resolver) {
+  private List<Authorizer> initCustomAuthorizers(EntitySpecResolver resolver) {
     final List<Authorizer> customAuthorizers = new ArrayList<>();
 
     Path pluginBaseDirectory = Paths.get(configurationProvider.getDatahub().getPlugin().getAuth().getPath());
@@ -99,7 +99,7 @@ private List<Authorizer> initCustomAuthorizers(ResourceSpecResolver resolver) {
     return customAuthorizers;
   }
 
-  private void registerAuthorizer(List<Authorizer> customAuthorizers, ResourceSpecResolver resolver, Config config) {
+  private void registerAuthorizer(List<Authorizer> customAuthorizers, EntitySpecResolver resolver, Config config) {
     PluginConfigFactory authorizerPluginPluginConfigFactory = new PluginConfigFactory(config);
     // Load only Authorizer configuration from plugin config factory
     List<PluginConfig> authorizers =
diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java
index ade49c876f168..207c2284e2673 100644
--- a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java
+++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java
@@ -45,8 +45,7 @@
 import io.datahubproject.openapi.util.OpenApiEntitiesUtil;
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.linkedin.metadata.models.EntitySpec;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.google.common.collect.ImmutableList;
 import com.datahub.authorization.AuthUtil;
@@ -377,7 +376,7 @@ public ResponseEntity<S> scroll(@Valid Boolean systemMetadata, @Valid List<Strin
                                     @Valid String scrollId, @Valid List<String> sort, @Valid SortOrder sortOrder, @Valid String query) {
 
         Authentication authentication = AuthenticationContext.getAuthentication();
-        EntitySpec entitySpec = OpenApiEntitiesUtil.responseClassToEntitySpec(_entityRegistry, _respClazz);
+        com.linkedin.metadata.models.EntitySpec entitySpec = OpenApiEntitiesUtil.responseClassToEntitySpec(_entityRegistry, _respClazz);
         checkScrollAuthorized(authentication, entitySpec);
 
         // TODO multi-field sort
@@ -410,12 +409,12 @@ public ResponseEntity<S> scroll(@Valid Boolean systemMetadata, @Valid List<Strin
         return ResponseEntity.of(OpenApiEntitiesUtil.convertToScrollResponse(_scrollRespClazz, result.getScrollId(), entities));
     }
 
-    private void checkScrollAuthorized(Authentication authentication, EntitySpec entitySpec) {
+    private void checkScrollAuthorized(Authentication authentication, com.linkedin.metadata.models.EntitySpec entitySpec) {
         String actorUrnStr = authentication.getActor().toUrnStr();
         DisjunctivePrivilegeGroup orGroup = new DisjunctivePrivilegeGroup(ImmutableList.of(new ConjunctivePrivilegeGroup(
                 ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType()))));
 
-        List<Optional<ResourceSpec>> resourceSpecs = List.of(Optional.of(new ResourceSpec(entitySpec.getName(), "")));
+        List<Optional<EntitySpec>> resourceSpecs = List.of(Optional.of(new EntitySpec(entitySpec.getName(), "")));
         if (_restApiAuthorizationEnabled && !AuthUtil.isAuthorizedForResources(_authorizationChain, actorUrnStr, resourceSpecs, orGroup)) {
             throw new UnauthorizedException(actorUrnStr + " is unauthorized to get entities.");
         }
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java
index 6439e2f31f7b0..898f768cf999a 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java
@@ -8,7 +8,7 @@
 import com.datahub.authorization.AuthorizerChain;
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
@@ -93,8 +93,8 @@ public ResponseEntity<UrnResponseMap> getEntities(
         ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())
     )));
 
-    List<Optional<ResourceSpec>> resourceSpecs = entityUrns.stream()
-        .map(urn -> Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+    List<Optional<EntitySpec>> resourceSpecs = entityUrns.stream()
+        .map(urn -> Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
         .collect(Collectors.toList());
     if (restApiAuthorizationEnabled && !AuthUtil.isAuthorizedForResources(_authorizerChain, actorUrnStr, resourceSpecs, orGroup)) {
       throw new UnauthorizedException(actorUrnStr + " is unauthorized to get entities.");
@@ -175,8 +175,8 @@ public ResponseEntity<List<RollbackRunResultDto>> deleteEntities(
         .map(URLDecoder::decode)
         .map(UrnUtils::getUrn).collect(Collectors.toSet());
 
-    List<Optional<ResourceSpec>> resourceSpecs = entityUrns.stream()
-        .map(urn -> Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+    List<Optional<EntitySpec>> resourceSpecs = entityUrns.stream()
+        .map(urn -> Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
         .collect(Collectors.toList());
     if (restApiAuthorizationEnabled && !AuthUtil.isAuthorizedForResources(_authorizerChain, actorUrnStr, resourceSpecs, orGroup)) {
       UnauthorizedException unauthorizedException = new UnauthorizedException(actorUrnStr + " is unauthorized to delete entities.");
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/relationships/RelationshipsController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/relationships/RelationshipsController.java
index 1e37170f37b3b..4641fed3a8610 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/relationships/RelationshipsController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/relationships/RelationshipsController.java
@@ -8,7 +8,7 @@
 import com.datahub.authorization.AuthorizerChain;
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
@@ -131,8 +131,8 @@ public ResponseEntity<RelatedEntitiesResult> getRelationships(
             // Re-using GET_ENTITY_PRIVILEGE here as it doesn't make sense to split the privileges between these APIs.
         )));
 
-    List<Optional<ResourceSpec>> resourceSpecs =
-        Collections.singletonList(Optional.of(new ResourceSpec(entityUrn.getEntityType(), entityUrn.toString())));
+    List<Optional<EntitySpec>> resourceSpecs =
+        Collections.singletonList(Optional.of(new EntitySpec(entityUrn.getEntityType(), entityUrn.toString())));
     if (restApiAuthorizationEnabled && !AuthUtil.isAuthorizedForResources(_authorizerChain, actorUrnStr, resourceSpecs,
         orGroup)) {
       throw new UnauthorizedException(actorUrnStr + " is unauthorized to get relationships.");
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java
index 5a0ce2e314e1b..fbde9e8072002 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java
@@ -6,7 +6,7 @@
 import com.datahub.authorization.AuthorizerChain;
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
@@ -67,7 +67,7 @@ public ResponseEntity<List<ChangeTransaction>> getTimeline(
     Urn urn = Urn.createFromString(rawUrn);
     Authentication authentication = AuthenticationContext.getAuthentication();
     String actorUrnStr = authentication.getActor().toUrnStr();
-    ResourceSpec resourceSpec = new ResourceSpec(urn.getEntityType(), rawUrn);
+    EntitySpec resourceSpec = new EntitySpec(urn.getEntityType(), rawUrn);
     DisjunctivePrivilegeGroup orGroup = new DisjunctivePrivilegeGroup(
         ImmutableList.of(new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.GET_TIMELINE_PRIVILEGE.getType()))));
     if (restApiAuthorizationEnabled && !AuthUtil.isAuthorized(_authorizerChain, actorUrnStr, Optional.of(resourceSpec), orGroup)) {
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java
index 2b3e84e2df20f..21dc5a4c8a0d6 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java
@@ -5,7 +5,7 @@
 import com.datahub.authorization.AuthUtil;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -27,7 +27,6 @@
 import com.linkedin.metadata.entity.ebean.transactions.AspectsBatchImpl;
 import com.linkedin.metadata.entity.transactions.AspectsBatch;
 import com.linkedin.metadata.entity.validation.ValidationException;
-import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.entity.AspectUtils;
 import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.metadata.utils.metrics.MetricUtils;
@@ -378,11 +377,11 @@ public static GenericAspect convertGenericAspect(@Nonnull io.datahubproject.open
 
   public static boolean authorizeProposals(List<com.linkedin.mxe.MetadataChangeProposal> proposals, EntityService entityService,
       Authorizer authorizer, String actorUrnStr, DisjunctivePrivilegeGroup orGroup) {
-    List<Optional<ResourceSpec>> resourceSpecs = proposals.stream()
+    List<Optional<EntitySpec>> resourceSpecs = proposals.stream()
         .map(proposal -> {
-            EntitySpec entitySpec = entityService.getEntityRegistry().getEntitySpec(proposal.getEntityType());
+            com.linkedin.metadata.models.EntitySpec entitySpec = entityService.getEntityRegistry().getEntitySpec(proposal.getEntityType());
             Urn entityUrn = EntityKeyUtils.getUrnFromProposal(proposal, entitySpec.getKeyAspectSpec());
-            return Optional.of(new ResourceSpec(proposal.getEntityType(), entityUrn.toString()));
+            return Optional.of(new EntitySpec(proposal.getEntityType(), entityUrn.toString()));
         })
         .collect(Collectors.toList());
     return AuthUtil.isAuthorizedForResources(authorizer, actorUrnStr, resourceSpecs, orGroup);
@@ -513,7 +512,7 @@ public static RollbackRunResultDto mapRollbackRunResult(RollbackRunResult rollba
   }
 
   public static UpsertAspectRequest createStatusRemoval(Urn urn, EntityService entityService) {
-    EntitySpec entitySpec = entityService.getEntityRegistry().getEntitySpec(urn.getEntityType());
+    com.linkedin.metadata.models.EntitySpec entitySpec = entityService.getEntityRegistry().getEntitySpec(urn.getEntityType());
     if (entitySpec == null || !entitySpec.getAspectSpecMap().containsKey(STATUS_ASPECT_NAME)) {
       throw new IllegalArgumentException("Entity type is not valid for soft deletes: " + urn.getEntityType());
     }
diff --git a/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java b/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java
index b6bc282f10b65..442ac1b0d287b 100644
--- a/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java
+++ b/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java
@@ -4,7 +4,7 @@
 import com.datahub.authorization.AuthorizationResult;
 import com.datahub.authorization.AuthorizedActors;
 import com.datahub.authorization.AuthorizerContext;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.PluginConstant;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import java.io.BufferedReader;
@@ -74,7 +74,7 @@ public AuthorizationResult authorize(@Nonnull AuthorizationRequest request) {
   }
 
   @Override
-  public AuthorizedActors authorizedActors(String privilege, Optional<ResourceSpec> resourceSpec) {
+  public AuthorizedActors authorizedActors(String privilege, Optional<EntitySpec> resourceSpec) {
     return new AuthorizedActors("ALL", null, null, true, true);
   }
 }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
index 936c8bb67e645..af76af90ce77f 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
@@ -3,7 +3,7 @@
 import com.codahale.metrics.MetricRegistry;
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
@@ -20,7 +20,6 @@
 import com.linkedin.metadata.entity.AspectUtils;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.validation.ValidationException;
-import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.SortCriterion;
 import com.linkedin.metadata.restli.RestliUtil;
@@ -123,7 +122,7 @@ public Task<AnyRecord> get(@Nonnull String urnStr, @QueryParam("aspect") @Option
       Authentication authentication = AuthenticationContext.getAuthentication();
       if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
           && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE),
-          new ResourceSpec(urn.getEntityType(), urn.toString()))) {
+          new EntitySpec(urn.getEntityType(), urn.toString()))) {
         throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to get aspect for " + urn);
       }
       final VersionedAspect aspect = _entityService.getVersionedAspect(urn, aspectName, version);
@@ -154,7 +153,7 @@ public Task<GetTimeseriesAspectValuesResponse> getTimeseriesAspectValues(
       Authentication authentication = AuthenticationContext.getAuthentication();
       if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
           && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.GET_TIMESERIES_ASPECT_PRIVILEGE),
-          new ResourceSpec(urn.getEntityType(), urn.toString()))) {
+          new EntitySpec(urn.getEntityType(), urn.toString()))) {
         throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to get timeseries aspect for " + urn);
       }
       GetTimeseriesAspectValuesResponse response = new GetTimeseriesAspectValuesResponse();
@@ -193,11 +192,11 @@ public Task<String> ingestProposal(
     }
 
     Authentication authentication = AuthenticationContext.getAuthentication();
-    EntitySpec entitySpec = _entityService.getEntityRegistry().getEntitySpec(metadataChangeProposal.getEntityType());
+    com.linkedin.metadata.models.EntitySpec entitySpec = _entityService.getEntityRegistry().getEntitySpec(metadataChangeProposal.getEntityType());
     Urn urn = EntityKeyUtils.getUrnFromProposal(metadataChangeProposal, entitySpec.getKeyAspectSpec());
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE),
-        new ResourceSpec(urn.getEntityType(), urn.toString()))) {
+        new EntitySpec(urn.getEntityType(), urn.toString()))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to modify entity " + urn);
     }
     String actorUrnStr = authentication.getActor().toUrnStr();
@@ -249,7 +248,7 @@ public Task<Integer> getCount(@ActionParam(PARAM_ASPECT) @Nonnull String aspectN
       Authentication authentication = AuthenticationContext.getAuthentication();
       if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
           && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.GET_COUNTS_PRIVILEGE),
-          (ResourceSpec) null)) {
+          (EntitySpec) null)) {
         throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to get aspect counts.");
       }
       return _entityService.getCountAspect(aspectName, urnLike);
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java
index 3ff22fb767676..9bab846d1bdcc 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java
@@ -4,7 +4,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
@@ -123,9 +123,9 @@ public Task<RollbackResponse> rollback(@ActionParam("runId") @Nonnull String run
         List<AspectRowSummary> aspectRowsToDelete;
         aspectRowsToDelete = _systemMetadataService.findByRunId(runId, doHardDelete, 0, ESUtils.MAX_RESULT_SIZE);
         Set<String> urns = aspectRowsToDelete.stream().collect(Collectors.groupingBy(AspectRowSummary::getUrn)).keySet();
-        List<java.util.Optional<ResourceSpec>> resourceSpecs = urns.stream()
+        List<java.util.Optional<EntitySpec>> resourceSpecs = urns.stream()
             .map(UrnUtils::getUrn)
-            .map(urn -> java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+            .map(urn -> java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
             .collect(Collectors.toList());
         Authentication auth = AuthenticationContext.getAuthentication();
         if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
index f6dedfb9a07c6..3ee98b3244718 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java
@@ -3,7 +3,7 @@
 import com.codahale.metrics.MetricRegistry;
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.AuditStamp;
@@ -173,7 +173,7 @@ public Task<AnyRecord> get(@Nonnull String urnStr,
     final Urn urn = Urn.createFromString(urnStr);
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), new ResourceSpec(urn.getEntityType(), urnStr))) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), new EntitySpec(urn.getEntityType(), urnStr))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity " + urn);
     }
@@ -198,8 +198,8 @@ public Task<Map<String, AnyRecord>> batchGet(@Nonnull Set<String> urnStrs,
     for (final String urnStr : urnStrs) {
       urns.add(Urn.createFromString(urnStr));
     }
-    List<java.util.Optional<ResourceSpec>> resourceSpecs = urns.stream()
-        .map(urn -> java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+    List<java.util.Optional<EntitySpec>> resourceSpecs = urns.stream()
+        .map(urn -> java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
         .collect(Collectors.toList());
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
@@ -242,7 +242,7 @@ public Task<Void> ingest(@ActionParam(PARAM_ENTITY) @Nonnull Entity entity,
     final Urn urn = com.datahub.util.ModelUtils.getUrnFromSnapshotUnion(entity.getValue());
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE),
-        new ResourceSpec(urn.getEntityType(), urn.toString()))) {
+        new EntitySpec(urn.getEntityType(), urn.toString()))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to edit entity " + urn);
     }
@@ -273,10 +273,10 @@ public Task<Void> batchIngest(@ActionParam(PARAM_ENTITIES) @Nonnull Entity[] ent
 
     Authentication authentication = AuthenticationContext.getAuthentication();
     String actorUrnStr = authentication.getActor().toUrnStr();
-    List<java.util.Optional<ResourceSpec>> resourceSpecs = Arrays.stream(entities)
+    List<java.util.Optional<EntitySpec>> resourceSpecs = Arrays.stream(entities)
         .map(Entity::getValue)
         .map(com.datahub.util.ModelUtils::getUrnFromSnapshotUnion)
-        .map(urn -> java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+        .map(urn -> java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
         .collect(Collectors.toList());
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE), resourceSpecs)) {
@@ -322,7 +322,7 @@ public Task<SearchResult> search(@ActionParam(PARAM_ENTITY) @Nonnull String enti
       @Optional @Nullable @ActionParam(PARAM_SEARCH_FLAGS) SearchFlags searchFlags) {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -347,7 +347,7 @@ public Task<SearchResult> searchAcrossEntities(@ActionParam(PARAM_ENTITIES) @Opt
       @ActionParam(PARAM_COUNT) int count, @ActionParam(PARAM_SEARCH_FLAGS) @Optional SearchFlags searchFlags) {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -391,7 +391,7 @@ public Task<LineageSearchResult> searchAcrossLineage(@ActionParam(PARAM_URN) @No
       @Optional @Nullable @ActionParam(PARAM_SEARCH_FLAGS) SearchFlags searchFlags) throws URISyntaxException {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -443,7 +443,7 @@ public Task<ListResult> list(@ActionParam(PARAM_ENTITY) @Nonnull String entityNa
 
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -462,7 +462,7 @@ public Task<AutoCompleteResult> autocomplete(@ActionParam(PARAM_ENTITY) @Nonnull
 
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -479,7 +479,7 @@ public Task<BrowseResult> browse(@ActionParam(PARAM_ENTITY) @Nonnull String enti
 
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -497,7 +497,7 @@ public Task<StringArray> getBrowsePaths(
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE),
-        new ResourceSpec(urn.getEntityType(), urn.toString()))) {
+        new EntitySpec(urn.getEntityType(), urn.toString()))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity: " + urn);
     }
@@ -546,9 +546,9 @@ public Task<RollbackResponse> deleteEntities(@ActionParam("registryId") @Optiona
       log.info("found {} rows to delete...", stringifyRowCount(aspectRowsToDelete.size()));
       response.setAspectsAffected(aspectRowsToDelete.size());
       Set<String> urns = aspectRowsToDelete.stream().collect(Collectors.groupingBy(AspectRowSummary::getUrn)).keySet();
-      List<java.util.Optional<ResourceSpec>> resourceSpecs = urns.stream()
+      List<java.util.Optional<EntitySpec>> resourceSpecs = urns.stream()
           .map(UrnUtils::getUrn)
-          .map(urn -> java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+          .map(urn -> java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
           .collect(Collectors.toList());
       Authentication auth = AuthenticationContext.getAuthentication();
       if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
@@ -590,7 +590,7 @@ public Task<DeleteEntityResponse> deleteEntity(@ActionParam(PARAM_URN) @Nonnull
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE),
-        Collections.singletonList(java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString()))))) {
+        Collections.singletonList(java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString()))))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to delete entity: " + urnStr);
     }
@@ -638,7 +638,7 @@ private Long deleteTimeseriesAspects(@Nonnull Urn urn, @Nullable Long startTimeM
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE),
-        new ResourceSpec(urn.getEntityType(), urn.toString()))) {
+        new EntitySpec(urn.getEntityType(), urn.toString()))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to delete entity " + urn);
     }
@@ -678,7 +678,7 @@ public Task<DeleteReferencesResponse> deleteReferencesTo(@ActionParam(PARAM_URN)
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE),
-        new ResourceSpec(urn.getEntityType(), urnStr))) {
+        new EntitySpec(urn.getEntityType(), urnStr))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to delete entity " + urnStr);
     }
@@ -695,7 +695,7 @@ public Task<DeleteReferencesResponse> deleteReferencesTo(@ActionParam(PARAM_URN)
   public Task<Void> setWriteable(@ActionParam(PARAM_VALUE) @Optional("true") @Nonnull Boolean value) {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SET_WRITEABLE_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SET_WRITEABLE_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to enable and disable write mode.");
     }
@@ -712,7 +712,7 @@ public Task<Void> setWriteable(@ActionParam(PARAM_VALUE) @Optional("true") @Nonn
   public Task<Long> getTotalEntityCount(@ActionParam(PARAM_ENTITY) @Nonnull String entityName) {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_COUNTS_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_COUNTS_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity counts.");
     }
@@ -725,7 +725,7 @@ public Task<Long> getTotalEntityCount(@ActionParam(PARAM_ENTITY) @Nonnull String
   public Task<LongMap> batchGetTotalEntityCount(@ActionParam(PARAM_ENTITIES) @Nonnull String[] entityNames) {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_COUNTS_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_COUNTS_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity counts.");
     }
@@ -739,7 +739,7 @@ public Task<ListUrnsResult> listUrns(@ActionParam(PARAM_ENTITY) @Nonnull String
       @ActionParam(PARAM_START) int start, @ActionParam(PARAM_COUNT) int count) throws URISyntaxException {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -757,10 +757,10 @@ public Task<String> applyRetention(@ActionParam(PARAM_START) @Optional @Nullable
                                      @ActionParam(PARAM_URN) @Optional @Nullable String urn
                                      ) {
     Authentication auth = AuthenticationContext.getAuthentication();
-    ResourceSpec resourceSpec = null;
+    EntitySpec resourceSpec = null;
     if (StringUtils.isNotBlank(urn)) {
       Urn resource = UrnUtils.getUrn(urn);
-      resourceSpec = new ResourceSpec(resource.getEntityType(), resource.toString());
+      resourceSpec = new EntitySpec(resource.getEntityType(), resource.toString());
     }
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.APPLY_RETENTION_PRIVILEGE), resourceSpec)) {
@@ -781,7 +781,7 @@ public Task<SearchResult> filter(@ActionParam(PARAM_ENTITY) @Nonnull String enti
 
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.SEARCH_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to search.");
     }
@@ -799,7 +799,7 @@ public Task<Boolean> exists(@ActionParam(PARAM_URN) @Nonnull String urnStr) thro
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE),
-        new ResourceSpec(urn.getEntityType(), urnStr))) {
+        new EntitySpec(urn.getEntityType(), urnStr))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized get entity: " + urnStr);
     }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java
index 7efb93c0f50e6..0c3e93273b863 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java
@@ -4,7 +4,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.EntityResponse;
@@ -68,7 +68,7 @@ public Task<EntityResponse> get(@Nonnull String urnStr,
     final Urn urn = Urn.createFromString(urnStr);
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), new ResourceSpec(urn.getEntityType(), urnStr))) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), new EntitySpec(urn.getEntityType(), urnStr))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity " + urn);
     }
@@ -96,8 +96,8 @@ public Task<Map<Urn, EntityResponse>> batchGet(@Nonnull Set<String> urnStrs,
       urns.add(Urn.createFromString(urnStr));
     }
     Authentication auth = AuthenticationContext.getAuthentication();
-    List<java.util.Optional<ResourceSpec>> resourceSpecs = urns.stream()
-        .map(urn -> java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+    List<java.util.Optional<EntitySpec>> resourceSpecs = urns.stream()
+        .map(urn -> java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
         .collect(Collectors.toList());
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), resourceSpecs)) {
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java
index fd5c3507b5408..05b7e6b3ff24b 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java
@@ -4,7 +4,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.VersionedUrn;
 import com.linkedin.common.urn.Urn;
@@ -65,9 +65,9 @@ public Task<Map<Urn, EntityResponse>> batchGetVersioned(
       @QueryParam(PARAM_ENTITY_TYPE) @Nonnull String entityType,
       @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) {
     Authentication auth = AuthenticationContext.getAuthentication();
-    List<java.util.Optional<ResourceSpec>> resourceSpecs = versionedUrnStrs.stream()
+    List<java.util.Optional<EntitySpec>> resourceSpecs = versionedUrnStrs.stream()
         .map(versionedUrn -> UrnUtils.getUrn(versionedUrn.getUrn()))
-        .map(urn -> java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString())))
+        .map(urn -> java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString())))
         .collect(Collectors.toList());
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE), resourceSpecs)) {
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java
index 313d16333f9e9..4a8e74c89039a 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java
@@ -4,7 +4,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.EntityRelationship;
 import com.linkedin.common.EntityRelationshipArray;
@@ -107,7 +107,7 @@ public Task<EntityRelationships> get(@QueryParam("urn") @Nonnull String rawUrn,
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE),
-        Collections.singletonList(java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString()))))) {
+        Collections.singletonList(java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString()))))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity lineage: " + rawUrn);
     }
@@ -142,7 +142,7 @@ public UpdateResponse delete(@QueryParam("urn") @Nonnull String rawUrn) throws E
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE),
-        Collections.singletonList(java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString()))))) {
+        Collections.singletonList(java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString()))))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to delete entity: " + rawUrn);
     }
@@ -162,7 +162,7 @@ public Task<EntityLineageResult> getLineage(@ActionParam(PARAM_URN) @Nonnull Str
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE),
-        Collections.singletonList(java.util.Optional.of(new ResourceSpec(urn.getEntityType(), urn.toString()))))) {
+        Collections.singletonList(java.util.Optional.of(new EntitySpec(urn.getEntityType(), urn.toString()))))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to get entity lineage: " + urnStr);
     }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java
index 188e5ae18ee8f..12586b66495a9 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java
@@ -2,7 +2,7 @@
 
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
@@ -37,10 +37,10 @@ public static String restoreIndices(
       @Nonnull EntityService entityService
   ) {
     Authentication authentication = AuthenticationContext.getAuthentication();
-    ResourceSpec resourceSpec = null;
+    EntitySpec resourceSpec = null;
     if (StringUtils.isNotBlank(urn)) {
       Urn resource = UrnUtils.getUrn(urn);
-      resourceSpec = new ResourceSpec(resource.getEntityType(), resource.toString());
+      resourceSpec = new EntitySpec(resource.getEntityType(), resource.toString());
     }
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(authentication, authorizer, ImmutableList.of(PoliciesConfig.RESTORE_INDICES_PRIVILEGE),
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/platform/PlatformResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/platform/PlatformResource.java
index f36841bb4abae..a8018074497c4 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/platform/PlatformResource.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/platform/PlatformResource.java
@@ -3,7 +3,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.entity.Entity;
 import com.linkedin.metadata.authorization.PoliciesConfig;
@@ -54,7 +54,7 @@ public Task<Void> producePlatformEvent(
       @ActionParam("event") @Nonnull PlatformEvent event) {
     Authentication auth = AuthenticationContext.getAuthentication();
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.PRODUCE_PLATFORM_EVENT_PRIVILEGE), (ResourceSpec) null)) {
+        && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.PRODUCE_PLATFORM_EVENT_PRIVILEGE), (EntitySpec) null)) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to produce platform events.");
     }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java
index 5c3b90a84aec1..9949556c99b81 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliUtils.java
@@ -4,7 +4,7 @@
 import com.datahub.authorization.AuthUtil;
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.metadata.authorization.PoliciesConfig;
@@ -82,13 +82,13 @@ public static RestLiServiceException invalidArgumentsException(@Nullable String
   }
 
   public static boolean isAuthorized(@Nonnull Authentication authentication, @Nonnull Authorizer authorizer,
-      @Nonnull final List<PoliciesConfig.Privilege> privileges, @Nonnull final List<java.util.Optional<ResourceSpec>> resources) {
+      @Nonnull final List<PoliciesConfig.Privilege> privileges, @Nonnull final List<java.util.Optional<EntitySpec>> resources) {
     DisjunctivePrivilegeGroup orGroup = convertPrivilegeGroup(privileges);
     return AuthUtil.isAuthorizedForResources(authorizer, authentication.getActor().toUrnStr(), resources, orGroup);
   }
 
   public static boolean isAuthorized(@Nonnull Authentication authentication, @Nonnull Authorizer authorizer,
-      @Nonnull final List<PoliciesConfig.Privilege> privileges, @Nullable final ResourceSpec resource) {
+      @Nonnull final List<PoliciesConfig.Privilege> privileges, @Nullable final EntitySpec resource) {
     DisjunctivePrivilegeGroup orGroup = convertPrivilegeGroup(privileges);
     return AuthUtil.isAuthorized(authorizer, authentication.getActor().toUrnStr(), java.util.Optional.ofNullable(resource), orGroup);
   }
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java
index be70cf9c494ef..02d413301f3b4 100644
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java
+++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java
@@ -4,7 +4,7 @@
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.plugins.auth.authorization.Authorizer;
-import com.datahub.authorization.ResourceSpec;
+import com.datahub.authorization.EntitySpec;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -125,7 +125,7 @@ public Task<Void> batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregat
     return RestliUtil.toTask(() -> {
       Authentication auth = AuthenticationContext.getAuthentication();
       if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
-          && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE), (ResourceSpec) null)) {
+          && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE), (EntitySpec) null)) {
         throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
             "User is unauthorized to edit entities.");
       }
@@ -323,7 +323,7 @@ public Task<UsageQueryResult> query(@ActionParam(PARAM_RESOURCE) @Nonnull String
       Urn resourceUrn = UrnUtils.getUrn(resource);
       if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
           && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE),
-          new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString()))) {
+          new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString()))) {
         throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
             "User is unauthorized to query usage.");
       }
@@ -383,7 +383,7 @@ public Task<UsageQueryResult> queryRange(@ActionParam(PARAM_RESOURCE) @Nonnull S
     Urn resourceUrn = UrnUtils.getUrn(resource);
     if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV))
         && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE),
-            new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString()))) {
+            new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString()))) {
       throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED,
           "User is unauthorized to query usage.");
     }

From d04d25bf428aa442b08a4011fcac81b3d1526a86 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Thu, 12 Oct 2023 15:50:20 -0400
Subject: [PATCH 041/259] smoke test(): Query plus filter search test (#8993)

---
 .../e2e/search/query_and_filter_search.js     | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js

diff --git a/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js b/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js
new file mode 100644
index 0000000000000..4637310b86496
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/search/query_and_filter_search.js
@@ -0,0 +1,57 @@
+describe("auto-complete dropdown, filter plus query search test", () => {
+
+  const platformQuerySearch = (query,test_id,active_filter) => {
+    cy.visit("/");
+    cy.get("input[data-testid=search-input]").type(query);
+    cy.get(`[data-testid="quick-filter-urn:li:dataPlatform:${test_id}"]`).click();
+    cy.focused().type("{enter}").wait(3000);
+    cy.url().should(
+      "include",
+      `?filter_platform___false___EQUAL___0=urn%3Ali%3AdataPlatform%3A${test_id}`
+    );
+    cy.get('[data-testid="search-input"]').should("have.value", query);
+    cy.get(`[data-testid="active-filter-${active_filter}"]`).should("be.visible");
+    cy.contains("of 0 results").should("not.exist");
+    cy.contains(/of [0-9]+ results/);
+  }
+
+  const entityQuerySearch = (query,test_id,active_filter) => {
+    cy.visit("/");
+    cy.get("input[data-testid=search-input]").type(query);
+    cy.get(`[data-testid="quick-filter-${test_id}"]`).click();
+    cy.focused().type("{enter}").wait(3000);
+    cy.url().should(
+      "include",
+      `?filter__entityType___false___EQUAL___0=${test_id}`
+    );
+    cy.get('[data-testid="search-input"]').should("have.value", query);
+    cy.get(`[data-testid="active-filter-${active_filter}"]`).should("be.visible");
+    cy.contains("of 0 results").should("not.exist");
+    cy.contains(/of [0-9]+ results/);
+  }
+
+  it("verify the 'filter by' section + query (result in search page with query applied + filter applied)", () => {
+    // Platform query plus filter test
+    cy.loginWithCredentials();
+    // Airflow
+    platformQuerySearch ("cypress","airflow","Airflow");
+    // BigQuery
+    platformQuerySearch ("cypress","bigquery","BigQuery");
+    // dbt
+    platformQuerySearch ("cypress","dbt","dbt");
+    // Hive 
+    platformQuerySearch ("cypress","hive","Hive");
+
+    // Entity type query plus filter test
+    // Datasets
+    entityQuerySearch ("cypress","DATASET","Datasets");
+    // Dashboards
+    entityQuerySearch ("cypress","DASHBOARD","Dashboards");
+    // Pipelines
+    entityQuerySearch ("cypress","DATA_FLOW","Pipelines");
+    // Domains
+    entityQuerySearch ("Marketing","DOMAIN","Domains");
+    // Glossary Terms
+    entityQuerySearch ("cypress","GLOSSARY_TERM","Glossary Terms");
+  });
+});
\ No newline at end of file

From a8f0080c08b5c816f0dae9d3bef07ea00220541e Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Fri, 13 Oct 2023 00:14:45 +0200
Subject: [PATCH 042/259] feat(ingest/teradata): Teradata source (#8977)

---
 .../docs/sources/teradata/teradata_pre.md     |  28 +++
 .../docs/sources/teradata/teradata_recipe.yml |  17 ++
 metadata-ingestion/setup.py                   |   3 +
 .../datahub/ingestion/source/sql/teradata.py  | 228 ++++++++++++++++++
 .../testing/check_sql_parser_result.py        |   5 +-
 .../src/datahub/utilities/sqlglot_lineage.py  |   5 +
 .../test_teradata_default_normalization.json  |  38 +++
 .../unit/sql_parsing/test_sqlglot_lineage.py  |  42 ++++
 8 files changed, 365 insertions(+), 1 deletion(-)
 create mode 100644 metadata-ingestion/docs/sources/teradata/teradata_pre.md
 create mode 100644 metadata-ingestion/docs/sources/teradata/teradata_recipe.yml
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json

diff --git a/metadata-ingestion/docs/sources/teradata/teradata_pre.md b/metadata-ingestion/docs/sources/teradata/teradata_pre.md
new file mode 100644
index 0000000000000..eb59caa29eb52
--- /dev/null
+++ b/metadata-ingestion/docs/sources/teradata/teradata_pre.md
@@ -0,0 +1,28 @@
+### Prerequisites
+1. Create a user which has access to the database you want to ingest.
+    ```sql
+    CREATE USER datahub FROM <database> AS PASSWORD = <password> PERM = 20000000;
+    ```
+2. Create a user with the following privileges:
+    ```sql
+    GRANT SELECT ON dbc.columns TO datahub;
+    GRANT SELECT ON dbc.databases TO datahub;
+    GRANT SELECT ON dbc.tables TO datahub;
+    GRANT SELECT ON DBC.All_RI_ChildrenV TO datahub;
+    GRANT SELECT ON DBC.ColumnsV TO datahub;
+    GRANT SELECT ON DBC.IndicesV TO datahub;
+    GRANT SELECT ON dbc.TableTextV TO datahub;
+    GRANT SELECT ON dbc.TablesV TO datahub;
+    GRANT SELECT ON dbc.dbqlogtbl TO datahub; -- if lineage or usage extraction is enabled
+    ```
+   
+    If you want to run profiling, you need to grant select permission on all the tables you want to profile.
+
+3. If linege or usage extraction is enabled, please, check if query logging is enabled and it is set to size which
+will fit for your queries (the default query text size Teradata captures is max 200 chars)
+   An example how you can set it for all users:
+    ```sql
+    REPLACE QUERY LOGGING LIMIT SQLTEXT=2000 ON ALL;
+    ```
+   See more here about query logging:
+      [https://docs.teradata.com/r/Teradata-VantageCloud-Lake/Database-Reference/Database-Administration/Tracking-Query-Behavior-with-Database-Query-Logging-Operational-DBAs]()
diff --git a/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml b/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml
new file mode 100644
index 0000000000000..8cf07ba4c3a01
--- /dev/null
+++ b/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml
@@ -0,0 +1,17 @@
+pipeline_name: my-teradata-ingestion-pipeline
+source:
+  type: teradata
+  config:
+    host_port: "myteradatainstance.teradata.com:1025"
+    #platform_instance: "myteradatainstance"
+    username: myuser
+    password: mypassword
+    #database_pattern:
+    #  allow:
+    #    - "demo_user"
+    #  ignoreCase: true
+    include_table_lineage: true
+    include_usage_statistics: true
+    stateful_ingestion:
+      enabled: true
+sink:
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 61e7b684682a4..3ea9a2ea61d74 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -373,6 +373,7 @@
     # FIXME: I don't think tableau uses sqllineage anymore so we should be able
     # to remove that dependency.
     "tableau": {"tableauserverclient>=0.17.0"} | sqllineage_lib | sqlglot_lib,
+    "teradata": sql_common | {"teradatasqlalchemy>=17.20.0.0"},
     "trino": sql_common | trino,
     "starburst-trino-usage": sql_common | usage_common | trino,
     "nifi": {"requests", "packaging", "requests-gssapi"},
@@ -499,6 +500,7 @@
             "s3",
             "snowflake",
             "tableau",
+            "teradata",
             "trino",
             "hive",
             "starburst-trino-usage",
@@ -597,6 +599,7 @@
         "tableau = datahub.ingestion.source.tableau:TableauSource",
         "openapi = datahub.ingestion.source.openapi:OpenApiSource",
         "metabase = datahub.ingestion.source.metabase:MetabaseSource",
+        "teradata = datahub.ingestion.source.sql.teradata:TeradataSource",
         "trino = datahub.ingestion.source.sql.trino:TrinoSource",
         "starburst-trino-usage = datahub.ingestion.source.usage.starburst_trino_usage:TrinoUsageSource",
         "nifi = datahub.ingestion.source.nifi:NifiSource",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
new file mode 100644
index 0000000000000..dd11cd840bed9
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -0,0 +1,228 @@
+import logging
+from dataclasses import dataclass
+from typing import Iterable, Optional, Set, Union
+
+# This import verifies that the dependencies are available.
+import teradatasqlalchemy  # noqa: F401
+import teradatasqlalchemy.types as custom_types
+from pydantic.fields import Field
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+
+from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.time_window_config import BaseTimeWindowConfig
+from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SourceCapability,
+    SupportStatus,
+    capability,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.source.sql.sql_common import SqlWorkUnit, register_custom_type
+from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
+from datahub.ingestion.source.sql.two_tier_sql_source import (
+    TwoTierSQLAlchemyConfig,
+    TwoTierSQLAlchemySource,
+)
+from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
+from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
+from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
+from datahub.metadata.com.linkedin.pegasus2avro.schema import (
+    BytesTypeClass,
+    TimeTypeClass,
+)
+from datahub.utilities.sqlglot_lineage import SchemaResolver, sqlglot_lineage
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+register_custom_type(custom_types.JSON, BytesTypeClass)
+register_custom_type(custom_types.INTERVAL_DAY, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_DAY_TO_SECOND, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_DAY_TO_MINUTE, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_DAY_TO_HOUR, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_SECOND, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_MINUTE, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_MINUTE_TO_SECOND, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_HOUR, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_HOUR_TO_MINUTE, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_HOUR_TO_SECOND, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_MONTH, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_YEAR, TimeTypeClass)
+register_custom_type(custom_types.INTERVAL_YEAR_TO_MONTH, TimeTypeClass)
+register_custom_type(custom_types.MBB, BytesTypeClass)
+register_custom_type(custom_types.MBR, BytesTypeClass)
+register_custom_type(custom_types.GEOMETRY, BytesTypeClass)
+register_custom_type(custom_types.TDUDT, BytesTypeClass)
+register_custom_type(custom_types.XML, BytesTypeClass)
+
+
+@dataclass
+class TeradataReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowReport):
+    num_queries_parsed: int = 0
+    num_table_parse_failures: int = 0
+
+
+class BaseTeradataConfig(TwoTierSQLAlchemyConfig):
+    scheme = Field(default="teradatasql", description="database scheme")
+
+
+class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
+    database_pattern = Field(
+        default=AllowDenyPattern(deny=["dbc"]),
+        description="Regex patterns for databases to filter in ingestion.",
+    )
+    include_table_lineage = Field(
+        default=False,
+        description="Whether to include table lineage in the ingestion. "
+        "This requires to have the table lineage feature enabled.",
+    )
+
+    usage: BaseUsageConfig = Field(
+        description="The usage config to use when generating usage statistics",
+        default=BaseUsageConfig(),
+    )
+
+    use_schema_resolver: bool = Field(
+        default=True,
+        description="Read SchemaMetadata aspects from DataHub to aid in SQL parsing. Turn off only for testing.",
+        hidden_from_docs=True,
+    )
+
+    default_db: Optional[str] = Field(
+        default=None,
+        description="The default database to use for unqualified table names",
+    )
+
+    include_usage_statistics: bool = Field(
+        default=False,
+        description="Generate usage statistic.",
+    )
+
+
+@platform_name("Teradata")
+@config_class(TeradataConfig)
+@support_status(SupportStatus.TESTING)
+@capability(SourceCapability.DOMAINS, "Enabled by default")
+@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.DELETION_DETECTION, "Optionally enabled via configuration")
+@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
+@capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration")
+@capability(SourceCapability.LINEAGE_FINE, "Optionally enabled via configuration")
+@capability(SourceCapability.USAGE_STATS, "Optionally enabled via configuration")
+class TeradataSource(TwoTierSQLAlchemySource):
+    """
+    This plugin extracts the following:
+
+    - Metadata for databases, schemas, views, and tables
+    - Column types associated with each table
+    - Table, row, and column statistics via optional SQL profiling
+    """
+
+    config: TeradataConfig
+
+    LINEAGE_QUERY: str = """SELECT ProcID, UserName as "user", StartTime AT TIME ZONE 'GMT' as "timestamp", DefaultDatabase as default_database, QueryText as query
+     FROM "DBC".DBQLogTbl
+     where ErrorCode = 0
+     and QueryText like 'create table demo_user.test_lineage%'
+     and "timestamp" >= TIMESTAMP '{start_time}'
+     and "timestamp" < TIMESTAMP '{end_time}'
+     """
+    urns: Optional[Set[str]]
+
+    def __init__(self, config: TeradataConfig, ctx: PipelineContext):
+        super().__init__(config, ctx, "teradata")
+
+        self.report: TeradataReport = TeradataReport()
+        self.graph: Optional[DataHubGraph] = ctx.graph
+
+        if self.graph:
+            if self.config.use_schema_resolver:
+                self.schema_resolver = (
+                    self.graph.initialize_schema_resolver_from_datahub(
+                        platform=self.platform,
+                        platform_instance=self.config.platform_instance,
+                        env=self.config.env,
+                    )
+                )
+                self.urns = self.schema_resolver.get_urns()
+            else:
+                self.schema_resolver = self.graph._make_schema_resolver(
+                    platform=self.platform,
+                    platform_instance=self.config.platform_instance,
+                    env=self.config.env,
+                )
+                self.urns = None
+        else:
+            self.schema_resolver = SchemaResolver(
+                platform=self.platform,
+                platform_instance=self.config.platform_instance,
+                graph=None,
+                env=self.config.env,
+            )
+            self.urns = None
+
+        self.builder: SqlParsingBuilder = SqlParsingBuilder(
+            usage_config=self.config.usage
+            if self.config.include_usage_statistics
+            else None,
+            generate_lineage=self.config.include_table_lineage,
+            generate_usage_statistics=self.config.include_usage_statistics,
+            generate_operations=self.config.usage.include_operational_stats,
+        )
+
+    @classmethod
+    def create(cls, config_dict, ctx):
+        config = TeradataConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    def get_audit_log_mcps(self) -> Iterable[MetadataWorkUnit]:
+        engine = self.get_metadata_engine()
+        for entry in engine.execute(
+            self.LINEAGE_QUERY.format(
+                start_time=self.config.start_time, end_time=self.config.end_time
+            )
+        ):
+            self.report.num_queries_parsed += 1
+            if self.report.num_queries_parsed % 1000 == 0:
+                logger.info(f"Parsed {self.report.num_queries_parsed} queries")
+
+            result = sqlglot_lineage(
+                sql=entry.query,
+                schema_resolver=self.schema_resolver,
+                default_db=None,
+                default_schema=entry.default_database
+                if entry.default_database
+                else self.config.default_db,
+            )
+            if result.debug_info.table_error:
+                logger.debug(
+                    f"Error parsing table lineage, {result.debug_info.table_error}"
+                )
+                self.report.num_table_parse_failures += 1
+                continue
+
+            yield from self.builder.process_sql_parsing_result(
+                result,
+                query=entry.query,
+                query_timestamp=entry.timestamp,
+                user=f"urn:li:corpuser:{entry.user}",
+                include_urns=self.urns,
+            )
+
+    def get_metadata_engine(self) -> Engine:
+        url = self.config.get_sql_alchemy_url()
+        logger.debug(f"sql_alchemy_url={url}")
+        return create_engine(url, **self.config.options)
+
+    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
+        yield from super().get_workunits_internal()
+        if self.config.include_table_lineage or self.config.include_usage_statistics:
+            self.report.report_ingestion_stage_start("audit log extraction")
+            yield from self.get_audit_log_mcps()
+            yield from self.builder.gen_workunits()
diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
index 8516a7054a9cd..b3b1331db768b 100644
--- a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
+++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
@@ -70,11 +70,14 @@ def assert_sql_result(
     sql: str,
     *,
     dialect: str,
+    platform_instance: Optional[str] = None,
     expected_file: pathlib.Path,
     schemas: Optional[Dict[str, SchemaInfo]] = None,
     **kwargs: Any,
 ) -> None:
-    schema_resolver = SchemaResolver(platform=dialect)
+    schema_resolver = SchemaResolver(
+        platform=dialect, platform_instance=platform_instance
+    )
     if schemas:
         for urn, schema in schemas.items():
             schema_resolver.add_raw_schema_info(urn, schema)
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 349eb40a5e865..c830ec8c02fd4 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -482,6 +482,11 @@ def _column_level_lineage(  # noqa: C901
         # Our snowflake source lowercases column identifiers, so we are forced
         # to do fuzzy (case-insensitive) resolution instead of exact resolution.
         "snowflake",
+        # Teradata column names are case-insensitive.
+        # A name, even when enclosed in double quotation marks, is not case sensitive. For example, CUSTOMER and Customer are the same.
+        # See more below:
+        # https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0ejgx4895bofnn14rlguktfx5r3.htm
+        "teradata",
     }
 
     sqlglot_db_schema = sqlglot.MappingSchema(
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
new file mode 100644
index 0000000000000..b0351a7e07ad2
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
@@ -0,0 +1,38 @@
+{
+    "query_type": "CREATE",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.pima_patient_diagnoses,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.pima_patient_features,PROD)"
+    ],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)",
+                "column": "PatientId",
+                "native_column_type": "INTEGER()"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.pima_patient_diagnoses,PROD)",
+                    "column": "PatientId"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)",
+                "column": "BMI",
+                "native_column_type": "FLOAT()"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.pima_patient_features,PROD)",
+                    "column": "BMI"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index bb6e5f1581754..059add8db67e4 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -630,3 +630,45 @@ def test_snowflake_column_cast():
 
 
 # TODO: Add a test for setting platform_instance or env
+
+
+def test_teradata_default_normalization():
+    assert_sql_result(
+        """
+create table demo_user.test_lineage2 as
+ (
+    select
+        ppd.PatientId,
+        ppf.bmi
+    from
+        demo_user.pima_patient_features ppf
+    join demo_user.pima_patient_diagnoses ppd on
+        ppd.PatientId = ppf.PatientId
+ ) with data;
+""",
+        dialect="teradata",
+        default_schema="dbc",
+        platform_instance="myteradata",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.pima_patient_diagnoses,PROD)": {
+                "HasDiabetes": "INTEGER()",
+                "PatientId": "INTEGER()",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.pima_patient_features,PROD)": {
+                "Age": "INTEGER()",
+                "BMI": "FLOAT()",
+                "BloodP": "INTEGER()",
+                "DiPedFunc": "FLOAT()",
+                "NumTimesPrg": "INTEGER()",
+                "PatientId": "INTEGER()",
+                "PlGlcConc": "INTEGER()",
+                "SkinThick": "INTEGER()",
+                "TwoHourSerIns": "INTEGER()",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)": {
+                "BMI": "FLOAT()",
+                "PatientId": "INTEGER()",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_teradata_default_normalization.json",
+    )

From 71c9bd3a495c1f3663d2268088f04d56dd8c37c9 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Fri, 13 Oct 2023 11:48:22 +0530
Subject: [PATCH 043/259] ci(ingest): update base requirements (#8995)

---
 .../base-requirements.txt                     | 398 +++++++++---------
 1 file changed, 205 insertions(+), 193 deletions(-)

diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt
index 82d9a93a9a2c3..eb082d50b3020 100644
--- a/docker/datahub-ingestion-base/base-requirements.txt
+++ b/docker/datahub-ingestion-base/base-requirements.txt
@@ -2,62 +2,58 @@
 # pyspark==3.0.3
 # pydeequ==1.0.1
 
-acryl-datahub-classify==0.0.6
-acryl-iceberg-legacy==0.0.4
-acryl-PyHive==0.6.13
-aenum==3.1.12
-aiohttp==3.8.4
+acryl-datahub-classify==0.0.8
+acryl-PyHive==0.6.14
+acryl-sqlglot==18.5.2.dev45
+aenum==3.1.15
+aiohttp==3.8.6
 aiosignal==1.3.1
-alembic==1.11.1
+alembic==1.12.0
 altair==4.2.0
-anyio==3.7.0
-apache-airflow==2.6.1
-apache-airflow-providers-common-sql==1.5.1
-apache-airflow-providers-ftp==3.4.1
-apache-airflow-providers-http==4.4.1
-apache-airflow-providers-imap==3.2.1
-apache-airflow-providers-sqlite==3.4.1
-apispec==5.2.2
+anyio==3.7.1
+apache-airflow==2.7.2
+apache-airflow-providers-common-sql==1.7.2
+apache-airflow-providers-ftp==3.5.2
+apache-airflow-providers-http==4.5.2
+apache-airflow-providers-imap==3.3.2
+apache-airflow-providers-sqlite==3.4.3
+apispec==6.3.0
 appdirs==1.4.4
 appnope==0.1.3
-argcomplete==3.0.8
-argon2-cffi==21.3.0
+argcomplete==3.1.2
+argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
 asgiref==3.7.2
 asn1crypto==1.5.1
-asttokens==2.2.1
-async-timeout==4.0.2
+asttokens==2.4.0
+async-timeout==4.0.3
 asynch==0.2.2
 attrs==23.1.0
 avro==1.10.2
-avro-gen3==0.7.10
-azure-core==1.26.4
-azure-identity==1.10.0
-azure-storage-blob==12.16.0
-azure-storage-file-datalake==12.11.0
-Babel==2.12.1
+avro-gen3==0.7.11
+Babel==2.13.0
 backcall==0.2.0
 backoff==2.2.1
 beautifulsoup4==4.12.2
-bleach==6.0.0
-blinker==1.6.2
-blis==0.7.9
-boto3==1.26.142
-botocore==1.29.142
+bleach==6.1.0
+blinker==1.6.3
+blis==0.7.11
+boto3==1.28.62
+botocore==1.31.62
 bowler==0.9.0
-bracex==2.3.post1
+bracex==2.4
 cached-property==1.5.2
 cachelib==0.9.0
 cachetools==5.3.1
-catalogue==2.0.8
-cattrs==22.2.0
-certifi==2023.5.7
-cffi==1.15.1
-chardet==5.1.0
-charset-normalizer==2.1.1
+catalogue==2.0.10
+cattrs==23.1.2
+certifi==2023.7.22
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.0
 ciso8601==2.3.0
-click==8.1.3
-click-default-group==1.2.2
+click==8.1.7
+click-default-group==1.2.4
 click-spinner==0.1.10
 clickclick==20.10.2
 clickhouse-cityhash==1.0.2.4
@@ -66,205 +62,217 @@ clickhouse-sqlalchemy==0.2.4
 cloudpickle==2.2.1
 colorama==0.4.6
 colorlog==4.8.0
-confection==0.0.4
+comm==0.1.4
+confection==0.1.3
 ConfigUpdater==3.1.1
 confluent-kafka==1.8.2
 connexion==2.14.2
 cron-descriptor==1.4.0
-croniter==1.3.15
-cryptography==37.0.4
+croniter==2.0.1
+cryptography==41.0.4
 cx-Oracle==8.3.0
-cymem==2.0.7
-dask==2023.5.1
-databricks-cli==0.17.7
+cymem==2.0.8
+dask==2023.9.3
+databricks-cli==0.18.0
 databricks-dbapi==0.6.0
-databricks-sdk==0.1.8
-debugpy==1.6.7
+databricks-sdk==0.10.0
+debugpy==1.8.0
 decorator==5.1.1
 defusedxml==0.7.1
-deltalake==0.9.0
+deltalake==0.11.0
 Deprecated==1.2.14
-dill==0.3.6
-dnspython==2.3.0
-docker==6.1.2
+dill==0.3.7
+dnspython==2.4.2
+docker==6.1.3
 docutils==0.20.1
 ecdsa==0.18.0
 elasticsearch==7.13.4
 email-validator==1.3.1
 entrypoints==0.4
 et-xmlfile==1.1.0
-exceptiongroup==1.1.1
-executing==1.2.0
-expandvars==0.9.0
-fastapi==0.95.2
-fastavro==1.7.4
-fastjsonschema==2.17.1
-feast==0.29.0
-filelock==3.12.0
+exceptiongroup==1.1.3
+executing==2.0.0
+expandvars==0.11.0
+fastapi==0.103.2
+fastavro==1.8.4
+fastjsonschema==2.18.1
+feast==0.31.1
+filelock==3.12.4
 fissix==21.11.13
 Flask==2.2.5
 flatdict==4.0.1
-frozenlist==1.3.3
-fsspec==2023.5.0
+frozenlist==1.4.0
+fsspec==2023.9.2
 future==0.18.3
-GeoAlchemy2==0.13.3
+GeoAlchemy2==0.14.1
 gitdb==4.0.10
-GitPython==3.1.31
-google-api-core==2.11.0
-google-auth==2.19.0
-google-cloud-appengine-logging==1.3.0
+GitPython==3.1.37
+google-api-core==2.12.0
+google-auth==2.23.3
+google-cloud-appengine-logging==1.3.2
 google-cloud-audit-log==0.2.5
-google-cloud-bigquery==3.10.0
-google-cloud-bigquery-storage==2.19.1
-google-cloud-core==2.3.2
+google-cloud-bigquery==3.12.0
+google-cloud-core==2.3.3
 google-cloud-datacatalog-lineage==0.2.2
 google-cloud-logging==3.5.0
 google-crc32c==1.5.0
-google-resumable-media==2.5.0
-googleapis-common-protos==1.59.0
+google-re2==1.1
+google-resumable-media==2.6.0
+googleapis-common-protos==1.60.0
 gql==3.4.1
 graphql-core==3.2.3
 graphviz==0.20.1
 great-expectations==0.15.50
-greenlet==2.0.2
+greenlet==3.0.0
 grpc-google-iam-v1==0.12.6
-grpcio==1.54.2
-grpcio-reflection==1.54.2
-grpcio-status==1.54.2
-grpcio-tools==1.54.2
-gssapi==1.8.2
-gunicorn==20.1.0
+grpcio==1.59.0
+grpcio-reflection==1.59.0
+grpcio-status==1.59.0
+grpcio-tools==1.59.0
+gssapi==1.8.3
+gunicorn==21.2.0
 h11==0.14.0
-hmsclient==0.1.1
-httpcore==0.17.2
-httptools==0.5.0
-httpx==0.24.1
+httpcore==0.18.0
+httptools==0.6.0
+httpx==0.25.0
 humanfriendly==10.0
 idna==3.4
-ijson==3.2.0.post0
-importlib-metadata==6.6.0
-importlib-resources==5.12.0
+ijson==3.2.3
+importlib-metadata==6.8.0
+importlib-resources==6.1.0
 inflection==0.5.1
 ipaddress==1.0.23
 ipykernel==6.17.1
-ipython==8.13.2
+ipython==8.16.1
 ipython-genutils==0.2.0
-ipywidgets==8.0.6
+ipywidgets==8.1.1
 iso3166==2.1.1
 isodate==0.6.1
 itsdangerous==2.1.2
-jedi==0.18.2
+jedi==0.19.1
 Jinja2==3.1.2
 jmespath==1.0.1
 JPype1==1.4.1
-jsonlines==3.1.0
-jsonpatch==1.32
-jsonpointer==2.3
+jsonlines==4.0.0
+jsonpatch==1.33
+jsonpointer==2.4
 jsonref==1.1.0
-jsonschema==4.17.3
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
 jupyter-server==1.24.0
 jupyter_client==7.4.9
 jupyter_core==4.12.0
 jupyterlab-pygments==0.2.2
-jupyterlab-widgets==3.0.7
+jupyterlab-widgets==3.0.9
 langcodes==3.3.0
 lark==1.1.4
 lazy-object-proxy==1.9.0
 leb128==1.0.5
-limits==3.5.0
+limits==3.6.0
 linear-tsv==1.1.0
 linkify-it-py==2.0.2
 lkml==1.3.1
 locket==1.0.0
 lockfile==0.12.2
 looker-sdk==23.0.0
-lxml==4.9.2
+lxml==4.9.3
 lz4==4.3.2
 makefun==1.15.1
 Mako==1.2.4
-Markdown==3.4.3
-markdown-it-py==2.2.0
-MarkupSafe==2.1.2
-marshmallow==3.19.0
-marshmallow-enum==1.5.1
+Markdown==3.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
 marshmallow-oneofschema==3.0.1
 marshmallow-sqlalchemy==0.26.1
 matplotlib-inline==0.1.6
-mdit-py-plugins==0.3.5
+mdit-py-plugins==0.4.0
 mdurl==0.1.2
-mistune==2.0.5
+mistune==3.0.2
 mixpanel==4.10.0
-mmh3==4.0.0
-more-itertools==9.1.0
+mlflow-skinny==2.7.1
+mmh3==4.0.1
+mmhash3==3.0.1
+more-itertools==10.1.0
 moreorless==0.4.0
-moto==4.1.10
-msal==1.16.0
-msal-extensions==1.0.0
+moto==4.2.5
+msal==1.22.0
 multidict==6.0.4
-murmurhash==1.0.9
-mypy==1.3.0
+murmurhash==1.0.10
+mypy==1.6.0
 mypy-extensions==1.0.0
 nbclassic==1.0.0
 nbclient==0.6.3
-nbconvert==7.4.0
-nbformat==5.8.0
-nest-asyncio==1.5.6
+nbconvert==7.9.2
+nbformat==5.9.1
+nest-asyncio==1.5.8
 networkx==3.1
-notebook==6.5.4
+notebook==6.5.6
 notebook_shim==0.2.3
-numpy==1.24.3
+numpy==1.26.0
 oauthlib==3.2.2
 okta==1.7.0
+openlineage-airflow==1.2.0
+openlineage-integration-common==1.2.0
+openlineage-python==1.2.0
+openlineage_sql==1.2.0
 openpyxl==3.1.2
+opentelemetry-api==1.20.0
+opentelemetry-exporter-otlp==1.20.0
+opentelemetry-exporter-otlp-proto-common==1.20.0
+opentelemetry-exporter-otlp-proto-grpc==1.20.0
+opentelemetry-exporter-otlp-proto-http==1.20.0
+opentelemetry-proto==1.20.0
+opentelemetry-sdk==1.20.0
+opentelemetry-semantic-conventions==0.41b0
 ordered-set==4.1.0
 oscrypto==1.3.0
-packaging==23.1
+packaging==23.2
 pandas==1.5.3
 pandavro==1.5.2
 pandocfilters==1.5.0
-parse==1.19.0
+parse==1.19.1
 parso==0.8.3
-partd==1.4.0
-pathspec==0.9.0
-pathy==0.10.1
+partd==1.4.1
+pathspec==0.11.2
+pathy==0.10.2
 pendulum==2.1.2
 pexpect==4.8.0
 phonenumbers==8.13.0
 pickleshare==0.7.5
-platformdirs==3.5.1
-pluggy==1.0.0
-portalocker==2.7.0
-preshed==3.0.8
+platformdirs==3.11.0
+pluggy==1.3.0
+preshed==3.0.9
 prison==0.2.1
 progressbar2==4.2.0
-prometheus-client==0.17.0
-prompt-toolkit==3.0.38
-proto-plus==1.22.2
-protobuf==4.23.2
+prometheus-client==0.17.1
+prompt-toolkit==3.0.39
+proto-plus==1.22.3
+protobuf==4.24.4
 psutil==5.9.5
-psycopg2-binary==2.9.6
+psycopg2-binary==2.9.9
 ptyprocess==0.7.0
 pure-eval==0.2.2
 pure-sasl==0.6.2
-py-partiql-parser==0.3.0
-pyarrow==8.0.0
+py-partiql-parser==0.3.7
+pyarrow==11.0.0
 pyasn1==0.5.0
 pyasn1-modules==0.3.0
 pyathena==2.4.1
 pycountry==22.3.5
 pycparser==2.21
-pycryptodome==3.18.0
-pycryptodomex==3.18.0
-pydantic==1.10.8
-pydash==7.0.3
+pycryptodome==3.19.0
+pycryptodomex==3.19.0
+pydantic==1.10.13
+pydash==7.0.6
 pydruid==0.6.5
-Pygments==2.15.1
-pymongo==4.3.3
-PyMySQL==1.0.3
-pyOpenSSL==22.0.0
+Pygments==2.16.1
+pyiceberg==0.4.0
+pymongo==4.5.0
+PyMySQL==1.1.0
+pyOpenSSL==23.2.0
 pyparsing==3.0.9
-pyrsistent==0.19.3
-pyspnego==0.9.0
+pyspnego==0.10.2
 python-daemon==3.0.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
@@ -272,111 +280,115 @@ python-jose==3.3.0
 python-ldap==3.4.3
 python-nvd3==0.15.0
 python-slugify==8.0.1
-python-stdnum==1.18
-python-tds==1.12.0
-python-utils==3.6.0
+python-stdnum==1.19
+python-tds==1.13.0
+python-utils==3.8.1
 python3-openid==3.2.0
-pytz==2023.3
+pytz==2023.3.post1
 pytzdata==2020.1
-PyYAML==6.0
-pyzmq==25.1.0
+PyYAML==6.0.1
+pyzmq==24.0.1
 ratelimiter==1.2.0.post0
 redash-toolbelt==0.1.9
-redshift-connector==2.0.910
-regex==2023.5.5
-requests==2.28.2
+redshift-connector==2.0.914
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
 requests-file==1.5.1
 requests-gssapi==1.2.3
 requests-ntlm==1.2.0
 requests-toolbelt==0.10.1
-responses==0.23.1
-retrying==1.3.4
+responses==0.23.3
 rfc3339-validator==0.1.4
 rfc3986==2.0.0
-rich==13.3.5
-rich_argparse==1.1.0
+rich==13.6.0
+rich-argparse==1.3.0
+rpds-py==0.10.6
 rsa==4.9
 ruamel.yaml==0.17.17
-s3transfer==0.6.1
-sasl3==0.2.11
-schwifty==2023.3.0
-scipy==1.10.1
+ruamel.yaml.clib==0.2.8
+s3transfer==0.7.0
+schwifty==2023.9.0
+scipy==1.11.3
 scramp==1.4.4
 Send2Trash==1.8.2
-setproctitle==1.3.2
-simple-salesforce==1.12.4
+sentry-sdk==1.32.0
+setproctitle==1.3.3
+simple-salesforce==1.12.5
 six==1.16.0
-smart-open==6.3.0
-smmap==5.0.0
+smart-open==6.4.0
+smmap==5.0.1
 sniffio==1.3.0
-snowflake-connector-python==2.9.0
-snowflake-sqlalchemy==1.4.7
-soupsieve==2.4.1
+snowflake-connector-python==3.2.1
+snowflake-sqlalchemy==1.5.0
+sortedcontainers==2.4.0
+soupsieve==2.5
 spacy==3.4.3
 spacy-legacy==3.0.12
-spacy-loggers==1.0.4
+spacy-loggers==1.0.5
 sql-metadata==2.2.2
-SQLAlchemy==1.4.41
-sqlalchemy-bigquery==1.6.1
+SQLAlchemy==1.4.44
+sqlalchemy-bigquery==1.8.0
 SQLAlchemy-JSONField==1.0.1.post0
 sqlalchemy-pytds==0.3.5
 sqlalchemy-redshift==0.8.14
 SQLAlchemy-Utils==0.41.1
-sqlalchemy2-stubs==0.0.2a34
-sqllineage==1.3.6
-sqlparse==0.4.3
-srsly==2.4.6
-stack-data==0.6.2
+sqlalchemy2-stubs==0.0.2a35
+sqllineage==1.3.8
+sqlparse==0.4.4
+srsly==2.4.8
+stack-data==0.6.3
 starlette==0.27.0
+strictyaml==1.7.3
 tableauserverclient==0.25
 tableschema==1.20.2
 tabulate==0.9.0
 tabulator==1.53.5
-tenacity==8.2.2
+tenacity==8.2.3
 termcolor==2.3.0
 terminado==0.17.1
 text-unidecode==1.3
-thinc==8.1.10
-thrift==0.16.0
+thinc==8.1.12
+thrift==0.13.0
 thrift-sasl==0.4.3
 tinycss2==1.2.1
 toml==0.10.2
 tomli==2.0.1
+tomlkit==0.12.1
 toolz==0.12.0
-tornado==6.3.2
-tqdm==4.65.0
+tornado==6.3.3
+tqdm==4.66.1
 traitlets==5.2.1.post0
-trino==0.324.0
+trino==0.327.0
 typeguard==2.13.3
 typer==0.7.0
-types-PyYAML==6.0.12.10
+types-PyYAML==6.0.12.12
 typing-inspect==0.9.0
-typing_extensions==4.5.0
-tzlocal==5.0.1
+typing_extensions==4.8.0
+tzlocal==5.1
 uc-micro-py==1.0.2
-ujson==5.7.0
+ujson==5.8.0
 unicodecsv==0.14.1
-urllib3==1.26.16
-uvicorn==0.22.0
+urllib3==1.26.17
+uvicorn==0.23.2
 uvloop==0.17.0
-vertica-python==1.3.2
-vertica-sqlalchemy-dialect==0.0.1
+vertica-python==1.3.5
+vertica-sqlalchemy-dialect==0.0.8
 vininfo==1.7.0
 volatile==2.1.0
 wasabi==0.10.1
-watchfiles==0.19.0
-wcmatch==8.4.1
-wcwidth==0.2.6
+watchfiles==0.20.0
+wcmatch==8.5
+wcwidth==0.2.8
 webencodings==0.5.1
-websocket-client==1.5.2
+websocket-client==1.6.4
 websockets==11.0.3
 Werkzeug==2.2.3
-widgetsnbextension==4.0.7
+widgetsnbextension==4.0.9
 wrapt==1.15.0
-WTForms==3.0.1
+WTForms==3.1.0
 xlrd==2.0.1
 xmltodict==0.13.0
 yarl==1.9.2
 zeep==4.2.1
-zipp==3.15.0
-zstd==1.5.5.1
+zstd==1.5.5.1
\ No newline at end of file

From c02cbb31e2896f9b596bc329af2e86459057b37e Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Fri, 13 Oct 2023 17:52:53 +0530
Subject: [PATCH 044/259] docs(Acryl DataHub): release notes for 0.2.12 (#9006)

---
 docs-website/sidebars.js                      |  1 +
 .../managed-datahub/release-notes/v_0_2_11.md |  2 +-
 .../managed-datahub/release-notes/v_0_2_12.md | 30 +++++++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 docs/managed-datahub/release-notes/v_0_2_12.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 21b3a1d3fe4d3..4fa73c995157a 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -608,6 +608,7 @@ module.exports = {
         },
         {
           "Managed DataHub Release History": [
+            "docs/managed-datahub/release-notes/v_0_2_12",
             "docs/managed-datahub/release-notes/v_0_2_11",
             "docs/managed-datahub/release-notes/v_0_2_10",
             "docs/managed-datahub/release-notes/v_0_2_9",
diff --git a/docs/managed-datahub/release-notes/v_0_2_11.md b/docs/managed-datahub/release-notes/v_0_2_11.md
index 1f42090848712..c99d10201e097 100644
--- a/docs/managed-datahub/release-notes/v_0_2_11.md
+++ b/docs/managed-datahub/release-notes/v_0_2_11.md
@@ -7,7 +7,7 @@ Release Availability Date
 
 Recommended CLI/SDK
 ---
-- `v0.11.0` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.10.5.5
+- `v0.11.0` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.11.0
 - [Deprecation] In LDAP ingestor, the manager_pagination_enabled changed to general pagination_enabled
 
 If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better.
diff --git a/docs/managed-datahub/release-notes/v_0_2_12.md b/docs/managed-datahub/release-notes/v_0_2_12.md
new file mode 100644
index 0000000000000..b13f471d9bf63
--- /dev/null
+++ b/docs/managed-datahub/release-notes/v_0_2_12.md
@@ -0,0 +1,30 @@
+# v0.2.12
+---
+
+Release Availability Date
+---
+13-Oct-2023
+
+Recommended CLI/SDK
+---
+- `v0.11.0.4` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.11.0.4
+- [breaking] Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
+- [breaking] Removed `urn:li:corpuser:datahub` owner for the `Measure`, `Dimension` and `Temporal` tags emitted by Looker and LookML source connectors.
+- [breaking] The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7.
+- [breaking] Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include `pip install 'acryl-datahub-airflow-plugin[plugin-v2]'`. To continue using the v1 plugin, set the `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN` environment variable to `true`.
+- [breaking] The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
+This is currently enabled by default to preserve compatibility, but will be disabled by default and then removed in the future.
+If stateful ingestion is enabled, simply setting `include_metastore: false` will perform all required cleanup.
+Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
+`datahub delete --platform databricks --soft` and then reingesting with `include_metastore: false`.
+
+
+If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better.
+
+
+## Release Changelog
+---
+- Since `v0.2.11` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/75252a3d9f6a576904be5a0790d644b9ae2df6ac...10a190470e8c932b6d34cba49de7dbcba687a088 have been pulled in.
+
+## Some notable features in this SaaS release
+- Nested Domains available in this release

From 6bc742535379f6cc4558daa67b6561d549d6e607 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Fri, 13 Oct 2023 12:36:18 -0400
Subject: [PATCH 045/259] feat(cli/datacontract): Add data quality assertion
 support (#8968)

Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
Co-authored-by: Aseem Bansal <asmbansal2@gmail.com>
---
 .../api/entities/datacontract/assertion.py    |   7 +
 .../datacontract/assertion_operator.py        | 162 ++++++++++++++++++
 .../datacontract/data_quality_assertion.py    |  60 ++++---
 .../api/entities/datacontract/datacontract.py |   2 +-
 .../datacontract/freshness_assertion.py       |  54 +++---
 .../entities/datacontract/schema_assertion.py |  17 +-
 .../api/entities/datacontract/__init__.py     |   0
 .../test_data_quality_assertion.py            |  55 ++++++
 8 files changed, 292 insertions(+), 65 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/assertion.py
 create mode 100644 metadata-ingestion/src/datahub/api/entities/datacontract/assertion_operator.py
 create mode 100644 metadata-ingestion/tests/unit/api/entities/datacontract/__init__.py
 create mode 100644 metadata-ingestion/tests/unit/api/entities/datacontract/test_data_quality_assertion.py

diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/assertion.py
new file mode 100644
index 0000000000000..c45d4ddc92458
--- /dev/null
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/assertion.py
@@ -0,0 +1,7 @@
+from typing import Optional
+
+from datahub.configuration import ConfigModel
+
+
+class BaseAssertion(ConfigModel):
+    description: Optional[str] = None
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/assertion_operator.py b/metadata-ingestion/src/datahub/api/entities/datacontract/assertion_operator.py
new file mode 100644
index 0000000000000..a41b0f7aafd9f
--- /dev/null
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/assertion_operator.py
@@ -0,0 +1,162 @@
+from typing import Optional, Union
+
+from typing_extensions import Literal, Protocol
+
+from datahub.configuration import ConfigModel
+from datahub.metadata.schema_classes import (
+    AssertionStdOperatorClass,
+    AssertionStdParameterClass,
+    AssertionStdParametersClass,
+    AssertionStdParameterTypeClass,
+)
+
+
+class Operator(Protocol):
+    """Specification for an assertion operator.
+
+    This class exists only for documentation (not used in typing checking).
+    """
+
+    operator: str
+
+    def id(self) -> str:
+        ...
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        ...
+
+
+def _generate_assertion_std_parameter(
+    value: Union[str, int, float]
+) -> AssertionStdParameterClass:
+    if isinstance(value, str):
+        return AssertionStdParameterClass(
+            value=value, type=AssertionStdParameterTypeClass.STRING
+        )
+    elif isinstance(value, (int, float)):
+        return AssertionStdParameterClass(
+            value=str(value), type=AssertionStdParameterTypeClass.NUMBER
+        )
+    else:
+        raise ValueError(
+            f"Unsupported assertion parameter {value} of type {type(value)}"
+        )
+
+
+Param = Union[str, int, float]
+
+
+def _generate_assertion_std_parameters(
+    value: Optional[Param] = None,
+    min_value: Optional[Param] = None,
+    max_value: Optional[Param] = None,
+) -> AssertionStdParametersClass:
+    return AssertionStdParametersClass(
+        value=_generate_assertion_std_parameter(value) if value else None,
+        minValue=_generate_assertion_std_parameter(min_value) if min_value else None,
+        maxValue=_generate_assertion_std_parameter(max_value) if max_value else None,
+    )
+
+
+class EqualToOperator(ConfigModel):
+    type: Literal["equal_to"]
+    value: Union[str, int, float]
+
+    operator: str = AssertionStdOperatorClass.EQUAL_TO
+
+    def id(self) -> str:
+        return f"{self.type}-{self.value}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters(value=self.value)
+
+
+class BetweenOperator(ConfigModel):
+    type: Literal["between"]
+    min: Union[int, float]
+    max: Union[int, float]
+
+    operator: str = AssertionStdOperatorClass.BETWEEN
+
+    def id(self) -> str:
+        return f"{self.type}-{self.min}-{self.max}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters(
+            min_value=self.min, max_value=self.max
+        )
+
+
+class LessThanOperator(ConfigModel):
+    type: Literal["less_than"]
+    value: Union[int, float]
+
+    operator: str = AssertionStdOperatorClass.LESS_THAN
+
+    def id(self) -> str:
+        return f"{self.type}-{self.value}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters(value=self.value)
+
+
+class GreaterThanOperator(ConfigModel):
+    type: Literal["greater_than"]
+    value: Union[int, float]
+
+    operator: str = AssertionStdOperatorClass.GREATER_THAN
+
+    def id(self) -> str:
+        return f"{self.type}-{self.value}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters(value=self.value)
+
+
+class LessThanOrEqualToOperator(ConfigModel):
+    type: Literal["less_than_or_equal_to"]
+    value: Union[int, float]
+
+    operator: str = AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO
+
+    def id(self) -> str:
+        return f"{self.type}-{self.value}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters(value=self.value)
+
+
+class GreaterThanOrEqualToOperator(ConfigModel):
+    type: Literal["greater_than_or_equal_to"]
+    value: Union[int, float]
+
+    operator: str = AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO
+
+    def id(self) -> str:
+        return f"{self.type}-{self.value}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters(value=self.value)
+
+
+class NotNullOperator(ConfigModel):
+    type: Literal["not_null"]
+
+    operator: str = AssertionStdOperatorClass.NOT_NULL
+
+    def id(self) -> str:
+        return f"{self.type}"
+
+    def generate_parameters(self) -> AssertionStdParametersClass:
+        return _generate_assertion_std_parameters()
+
+
+Operators = Union[
+    EqualToOperator,
+    BetweenOperator,
+    LessThanOperator,
+    LessThanOrEqualToOperator,
+    GreaterThanOperator,
+    GreaterThanOrEqualToOperator,
+    NotNullOperator,
+]
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py
index a665e95e93c43..6a3944ba36baf 100644
--- a/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/data_quality_assertion.py
@@ -4,6 +4,8 @@
 from typing_extensions import Literal
 
 import datahub.emitter.mce_builder as builder
+from datahub.api.entities.datacontract.assertion import BaseAssertion
+from datahub.api.entities.datacontract.assertion_operator import Operators
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -14,12 +16,15 @@
     AssertionStdParametersClass,
     AssertionStdParameterTypeClass,
     AssertionTypeClass,
+    AssertionValueChangeTypeClass,
     DatasetAssertionInfoClass,
     DatasetAssertionScopeClass,
+    SqlAssertionInfoClass,
+    SqlAssertionTypeClass,
 )
 
 
-class IdConfigMixin(ConfigModel):
+class IdConfigMixin(BaseAssertion):
     id_raw: Optional[str] = pydantic.Field(
         default=None,
         alias="id",
@@ -30,25 +35,32 @@ def generate_default_id(self) -> str:
         raise NotImplementedError
 
 
-class CustomSQLAssertion(IdConfigMixin, ConfigModel):
+class CustomSQLAssertion(IdConfigMixin, BaseAssertion):
     type: Literal["custom_sql"]
-
     sql: str
+    operator: Operators = pydantic.Field(discriminator="type")
 
-    def generate_dataset_assertion_info(
-        self, entity_urn: str
-    ) -> DatasetAssertionInfoClass:
-        return DatasetAssertionInfoClass(
-            dataset=entity_urn,
-            scope=DatasetAssertionScopeClass.UNKNOWN,
-            fields=[],
-            operator=AssertionStdOperatorClass._NATIVE_,
-            aggregation=AssertionStdAggregationClass._NATIVE_,
-            logic=self.sql,
+    def generate_default_id(self) -> str:
+        return f"{self.type}-{self.sql}-{self.operator.id()}"
+
+    def generate_assertion_info(self, entity_urn: str) -> AssertionInfoClass:
+        sql_assertion_info = SqlAssertionInfoClass(
+            entity=entity_urn,
+            statement=self.sql,
+            operator=self.operator.operator,
+            parameters=self.operator.generate_parameters(),
+            # TODO: Support other types of assertions
+            type=SqlAssertionTypeClass.METRIC,
+            changeType=AssertionValueChangeTypeClass.ABSOLUTE,
+        )
+        return AssertionInfoClass(
+            type=AssertionTypeClass.SQL,
+            sqlAssertion=sql_assertion_info,
+            description=self.description,
         )
 
 
-class ColumnUniqueAssertion(IdConfigMixin, ConfigModel):
+class ColumnUniqueAssertion(IdConfigMixin, BaseAssertion):
     type: Literal["unique"]
 
     # TODO: support multiple columns?
@@ -57,10 +69,8 @@ class ColumnUniqueAssertion(IdConfigMixin, ConfigModel):
     def generate_default_id(self) -> str:
         return f"{self.type}-{self.column}"
 
-    def generate_dataset_assertion_info(
-        self, entity_urn: str
-    ) -> DatasetAssertionInfoClass:
-        return DatasetAssertionInfoClass(
+    def generate_assertion_info(self, entity_urn: str) -> AssertionInfoClass:
+        dataset_assertion_info = DatasetAssertionInfoClass(
             dataset=entity_urn,
             scope=DatasetAssertionScopeClass.DATASET_COLUMN,
             fields=[builder.make_schema_field_urn(entity_urn, self.column)],
@@ -72,6 +82,11 @@ def generate_dataset_assertion_info(
                 )
             ),
         )
+        return AssertionInfoClass(
+            type=AssertionTypeClass.DATASET,
+            datasetAssertion=dataset_assertion_info,
+            description=self.description,
+        )
 
 
 class DataQualityAssertion(ConfigModel):
@@ -92,16 +107,9 @@ def id(self) -> str:
     def generate_mcp(
         self, assertion_urn: str, entity_urn: str
     ) -> List[MetadataChangeProposalWrapper]:
-        dataset_assertion_info = self.__root__.generate_dataset_assertion_info(
-            entity_urn
-        )
-
         return [
             MetadataChangeProposalWrapper(
                 entityUrn=assertion_urn,
-                aspect=AssertionInfoClass(
-                    type=AssertionTypeClass.DATASET,
-                    datasetAssertion=dataset_assertion_info,
-                ),
+                aspect=self.__root__.generate_assertion_info(entity_urn),
             )
         ]
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py b/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
index 2df446623a9d6..f3c6be55e5fea 100644
--- a/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
@@ -54,7 +54,7 @@ class DataContract(ConfigModel):
     freshness: Optional[FreshnessAssertion] = pydantic.Field(default=None)
 
     # TODO: Add a validator to ensure that ids are unique
-    data_quality: Optional[List[DataQualityAssertion]] = None
+    data_quality: Optional[List[DataQualityAssertion]] = pydantic.Field(default=None)
 
     _original_yaml_dict: Optional[dict] = None
 
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py
index ee8fa1181e614..71741d76b22fc 100644
--- a/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/freshness_assertion.py
@@ -6,6 +6,7 @@
 import pydantic
 from typing_extensions import Literal
 
+from datahub.api.entities.datacontract.assertion import BaseAssertion
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -21,7 +22,7 @@
 )
 
 
-class CronFreshnessAssertion(ConfigModel):
+class CronFreshnessAssertion(BaseAssertion):
     type: Literal["cron"]
 
     cron: str = pydantic.Field(
@@ -32,12 +33,30 @@ class CronFreshnessAssertion(ConfigModel):
         description="The timezone to use for the cron schedule. Defaults to UTC.",
     )
 
+    def generate_freshness_assertion_schedule(self) -> FreshnessAssertionScheduleClass:
+        return FreshnessAssertionScheduleClass(
+            type=FreshnessAssertionScheduleTypeClass.CRON,
+            cron=FreshnessCronScheduleClass(
+                cron=self.cron,
+                timezone=self.timezone,
+            ),
+        )
+
 
-class FixedIntervalFreshnessAssertion(ConfigModel):
+class FixedIntervalFreshnessAssertion(BaseAssertion):
     type: Literal["interval"]
 
     interval: timedelta
 
+    def generate_freshness_assertion_schedule(self) -> FreshnessAssertionScheduleClass:
+        return FreshnessAssertionScheduleClass(
+            type=FreshnessAssertionScheduleTypeClass.FIXED_INTERVAL,
+            fixedInterval=FixedIntervalScheduleClass(
+                unit=CalendarIntervalClass.SECOND,
+                multiple=int(self.interval.total_seconds()),
+            ),
+        )
+
 
 class FreshnessAssertion(ConfigModel):
     __root__: Union[
@@ -51,36 +70,13 @@ def id(self):
     def generate_mcp(
         self, assertion_urn: str, entity_urn: str
     ) -> List[MetadataChangeProposalWrapper]:
-        freshness = self.__root__
-
-        if isinstance(freshness, CronFreshnessAssertion):
-            schedule = FreshnessAssertionScheduleClass(
-                type=FreshnessAssertionScheduleTypeClass.CRON,
-                cron=FreshnessCronScheduleClass(
-                    cron=freshness.cron,
-                    timezone=freshness.timezone,
-                ),
-            )
-        elif isinstance(freshness, FixedIntervalFreshnessAssertion):
-            schedule = FreshnessAssertionScheduleClass(
-                type=FreshnessAssertionScheduleTypeClass.FIXED_INTERVAL,
-                fixedInterval=FixedIntervalScheduleClass(
-                    unit=CalendarIntervalClass.SECOND,
-                    multiple=int(freshness.interval.total_seconds()),
-                ),
-            )
-        else:
-            raise ValueError(f"Unknown freshness type {freshness}")
-
-        assertionInfo = AssertionInfoClass(
+        aspect = AssertionInfoClass(
             type=AssertionTypeClass.FRESHNESS,
             freshnessAssertion=FreshnessAssertionInfoClass(
                 entity=entity_urn,
                 type=FreshnessAssertionTypeClass.DATASET_CHANGE,
-                schedule=schedule,
+                schedule=self.__root__.generate_freshness_assertion_schedule(),
             ),
+            description=self.__root__.description,
         )
-
-        return [
-            MetadataChangeProposalWrapper(entityUrn=assertion_urn, aspect=assertionInfo)
-        ]
+        return [MetadataChangeProposalWrapper(entityUrn=assertion_urn, aspect=aspect)]
diff --git a/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py b/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py
index b5b592e01f58f..b62f94e0592fc 100644
--- a/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py
+++ b/metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py
@@ -6,6 +6,7 @@
 import pydantic
 from typing_extensions import Literal
 
+from datahub.api.entities.datacontract.assertion import BaseAssertion
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.extractor.json_schema_util import get_schema_metadata
@@ -19,7 +20,7 @@
 )
 
 
-class JsonSchemaContract(ConfigModel):
+class JsonSchemaContract(BaseAssertion):
     type: Literal["json-schema"]
 
     json_schema: dict = pydantic.Field(alias="json-schema")
@@ -36,7 +37,7 @@ def _init_private_attributes(self) -> None:
         )
 
 
-class FieldListSchemaContract(ConfigModel, arbitrary_types_allowed=True):
+class FieldListSchemaContract(BaseAssertion, arbitrary_types_allowed=True):
     type: Literal["field-list"]
 
     fields: List[SchemaFieldClass]
@@ -67,15 +68,13 @@ def id(self):
     def generate_mcp(
         self, assertion_urn: str, entity_urn: str
     ) -> List[MetadataChangeProposalWrapper]:
-        schema_metadata = self.__root__._schema_metadata
-
-        assertionInfo = AssertionInfoClass(
+        aspect = AssertionInfoClass(
             type=AssertionTypeClass.DATA_SCHEMA,
             schemaAssertion=SchemaAssertionInfoClass(
-                entity=entity_urn, schema=schema_metadata
+                entity=entity_urn,
+                schema=self.__root__._schema_metadata,
             ),
+            description=self.__root__.description,
         )
 
-        return [
-            MetadataChangeProposalWrapper(entityUrn=assertion_urn, aspect=assertionInfo)
-        ]
+        return [MetadataChangeProposalWrapper(entityUrn=assertion_urn, aspect=aspect)]
diff --git a/metadata-ingestion/tests/unit/api/entities/datacontract/__init__.py b/metadata-ingestion/tests/unit/api/entities/datacontract/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/tests/unit/api/entities/datacontract/test_data_quality_assertion.py b/metadata-ingestion/tests/unit/api/entities/datacontract/test_data_quality_assertion.py
new file mode 100644
index 0000000000000..7be8b667a500b
--- /dev/null
+++ b/metadata-ingestion/tests/unit/api/entities/datacontract/test_data_quality_assertion.py
@@ -0,0 +1,55 @@
+from datahub.api.entities.datacontract.data_quality_assertion import (
+    DataQualityAssertion,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.schema_classes import (
+    AssertionInfoClass,
+    AssertionStdOperatorClass,
+    AssertionStdParameterClass,
+    AssertionStdParametersClass,
+    AssertionStdParameterTypeClass,
+    AssertionTypeClass,
+    AssertionValueChangeTypeClass,
+    SqlAssertionInfoClass,
+    SqlAssertionTypeClass,
+)
+
+
+def test_parse_sql_assertion():
+    assertion_urn = "urn:li:assertion:a"
+    entity_urn = "urn:li:dataset:d"
+    statement = "SELECT COUNT(*) FROM my_table WHERE value IS NOT NULL"
+
+    d = {
+        "type": "custom_sql",
+        "sql": statement,
+        "operator": {"type": "between", "min": 5, "max": 10},
+    }
+
+    assert DataQualityAssertion.parse_obj(d).generate_mcp(
+        assertion_urn, entity_urn
+    ) == [
+        MetadataChangeProposalWrapper(
+            entityUrn=assertion_urn,
+            aspect=AssertionInfoClass(
+                type=AssertionTypeClass.SQL,
+                sqlAssertion=SqlAssertionInfoClass(
+                    type=SqlAssertionTypeClass.METRIC,
+                    changeType=AssertionValueChangeTypeClass.ABSOLUTE,
+                    entity=entity_urn,
+                    statement="SELECT COUNT(*) FROM my_table WHERE value IS NOT NULL",
+                    operator=AssertionStdOperatorClass.BETWEEN,
+                    parameters=AssertionStdParametersClass(
+                        minValue=AssertionStdParameterClass(
+                            value="5",
+                            type=AssertionStdParameterTypeClass.NUMBER,
+                        ),
+                        maxValue=AssertionStdParameterClass(
+                            value="10",
+                            type=AssertionStdParameterTypeClass.NUMBER,
+                        ),
+                    ),
+                ),
+            ),
+        )
+    ]

From 1007204cda802f02a5639e074d95b634b2be9ddf Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Fri, 13 Oct 2023 21:07:19 +0200
Subject: [PATCH 046/259] feat(ingest/teradata): view parsing (#9005)

---
 .../docs/sources/teradata/teradata_pre.md     |   2 +-
 .../docs/sources/teradata/teradata_recipe.yml |   3 +-
 .../datahub/ingestion/source/sql/teradata.py  | 156 ++++++++++++------
 3 files changed, 106 insertions(+), 55 deletions(-)

diff --git a/metadata-ingestion/docs/sources/teradata/teradata_pre.md b/metadata-ingestion/docs/sources/teradata/teradata_pre.md
index eb59caa29eb52..7263a59f5ea3d 100644
--- a/metadata-ingestion/docs/sources/teradata/teradata_pre.md
+++ b/metadata-ingestion/docs/sources/teradata/teradata_pre.md
@@ -18,7 +18,7 @@
    
     If you want to run profiling, you need to grant select permission on all the tables you want to profile.
 
-3. If linege or usage extraction is enabled, please, check if query logging is enabled and it is set to size which
+3. If lineage or usage extraction is enabled, please, check if query logging is enabled and it is set to size which
 will fit for your queries (the default query text size Teradata captures is max 200 chars)
    An example how you can set it for all users:
     ```sql
diff --git a/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml b/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml
index 8cf07ba4c3a01..cc94de20110fe 100644
--- a/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml
+++ b/metadata-ingestion/docs/sources/teradata/teradata_recipe.yml
@@ -3,12 +3,11 @@ source:
   type: teradata
   config:
     host_port: "myteradatainstance.teradata.com:1025"
-    #platform_instance: "myteradatainstance"
     username: myuser
     password: mypassword
     #database_pattern:
     #  allow:
-    #    - "demo_user"
+    #    - "my_database"
     #  ignoreCase: true
     include_table_lineage: true
     include_usage_statistics: true
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index dd11cd840bed9..6080cf7b371e3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -1,5 +1,6 @@
 import logging
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Iterable, Optional, Set, Union
 
 # This import verifies that the dependencies are available.
@@ -11,6 +12,7 @@
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
@@ -32,11 +34,18 @@
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
+from datahub.metadata._schema_classes import (
+    MetadataChangeEventClass,
+    SchemaMetadataClass,
+    ViewPropertiesClass,
+)
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BytesTypeClass,
     TimeTypeClass,
 )
+from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.sqlglot_lineage import SchemaResolver, sqlglot_lineage
+from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -64,6 +73,7 @@
 @dataclass
 class TeradataReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowReport):
     num_queries_parsed: int = 0
+    num_view_ddl_parsed: int = 0
     num_table_parse_failures: int = 0
 
 
@@ -82,17 +92,16 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         "This requires to have the table lineage feature enabled.",
     )
 
+    include_view_lineage = Field(
+        default=True,
+        description="Whether to include view lineage in the ingestion. "
+        "This requires to have the view lineage feature enabled.",
+    )
     usage: BaseUsageConfig = Field(
         description="The usage config to use when generating usage statistics",
         default=BaseUsageConfig(),
     )
 
-    use_schema_resolver: bool = Field(
-        default=True,
-        description="Read SchemaMetadata aspects from DataHub to aid in SQL parsing. Turn off only for testing.",
-        hidden_from_docs=True,
-    )
-
     default_db: Optional[str] = Field(
         default=None,
         description="The default database to use for unqualified table names",
@@ -141,46 +150,47 @@ def __init__(self, config: TeradataConfig, ctx: PipelineContext):
         self.report: TeradataReport = TeradataReport()
         self.graph: Optional[DataHubGraph] = ctx.graph
 
-        if self.graph:
-            if self.config.use_schema_resolver:
-                self.schema_resolver = (
-                    self.graph.initialize_schema_resolver_from_datahub(
-                        platform=self.platform,
-                        platform_instance=self.config.platform_instance,
-                        env=self.config.env,
-                    )
-                )
-                self.urns = self.schema_resolver.get_urns()
-            else:
-                self.schema_resolver = self.graph._make_schema_resolver(
-                    platform=self.platform,
-                    platform_instance=self.config.platform_instance,
-                    env=self.config.env,
-                )
-                self.urns = None
-        else:
-            self.schema_resolver = SchemaResolver(
-                platform=self.platform,
-                platform_instance=self.config.platform_instance,
-                graph=None,
-                env=self.config.env,
-            )
-            self.urns = None
-
         self.builder: SqlParsingBuilder = SqlParsingBuilder(
             usage_config=self.config.usage
             if self.config.include_usage_statistics
             else None,
-            generate_lineage=self.config.include_table_lineage,
+            generate_lineage=True,
             generate_usage_statistics=self.config.include_usage_statistics,
             generate_operations=self.config.usage.include_operational_stats,
         )
 
+        self.schema_resolver = SchemaResolver(
+            platform=self.platform,
+            platform_instance=self.config.platform_instance,
+            graph=None,
+            env=self.config.env,
+        )
+
+        self._view_definition_cache: FileBackedDict[str] = FileBackedDict()
+
     @classmethod
     def create(cls, config_dict, ctx):
         config = TeradataConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
+    def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
+        for key in self._view_definition_cache.keys():
+            view_definition = self._view_definition_cache[key]
+            dataset_urn = DatasetUrn.create_from_string(key)
+
+            db_name: Optional[str] = None
+            # We need to get the default db from the dataset urn otherwise the builder generates the wrong urns
+            if "." in dataset_urn.get_dataset_name():
+                db_name = dataset_urn.get_dataset_name().split(".", 1)[0]
+
+            self.report.num_view_ddl_parsed += 1
+            if self.report.num_view_ddl_parsed % 1000 == 0:
+                logger.info(f"Parsed {self.report.num_queries_parsed} view ddl")
+
+            yield from self.gen_lineage_from_query(
+                query=view_definition, default_database=db_name, is_view_ddl=True
+            )
+
     def get_audit_log_mcps(self) -> Iterable[MetadataWorkUnit]:
         engine = self.get_metadata_engine()
         for entry in engine.execute(
@@ -192,27 +202,43 @@ def get_audit_log_mcps(self) -> Iterable[MetadataWorkUnit]:
             if self.report.num_queries_parsed % 1000 == 0:
                 logger.info(f"Parsed {self.report.num_queries_parsed} queries")
 
-            result = sqlglot_lineage(
-                sql=entry.query,
-                schema_resolver=self.schema_resolver,
-                default_db=None,
-                default_schema=entry.default_database
-                if entry.default_database
-                else self.config.default_db,
+            yield from self.gen_lineage_from_query(
+                query=entry.query,
+                default_database=entry.default_database,
+                timestamp=entry.timestamp,
+                user=entry.user,
+                is_view_ddl=False,
             )
-            if result.debug_info.table_error:
-                logger.debug(
-                    f"Error parsing table lineage, {result.debug_info.table_error}"
-                )
-                self.report.num_table_parse_failures += 1
-                continue
 
+    def gen_lineage_from_query(
+        self,
+        query: str,
+        default_database: Optional[str] = None,
+        timestamp: Optional[datetime] = None,
+        user: Optional[str] = None,
+        is_view_ddl: bool = False,
+    ) -> Iterable[MetadataWorkUnit]:
+        result = sqlglot_lineage(
+            sql=query,
+            schema_resolver=self.schema_resolver,
+            default_db=None,
+            default_schema=default_database
+            if default_database
+            else self.config.default_db,
+        )
+        if result.debug_info.table_error:
+            logger.debug(
+                f"Error parsing table lineage, {result.debug_info.table_error}"
+            )
+            self.report.num_table_parse_failures += 1
+        else:
             yield from self.builder.process_sql_parsing_result(
                 result,
-                query=entry.query,
-                query_timestamp=entry.timestamp,
-                user=f"urn:li:corpuser:{entry.user}",
-                include_urns=self.urns,
+                query=query,
+                is_view_ddl=is_view_ddl,
+                query_timestamp=timestamp,
+                user=f"urn:li:corpuser:{user}",
+                include_urns=self.schema_resolver.get_urns(),
             )
 
     def get_metadata_engine(self) -> Engine:
@@ -221,8 +247,34 @@ def get_metadata_engine(self) -> Engine:
         return create_engine(url, **self.config.options)
 
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
-        yield from super().get_workunits_internal()
+        # Add all schemas to the schema resolver
+        for wu in super().get_workunits_internal():
+            if isinstance(wu.metadata, MetadataChangeEventClass):
+                if wu.metadata.proposedSnapshot:
+                    for aspect in wu.metadata.proposedSnapshot.aspects:
+                        if isinstance(aspect, SchemaMetadataClass):
+                            self.schema_resolver.add_schema_metadata(
+                                wu.metadata.proposedSnapshot.urn,
+                                aspect,
+                            )
+                            break
+            if isinstance(wu.metadata, MetadataChangeProposalWrapper):
+                if (
+                    wu.metadata.entityUrn
+                    and isinstance(wu.metadata.aspect, ViewPropertiesClass)
+                    and wu.metadata.aspect.viewLogic
+                ):
+                    self._view_definition_cache[
+                        wu.metadata.entityUrn
+                    ] = wu.metadata.aspect.viewLogic
+            yield wu
+
+        if self.config.include_view_lineage:
+            self.report.report_ingestion_stage_start("view lineage extraction")
+            yield from self.get_view_lineage()
+
         if self.config.include_table_lineage or self.config.include_usage_statistics:
             self.report.report_ingestion_stage_start("audit log extraction")
             yield from self.get_audit_log_mcps()
-            yield from self.builder.gen_workunits()
+
+        yield from self.builder.gen_workunits()

From c2e8041d771db1a20889255372312791fb6d911c Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Fri, 13 Oct 2023 22:59:18 +0200
Subject: [PATCH 047/259] Adding missing sqlparser libs to setup.py (#9015)

---
 metadata-ingestion/setup.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 3ea9a2ea61d74..545cafca9d4df 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -373,7 +373,10 @@
     # FIXME: I don't think tableau uses sqllineage anymore so we should be able
     # to remove that dependency.
     "tableau": {"tableauserverclient>=0.17.0"} | sqllineage_lib | sqlglot_lib,
-    "teradata": sql_common | {"teradatasqlalchemy>=17.20.0.0"},
+    "teradata": sql_common
+    | usage_common
+    | sqlglot_lib
+    | {"teradatasqlalchemy>=17.20.0.0"},
     "trino": sql_common | trino,
     "starburst-trino-usage": sql_common | usage_common | trino,
     "nifi": {"requests", "packaging", "requests-gssapi"},
@@ -432,9 +435,7 @@
 deepdiff_dep = "deepdiff"
 test_api_requirements = {pytest_dep, deepdiff_dep, "PyYAML"}
 
-debug_requirements = {
-    "memray"
-}
+debug_requirements = {"memray"}
 
 base_dev_requirements = {
     *base_requirements,

From 78b342f441b340189e4eab60574daa60074457e0 Mon Sep 17 00:00:00 2001
From: Indy Prentice <iprentic@users.noreply.github.com>
Date: Fri, 13 Oct 2023 19:04:44 -0300
Subject: [PATCH 048/259] feat(graphql): support filtering based on greater
 than/less than criteria (#9001)

Co-authored-by: Indy Prentice <indy@Indys-MacBook-Pro.local>
---
 .../src/main/resources/search.graphql         | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql
index 4cabdb04afe77..e0cde5a2db9f9 100644
--- a/datahub-graphql-core/src/main/resources/search.graphql
+++ b/datahub-graphql-core/src/main/resources/search.graphql
@@ -458,6 +458,26 @@ enum FilterOperator {
   Represents the relation: The field exists. If the field is an array, the field is either not present or empty.
   """
   EXISTS
+
+  """
+  Represent the relation greater than, e.g. ownerCount > 5
+  """
+  GREATER_THAN
+
+  """
+   Represent the relation greater than or equal to, e.g. ownerCount >= 5
+  """
+  GREATER_THAN_OR_EQUAL_TO
+
+  """
+  Represent the relation less than, e.g. ownerCount < 3
+  """
+  LESS_THAN
+
+  """
+  Represent the relation less than or equal to, e.g. ownerCount <= 3
+  """
+  LESS_THAN_OR_EQUAL_TO
 }
 
 """

From c81a339bfc3a57161e433c64bd331ca6af4f6f2d Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Mon, 16 Oct 2023 21:57:57 +0530
Subject: [PATCH 049/259] build(ingest): remove ratelimiter dependency (#9008)

---
 metadata-ingestion/setup.py                   |  1 -
 .../bigquery_v2/bigquery_audit_log_api.py     |  2 +-
 .../src/datahub/utilities/ratelimiter.py      | 56 +++++++++++++++++++
 .../tests/unit/utilities/test_ratelimiter.py  | 20 +++++++
 4 files changed, 77 insertions(+), 2 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/utilities/ratelimiter.py
 create mode 100644 metadata-ingestion/tests/unit/utilities/test_ratelimiter.py

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 545cafca9d4df..1f4f0a0bad9b2 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -38,7 +38,6 @@
     "progressbar2",
     "termcolor>=1.0.0",
     "psutil>=5.8.0",
-    "ratelimiter",
     "Deprecated",
     "humanfriendly",
     "packaging",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py
index 03b12c61ee5c6..db552c09cd0a7 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py
@@ -4,7 +4,6 @@
 
 from google.cloud import bigquery
 from google.cloud.logging_v2.client import Client as GCPLoggingClient
-from ratelimiter import RateLimiter
 
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import (
     AuditLogEntry,
@@ -17,6 +16,7 @@
     BQ_DATE_SHARD_FORMAT,
     BQ_DATETIME_FORMAT,
 )
+from datahub.utilities.ratelimiter import RateLimiter
 
 logger: logging.Logger = logging.getLogger(__name__)
 
diff --git a/metadata-ingestion/src/datahub/utilities/ratelimiter.py b/metadata-ingestion/src/datahub/utilities/ratelimiter.py
new file mode 100644
index 0000000000000..3d47d25e14c49
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/ratelimiter.py
@@ -0,0 +1,56 @@
+import collections
+import threading
+import time
+from contextlib import AbstractContextManager
+from typing import Any, Deque
+
+
+# Modified version of https://github.com/RazerM/ratelimiter/blob/master/ratelimiter/_sync.py
+class RateLimiter(AbstractContextManager):
+
+    """Provides rate limiting for an operation with a configurable number of
+    requests for a time period.
+    """
+
+    def __init__(self, max_calls: int, period: float = 1.0) -> None:
+        """Initialize a RateLimiter object which enforces as much as max_calls
+        operations on period (eventually floating) number of seconds.
+        """
+        if period <= 0:
+            raise ValueError("Rate limiting period should be > 0")
+        if max_calls <= 0:
+            raise ValueError("Rate limiting number of calls should be > 0")
+
+        # We're using a deque to store the last execution timestamps, not for
+        # its maxlen attribute, but to allow constant time front removal.
+        self.calls: Deque = collections.deque()
+
+        self.period = period
+        self.max_calls = max_calls
+        self._lock = threading.Lock()
+
+    def __enter__(self) -> "RateLimiter":
+        with self._lock:
+            # We want to ensure that no more than max_calls were run in the allowed
+            # period. For this, we store the last timestamps of each call and run
+            # the rate verification upon each __enter__ call.
+            if len(self.calls) >= self.max_calls:
+                until = time.time() + self.period - self._timespan
+                sleeptime = until - time.time()
+                if sleeptime > 0:
+                    time.sleep(sleeptime)
+            return self
+
+    def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None:
+        with self._lock:
+            # Store the last operation timestamp.
+            self.calls.append(time.time())
+
+            # Pop the timestamp list front (ie: the older calls) until the sum goes
+            # back below the period. This is our 'sliding period' window.
+            while self._timespan >= self.period:
+                self.calls.popleft()
+
+    @property
+    def _timespan(self) -> float:
+        return self.calls[-1] - self.calls[0]
diff --git a/metadata-ingestion/tests/unit/utilities/test_ratelimiter.py b/metadata-ingestion/tests/unit/utilities/test_ratelimiter.py
new file mode 100644
index 0000000000000..0384e1f918881
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_ratelimiter.py
@@ -0,0 +1,20 @@
+from collections import defaultdict
+from datetime import datetime
+from typing import Dict
+
+from datahub.utilities.ratelimiter import RateLimiter
+
+
+def test_rate_is_limited():
+    MAX_CALLS_PER_SEC = 5
+    TOTAL_CALLS = 18
+    actual_calls: Dict[float, int] = defaultdict(lambda: 0)
+
+    ratelimiter = RateLimiter(max_calls=MAX_CALLS_PER_SEC, period=1)
+    for _ in range(TOTAL_CALLS):
+        with ratelimiter:
+            actual_calls[datetime.now().replace(microsecond=0).timestamp()] += 1
+
+    assert len(actual_calls) == round(TOTAL_CALLS / MAX_CALLS_PER_SEC)
+    assert all(calls <= MAX_CALLS_PER_SEC for calls in actual_calls.values())
+    assert sum(actual_calls.values()) == TOTAL_CALLS

From 9ccd1d4f5da8f3c93cb9aaacdb5de66600c99c99 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Mon, 16 Oct 2023 14:34:15 -0400
Subject: [PATCH 050/259] build(ingest/redshift): Add sqlglot dependency
 (#9021)

---
 metadata-ingestion/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 1f4f0a0bad9b2..7be565d51260d 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -353,7 +353,7 @@
     | {"psycopg2-binary", "pymysql>=1.0.2"},
     "pulsar": {"requests"},
     "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib,
-    "redshift": sql_common | redshift_common | usage_common | {"redshift-connector"},
+    "redshift": sql_common | redshift_common | usage_common | sqlglot_lib | {"redshift-connector"},
     "redshift-legacy": sql_common | redshift_common,
     "redshift-usage-legacy": sql_common | usage_common | redshift_common,
     "s3": {*s3_base, *data_lake_profiling},

From 6366b63e48d37de883af61fb801632e9a43d6e48 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Mon, 16 Oct 2023 19:13:23 -0400
Subject: [PATCH 051/259] feat(ingest/teradata): Add option to not use file
 backed dict for view definitions (#9024)

---
 .../datahub/ingestion/source/sql/teradata.py  | 47 ++++++++-----------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index 6080cf7b371e3..e628e4dbd3446 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -1,7 +1,7 @@
 import logging
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Iterable, Optional, Set, Union
+from typing import Iterable, MutableMapping, Optional, Union
 
 # This import verifies that the dependencies are available.
 import teradatasqlalchemy  # noqa: F401
@@ -12,7 +12,6 @@
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
@@ -34,11 +33,7 @@
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
-from datahub.metadata._schema_classes import (
-    MetadataChangeEventClass,
-    SchemaMetadataClass,
-    ViewPropertiesClass,
-)
+from datahub.metadata._schema_classes import SchemaMetadataClass, ViewPropertiesClass
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BytesTypeClass,
     TimeTypeClass,
@@ -112,6 +107,11 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         description="Generate usage statistic.",
     )
 
+    use_file_backed_cache: bool = Field(
+        default=True,
+        description="Whether to use a file backed cache for the view definitions.",
+    )
+
 
 @platform_name("Teradata")
 @config_class(TeradataConfig)
@@ -142,7 +142,8 @@ class TeradataSource(TwoTierSQLAlchemySource):
      and "timestamp" >= TIMESTAMP '{start_time}'
      and "timestamp" < TIMESTAMP '{end_time}'
      """
-    urns: Optional[Set[str]]
+
+    _view_definition_cache: MutableMapping[str, str]
 
     def __init__(self, config: TeradataConfig, ctx: PipelineContext):
         super().__init__(config, ctx, "teradata")
@@ -166,7 +167,10 @@ def __init__(self, config: TeradataConfig, ctx: PipelineContext):
             env=self.config.env,
         )
 
-        self._view_definition_cache: FileBackedDict[str] = FileBackedDict()
+        if self.config.use_file_backed_cache:
+            self._view_definition_cache = FileBackedDict[str]()
+        else:
+            self._view_definition_cache = {}
 
     @classmethod
     def create(cls, config_dict, ctx):
@@ -249,24 +253,13 @@ def get_metadata_engine(self) -> Engine:
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
         # Add all schemas to the schema resolver
         for wu in super().get_workunits_internal():
-            if isinstance(wu.metadata, MetadataChangeEventClass):
-                if wu.metadata.proposedSnapshot:
-                    for aspect in wu.metadata.proposedSnapshot.aspects:
-                        if isinstance(aspect, SchemaMetadataClass):
-                            self.schema_resolver.add_schema_metadata(
-                                wu.metadata.proposedSnapshot.urn,
-                                aspect,
-                            )
-                            break
-            if isinstance(wu.metadata, MetadataChangeProposalWrapper):
-                if (
-                    wu.metadata.entityUrn
-                    and isinstance(wu.metadata.aspect, ViewPropertiesClass)
-                    and wu.metadata.aspect.viewLogic
-                ):
-                    self._view_definition_cache[
-                        wu.metadata.entityUrn
-                    ] = wu.metadata.aspect.viewLogic
+            urn = wu.get_urn()
+            schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass)
+            if schema_metadata:
+                self.schema_resolver.add_schema_metadata(urn, schema_metadata)
+            view_properties = wu.get_aspect_of_type(ViewPropertiesClass)
+            if view_properties and self.config.include_view_lineage:
+                self._view_definition_cache[urn] = view_properties.viewLogic
             yield wu
 
         if self.config.include_view_lineage:

From 9fec6024fb177a321860e49f3c9977b41bb9e65f Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 17 Oct 2023 09:58:38 -0400
Subject: [PATCH 052/259] feat(ingest/unity-catalog): Support external S3
 lineage (#9025)

---
 .../datahub/ingestion/source/aws/s3_util.py   | 11 +++++--
 .../source/snowflake/snowflake_lineage_v2.py  |  6 ++--
 .../datahub/ingestion/source/unity/config.py  |  8 +++++
 .../datahub/ingestion/source/unity/proxy.py   |  8 +++++
 .../ingestion/source/unity/proxy_types.py     | 31 +++++++++++++++++++
 .../datahub/ingestion/source/unity/report.py  |  2 ++
 .../datahub/ingestion/source/unity/source.py  | 23 ++++++++++++++
 7 files changed, 84 insertions(+), 5 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py
index 501162455cc45..878b8dd1bb9a5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py
@@ -34,21 +34,26 @@ def get_bucket_relative_path(s3_uri: str) -> str:
     return "/".join(strip_s3_prefix(s3_uri).split("/")[1:])
 
 
-def make_s3_urn(s3_uri: str, env: str) -> str:
+def make_s3_urn(s3_uri: str, env: str, remove_extension: bool = True) -> str:
     s3_name = strip_s3_prefix(s3_uri)
 
     if s3_name.endswith("/"):
         s3_name = s3_name[:-1]
 
     name, extension = os.path.splitext(s3_name)
-
-    if extension != "":
+    if remove_extension and extension != "":
         extension = extension[1:]  # remove the dot
         return f"urn:li:dataset:(urn:li:dataPlatform:s3,{name}_{extension},{env})"
 
     return f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name},{env})"
 
 
+def make_s3_urn_for_lineage(s3_uri: str, env: str) -> str:
+    # Ideally this is the implementation for all S3 URNs
+    # Don't feel comfortable changing `make_s3_urn` for glue, sagemaker, and athena
+    return make_s3_urn(s3_uri, env, remove_extension=False)
+
+
 def get_bucket_name(s3_uri: str) -> str:
     if not is_s3_uri(s3_uri):
         raise ValueError(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
index 9a993f5774032..0a15c352fc842 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
@@ -21,7 +21,7 @@
 import datahub.emitter.mce_builder as builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.aws.s3_util import make_s3_urn
+from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage
 from datahub.ingestion.source.snowflake.constants import (
     LINEAGE_PERMISSION_ERROR,
     SnowflakeEdition,
@@ -652,7 +652,9 @@ def get_external_upstreams(self, external_lineage: Set[str]) -> List[UpstreamCla
             # For now, populate only for S3
             if external_lineage_entry.startswith("s3://"):
                 external_upstream_table = UpstreamClass(
-                    dataset=make_s3_urn(external_lineage_entry, self.config.env),
+                    dataset=make_s3_urn_for_lineage(
+                        external_lineage_entry, self.config.env
+                    ),
                     type=DatasetLineageTypeClass.COPY,
                 )
                 external_upstreams.append(external_upstream_table)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index a57ee39848855..16820c37d546e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -166,6 +166,14 @@ class UnityCatalogSourceConfig(
         description="Option to enable/disable lineage generation.",
     )
 
+    include_external_lineage: bool = pydantic.Field(
+        default=True,
+        description=(
+            "Option to enable/disable lineage generation for external tables."
+            " Only external S3 tables are supported at the moment."
+        ),
+    )
+
     include_notebooks: bool = pydantic.Field(
         default=False,
         description="Ingest notebooks, represented as DataHub datasets.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
index 9bcdb200f180e..3fb77ce512ed2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
@@ -33,6 +33,7 @@
     ALLOWED_STATEMENT_TYPES,
     Catalog,
     Column,
+    ExternalTableReference,
     Metastore,
     Notebook,
     Query,
@@ -248,6 +249,13 @@ def table_lineage(self, table: Table, include_entity_lineage: bool) -> None:
                     )
                     if table_ref:
                         table.upstreams[table_ref] = {}
+                elif "fileInfo" in item:
+                    external_ref = ExternalTableReference.create_from_lineage(
+                        item["fileInfo"]
+                    )
+                    if external_ref:
+                        table.external_upstreams.add(external_ref)
+
                 for notebook in item.get("notebookInfos") or []:
                     table.upstream_notebooks.add(notebook["notebook_id"])
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
index 18ac2475b51e0..315c1c0d20186 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py
@@ -10,6 +10,7 @@
     CatalogType,
     ColumnTypeName,
     DataSourceFormat,
+    SecurableType,
     TableType,
 )
 from databricks.sdk.service.sql import QueryStatementType
@@ -176,6 +177,35 @@ def external_path(self) -> str:
         return f"{self.catalog}/{self.schema}/{self.table}"
 
 
+@dataclass(frozen=True, order=True)
+class ExternalTableReference:
+    path: str
+    has_permission: bool
+    name: Optional[str]
+    type: Optional[SecurableType]
+    storage_location: Optional[str]
+
+    @classmethod
+    def create_from_lineage(cls, d: dict) -> Optional["ExternalTableReference"]:
+        try:
+            securable_type: Optional[SecurableType]
+            try:
+                securable_type = SecurableType(d.get("securable_type", "").lower())
+            except ValueError:
+                securable_type = None
+
+            return cls(
+                path=d["path"],
+                has_permission=d.get("has_permission") or True,
+                name=d.get("securable_name"),
+                type=securable_type,
+                storage_location=d.get("storage_location"),
+            )
+        except Exception as e:
+            logger.warning(f"Failed to create ExternalTableReference from {d}: {e}")
+            return None
+
+
 @dataclass
 class Table(CommonProperty):
     schema: Schema
@@ -193,6 +223,7 @@ class Table(CommonProperty):
     view_definition: Optional[str]
     properties: Dict[str, str]
     upstreams: Dict[TableReference, Dict[str, List[str]]] = field(default_factory=dict)
+    external_upstreams: Set[ExternalTableReference] = field(default_factory=set)
     upstream_notebooks: Set[NotebookId] = field(default_factory=set)
     downstream_notebooks: Set[NotebookId] = field(default_factory=set)
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
index fa61571fa92cb..4153d9dd88eb8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py
@@ -19,6 +19,8 @@ class UnityCatalogReport(IngestionStageReport, StaleEntityRemovalSourceReport):
     notebooks: EntityFilterReport = EntityFilterReport.field(type="notebook")
 
     num_column_lineage_skipped_column_count: int = 0
+    num_external_upstreams_lacking_permissions: int = 0
+    num_external_upstreams_unsupported: int = 0
 
     num_queries: int = 0
     num_queries_dropped_parse_failure: int = 0
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
index 27c1f341aa84d..b63cf65d55dc8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
@@ -41,6 +41,7 @@
     TestConnectionReport,
 )
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage
 from datahub.ingestion.source.common.subtypes import (
     DatasetContainerSubTypes,
     DatasetSubTypes,
@@ -455,6 +456,28 @@ def _generate_lineage_aspect(
                 )
             )
 
+        if self.config.include_external_lineage:
+            for external_ref in table.external_upstreams:
+                if not external_ref.has_permission or not external_ref.path:
+                    self.report.num_external_upstreams_lacking_permissions += 1
+                    logger.warning(
+                        f"Lacking permissions for external file upstream on {table.ref}"
+                    )
+                elif external_ref.path.startswith("s3://"):
+                    upstreams.append(
+                        UpstreamClass(
+                            dataset=make_s3_urn_for_lineage(
+                                external_ref.path, self.config.env
+                            ),
+                            type=DatasetLineageTypeClass.COPY,
+                        )
+                    )
+                else:
+                    self.report.num_external_upstreams_unsupported += 1
+                    logger.warning(
+                        f"Unsupported external file upstream on {table.ref}: {external_ref.path}"
+                    )
+
         if upstreams:
             return UpstreamLineageClass(
                 upstreams=upstreams,

From 10eb205cb8d455639c6d09dcc0c8f3853264f96f Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Tue, 17 Oct 2023 16:16:25 +0200
Subject: [PATCH 053/259] fix(ingest) - Fix file backed collection temp
 directory removal (#9027)

---
 .../src/datahub/utilities/file_backed_collections.py   | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py
index c04d2138bc116..18493edded4b7 100644
--- a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py
+++ b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py
@@ -3,6 +3,7 @@
 import logging
 import pathlib
 import pickle
+import shutil
 import sqlite3
 import tempfile
 from dataclasses import dataclass, field
@@ -56,15 +57,15 @@ class ConnectionWrapper:
     conn: sqlite3.Connection
     filename: pathlib.Path
 
-    _temp_directory: Optional[tempfile.TemporaryDirectory]
+    _temp_directory: Optional[str]
 
     def __init__(self, filename: Optional[pathlib.Path] = None):
         self._temp_directory = None
 
         # Warning: If filename is provided, the file will not be automatically cleaned up.
         if not filename:
-            self._temp_directory = tempfile.TemporaryDirectory()
-            filename = pathlib.Path(self._temp_directory.name) / _DEFAULT_FILE_NAME
+            self._temp_directory = tempfile.mkdtemp()
+            filename = pathlib.Path(self._temp_directory) / _DEFAULT_FILE_NAME
 
         self.conn = sqlite3.connect(filename, isolation_level=None)
         self.conn.row_factory = sqlite3.Row
@@ -101,7 +102,8 @@ def executemany(
     def close(self) -> None:
         self.conn.close()
         if self._temp_directory:
-            self._temp_directory.cleanup()
+            shutil.rmtree(self._temp_directory)
+            self._temp_directory = None
 
     def __enter__(self) -> "ConnectionWrapper":
         return self

From e7c662a0aca0be97e34bec55161766ea84036ced Mon Sep 17 00:00:00 2001
From: ethan-cartwright <ethan.cartwright.m@gmail.com>
Date: Tue, 17 Oct 2023 10:54:07 -0400
Subject: [PATCH 054/259] add dependency level to scrollAcrossLineage search
 results (#9016)

---
 datahub-web-react/src/graphql/scroll.graphql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/datahub-web-react/src/graphql/scroll.graphql b/datahub-web-react/src/graphql/scroll.graphql
index 18274c50c2166..1031fed7b9e13 100644
--- a/datahub-web-react/src/graphql/scroll.graphql
+++ b/datahub-web-react/src/graphql/scroll.graphql
@@ -408,6 +408,7 @@ fragment downloadScrollAcrossLineageResult on ScrollAcrossLineageResults {
     count
     total
     searchResults {
+        degree
         entity {
             ...downloadSearchResults
         }

From ae5fd90c73ff29e00f4b8e20735ce0b72e7b823b Mon Sep 17 00:00:00 2001
From: ethan-cartwright <ethan.cartwright.m@gmail.com>
Date: Tue, 17 Oct 2023 10:55:07 -0400
Subject: [PATCH 055/259] add create dataproduct example (#9009)

---
 .../examples/library/create_dataproduct.py    | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 metadata-ingestion/examples/library/create_dataproduct.py

diff --git a/metadata-ingestion/examples/library/create_dataproduct.py b/metadata-ingestion/examples/library/create_dataproduct.py
new file mode 100644
index 0000000000000..245395b602480
--- /dev/null
+++ b/metadata-ingestion/examples/library/create_dataproduct.py
@@ -0,0 +1,25 @@
+from datahub.api.entities.dataproduct.dataproduct import DataProduct
+from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
+
+gms_endpoint = "http://localhost:8080"
+graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint))
+
+data_product = DataProduct(
+    id="pet_of_the_week",
+    display_name="Pet of the Week Campagin",
+    domain="urn:li:domain:ef39e99a-9d61-406d-b4a8-c70b16380206",
+    description="This campaign includes Pet of the Week data.",
+    assets=[
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD)",
+        "urn:li:dashboard:(looker,baz)",
+        "urn:li:dataFlow:(airflow,dag_abc,PROD)",
+    ],
+    owners=[{"id": "urn:li:corpuser:jdoe", "type": "BUSINESS_OWNER"}],
+    terms=["urn:li:glossaryTerm:ClientsAndAccounts.AccountBalance"],
+    tags=["urn:li:tag:adoption"],
+    properties={"lifecycle": "production", "sla": "7am every day"},
+    external_url="https://en.wikipedia.org/wiki/Sloth",
+)
+
+for mcp in data_product.generate_mcp(upsert=False):
+    graph.emit(mcp)

From 75108ceb2ff125af52fb1e37f7f6d371a77de3b7 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Tue, 17 Oct 2023 14:13:31 -0400
Subject: [PATCH 056/259] Download Lineage Results Cypress Test (#9017)

---
 .../styled/search/DownloadAsCsvModal.tsx      |  2 +
 .../styled/search/SearchExtendedMenu.tsx      |  4 +-
 .../e2e/lineage/download_lineage_results.js   | 80 +++++++++++++++++++
 3 files changed, 84 insertions(+), 2 deletions(-)
 create mode 100644 smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js

diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx
index 452658583cf61..92e859ee1b329 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx
@@ -130,6 +130,7 @@ export default function DownloadAsCsvModal({
                         Close
                     </Button>
                     <Button
+                        data-testid="csv-modal-download-button"
                         onClick={() => {
                             setShowDownloadAsCsvModal(false);
                             triggerCsvDownload(saveAsTitle);
@@ -142,6 +143,7 @@ export default function DownloadAsCsvModal({
             }
         >
             <Input
+                data-testid="download-as-csv-input"
                 placeholder="datahub.csv"
                 value={saveAsTitle}
                 onChange={(e) => {
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx
index d81ec673324bf..a26749d8a37a1 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx
@@ -45,7 +45,7 @@ export default function SearchExtendedMenu({
 
     const menu = (
         <Menu>
-            <MenuItem key="0">
+            <MenuItem key="0" data-testid="download-as-csv-menu-item">
                 <DownloadAsCsvButton
                     isDownloadingCsv={isDownloadingCsv}
                     setShowDownloadAsCsvModal={setShowDownloadAsCsvModal}
@@ -75,7 +75,7 @@ export default function SearchExtendedMenu({
                 totalResults={totalResults}
             />
             <Dropdown overlay={menu} trigger={['click']}>
-                <MenuIcon />
+                <MenuIcon data-testid="three-dot-menu" />
             </Dropdown>
         </>
     );
diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
new file mode 100644
index 0000000000000..315aa7b22b9da
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
@@ -0,0 +1,80 @@
+const test_dataset = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)";
+const first_degree = [
+    "urn:li:chart:(looker,cypress_baz1)",
+    "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)",
+    "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)"
+];
+const second_degree = [
+    "urn:li:chart:(looker,cypress_baz2)",
+    "urn:li:dashboard:(looker,cypress_baz)",
+    "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)",
+    "urn:li:mlPrimaryKey:(cypress-test-2,some-cypress-feature-2)"
+];
+const third_degree_plus = [
+    "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_123)",
+    "urn:li:dataJob:(urn:li:dataFlow:(airflow,cypress_dag_abc,PROD),cypress_task_456)",
+    "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)",
+    "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)",
+    "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)",
+    "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD)"
+];
+const downloadCsvFile = (filename) => {
+    cy.get('[data-testid="three-dot-menu"]').click();
+    cy.get('[data-testid="download-as-csv-menu-item"]').click();
+    cy.get('[data-testid="download-as-csv-input"]').clear().type(filename);
+    cy.get('[data-testid="csv-modal-download-button"]').click().wait(5000);
+    cy.ensureTextNotPresent("Creating CSV to download");
+};
+
+describe("download lineage results to .csv file", () => {
+
+    it("download and verify lineage results for 1st, 2nd and 3+ degree of dependencies", () => {
+      cy.loginWithCredentials();
+      cy.goToDataset(test_dataset,"SampleCypressKafkaDataset");
+      cy.openEntityTab("Lineage");
+
+      // Verify 1st degree of dependencies
+      cy.contains(/1 - 3 of 3/);
+      downloadCsvFile("first_degree_results.csv");
+      let first_degree_csv = cy.readFile('cypress/downloads/first_degree_results.csv');
+      first_degree.forEach(function (urn) {
+        first_degree_csv.should('contain', urn) 
+      });
+      second_degree.forEach(function (urn) {
+        first_degree_csv.should('not.contain', urn)
+      });
+      third_degree_plus.forEach(function (urn) {
+        first_degree_csv.should('not.contain', urn);
+      });
+
+      // Verify 1st and 2nd degree of dependencies
+      cy.get('[data-testid="facet-degree-2"]').click().wait(5000);
+      cy.contains(/1 - 7 of 7/);
+      downloadCsvFile("second_degree_results.csv");
+      let second_degree_csv = cy.readFile('cypress/downloads/second_degree_results.csv');
+      first_degree.forEach(function (urn) {
+        second_degree_csv.should('contain', urn) 
+      });
+      second_degree.forEach(function (urn) {
+        second_degree_csv.should('contain', urn)
+      });
+      third_degree_plus.forEach(function (urn) {
+        second_degree_csv.should('not.contain', urn);
+      });
+
+      // Verify 1st 2nd and 3+ degree of dependencies(Verify multi page download)
+      cy.get('[data-testid="facet-degree-3+"]').click().wait(5000);
+      cy.contains(/1 - 10 of 13/);
+      downloadCsvFile("third_plus_degree_results.csv");
+      let third_degree_csv = cy.readFile('cypress/downloads/third_plus_degree_results.csv');
+      first_degree.forEach(function (urn) {
+        third_degree_csv.should('contain', urn) 
+      });
+      second_degree.forEach(function (urn) {
+        third_degree_csv.should('contain', urn)
+      });
+      third_degree_plus.forEach(function (urn) {
+        third_degree_csv.should('contain', urn);
+      });
+    });
+}); 
\ No newline at end of file

From da6cc54d63e8b3814d2bb8dc4e14b90d74d67a5d Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 17 Oct 2023 16:18:39 -0400
Subject: [PATCH 057/259] fix(ingest/bigquery): Remove table name restrictions
 (allow $ and @) (#9030)

---
 .../source/bigquery_v2/bigquery_audit.py        | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
index 88060a9cdc91d..55366d6c57cf8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py
@@ -3,7 +3,7 @@
 import re
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any, ClassVar, Dict, List, Optional, Pattern, Set, Tuple, Union
+from typing import Any, ClassVar, Dict, List, Optional, Pattern, Tuple, Union
 
 from dateutil import parser
 
@@ -35,8 +35,6 @@ class BigqueryTableIdentifier:
     dataset: str
     table: str
 
-    invalid_chars: ClassVar[Set[str]] = {"$", "@"}
-
     # Note: this regex may get overwritten by the sharded_table_pattern config.
     # The class-level constant, however, will not be overwritten.
     _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX: ClassVar[
@@ -105,18 +103,7 @@ def get_table_display_name(self) -> str:
             )
 
         table_name, _ = self.get_table_and_shard(shortened_table_name)
-        if not table_name:
-            table_name = self.dataset
-
-        # Handle exceptions
-        invalid_chars_in_table_name: List[str] = [
-            c for c in self.invalid_chars if c in table_name
-        ]
-        if invalid_chars_in_table_name:
-            raise ValueError(
-                f"Cannot handle {self.raw_table_name()} - poorly formatted table name, contains {invalid_chars_in_table_name}"
-            )
-        return table_name
+        return table_name or self.dataset
 
     def get_table_name(self) -> str:
         """

From 6f466b0e5080dfb7d5fee504821006223aeeddc7 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Tue, 17 Oct 2023 15:40:38 -0500
Subject: [PATCH 058/259] chore(docker): update base images to alpine 3.18
 (#8967)

---
 docker/build.gradle                      | 2 +-
 docker/datahub-gms/Dockerfile            | 2 +-
 docker/datahub-ingestion-base/Dockerfile | 2 +-
 docker/datahub-mae-consumer/Dockerfile   | 2 +-
 docker/datahub-mce-consumer/Dockerfile   | 2 +-
 docker/datahub-upgrade/Dockerfile        | 2 +-
 docker/elasticsearch-setup/Dockerfile    | 2 +-
 docker/mysql-setup/Dockerfile            | 2 +-
 docker/postgres-setup/Dockerfile         | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/docker/build.gradle b/docker/build.gradle
index 0faea626e982d..c8fdbc86b18b7 100644
--- a/docker/build.gradle
+++ b/docker/build.gradle
@@ -35,7 +35,7 @@ task quickstart(type: Exec, dependsOn: ':metadata-ingestion:install') {
 
     environment "DATAHUB_TELEMETRY_ENABLED", "false"
     environment "DOCKER_COMPOSE_BASE", "file://${rootProject.projectDir}"
-    // environment "ACTIONS_VERSION", 'alpine3.17-slim'
+    // environment "ACTIONS_VERSION", 'alpine3.18-slim'
     // environment "DATAHUB_ACTIONS_IMAGE", 'nginx'
 
     // Elastic
diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
index 2d74a288b8c99..f5428f7480403 100644
--- a/docker/datahub-gms/Dockerfile
+++ b/docker/datahub-gms/Dockerfile
@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile
index 564cc19cc9a5f..25afe9b8b3dce 100644
--- a/docker/datahub-ingestion-base/Dockerfile
+++ b/docker/datahub-ingestion-base/Dockerfile
@@ -1,7 +1,7 @@
 ARG APP_ENV=full
 ARG BASE_IMAGE=base
 
-FROM golang:1-alpine3.17 AS dockerize-binary
+FROM golang:1-alpine3.18 AS dockerize-binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
index 734f8ba452f3e..4b321b1639c1b 100644
--- a/docker/datahub-mae-consumer/Dockerfile
+++ b/docker/datahub-mae-consumer/Dockerfile
@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
index ee5d927fb1ddb..4d38ee6daa235 100644
--- a/docker/datahub-mce-consumer/Dockerfile
+++ b/docker/datahub-mce-consumer/Dockerfile
@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
index 4e1521cc0561e..945be54678a24 100644
--- a/docker/datahub-upgrade/Dockerfile
+++ b/docker/datahub-upgrade/Dockerfile
@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile
index af3c8c9df762a..c8fb2eba911b8 100644
--- a/docker/elasticsearch-setup/Dockerfile
+++ b/docker/elasticsearch-setup/Dockerfile
@@ -3,7 +3,7 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/mysql-setup/Dockerfile b/docker/mysql-setup/Dockerfile
index 732b860a58f07..56bab61180489 100644
--- a/docker/mysql-setup/Dockerfile
+++ b/docker/mysql-setup/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
diff --git a/docker/postgres-setup/Dockerfile b/docker/postgres-setup/Dockerfile
index 313615ac3465b..7f4d53ae044d4 100644
--- a/docker/postgres-setup/Dockerfile
+++ b/docker/postgres-setup/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.18 AS binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder

From 60c1aab58765c2f643021efbba2719a845be2033 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Tue, 17 Oct 2023 15:50:32 -0500
Subject: [PATCH 059/259] fix(frontend): update cookie module (#8862)

---
 datahub-frontend/app/auth/AuthModule.java     |  4 +-
 datahub-frontend/app/auth/AuthUtils.java      |  9 ++-
 .../app/auth/cookie/CustomCookiesModule.java  | 22 ++++++
 .../cookie/CustomSessionCookieBaker.scala     | 25 ++++++
 .../sso/oidc/OidcAuthorizationGenerator.java  | 37 ---------
 .../app/auth/sso/oidc/OidcCallbackLogic.java  | 19 ++++-
 .../controllers/AuthenticationController.java | 24 +++---
 datahub-frontend/conf/application.conf        | 15 +++-
 .../test/app/ApplicationTest.java             | 28 +++++--
 docs/authentication/README.md                 |  5 +-
 .../guides/sso/configure-oidc-react.md        |  3 +-
 docs/deploy/environment-vars.md               | 13 +--
 docs/how/updating-datahub.md                  |  2 +
 .../AuthenticationConfiguration.java          |  2 +
 .../TokenServiceConfiguration.java            | 15 ++++
 metadata-service/auth-filter/build.gradle     |  2 +
 .../authentication/AuthTestConfiguration.java | 79 +++++++++++++++++++
 .../AuthenticationFilterTest.java             | 53 +++++++++++++
 .../src/main/resources/application.yml        |  2 +
 .../auth/DataHubTokenServiceFactory.java      |  4 +-
 20 files changed, 288 insertions(+), 75 deletions(-)
 create mode 100644 datahub-frontend/app/auth/cookie/CustomCookiesModule.java
 create mode 100644 datahub-frontend/app/auth/cookie/CustomSessionCookieBaker.scala
 create mode 100644 metadata-service/auth-config/src/main/java/com/datahub/authentication/TokenServiceConfiguration.java
 create mode 100644 metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthTestConfiguration.java
 create mode 100644 metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthenticationFilterTest.java

diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java
index 98f3b82285eda..fe04c3629fe58 100644
--- a/datahub-frontend/app/auth/AuthModule.java
+++ b/datahub-frontend/app/auth/AuthModule.java
@@ -56,7 +56,7 @@ public class AuthModule extends AbstractModule {
      * Pac4j Stores Session State in a browser-side cookie in encrypted fashion. This configuration
      * value provides a stable encryption base from which to derive the encryption key.
      *
-     * We hash this value (SHA1), then take the first 16 bytes as the AES key.
+     * We hash this value (SHA256), then take the first 16 bytes as the AES key.
      */
     private static final String PAC4J_AES_KEY_BASE_CONF = "play.http.secret.key";
     private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
@@ -93,7 +93,7 @@ protected void configure() {
                 // it to hex and slice the first 16 bytes, because AES key length must strictly
                 // have a specific length.
                 final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF);
-                final String aesKeyHash = DigestUtils.sha1Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
+                final String aesKeyHash = DigestUtils.sha256Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
                 final String aesEncryptionKey = aesKeyHash.substring(0, 16);
                 playCacheCookieStore = new PlayCookieSessionStore(
                         new ShiroAesDataEncrypter(aesEncryptionKey.getBytes()));
diff --git a/datahub-frontend/app/auth/AuthUtils.java b/datahub-frontend/app/auth/AuthUtils.java
index 80bd631d0db70..386eee725c83d 100644
--- a/datahub-frontend/app/auth/AuthUtils.java
+++ b/datahub-frontend/app/auth/AuthUtils.java
@@ -41,6 +41,11 @@ public class AuthUtils {
      */
     public static final String SYSTEM_CLIENT_SECRET_CONFIG_PATH = "systemClientSecret";
 
+    /**
+     * Cookie name for redirect url that is manually separated from the session to reduce size
+     */
+    public static final String REDIRECT_URL_COOKIE_NAME = "REDIRECT_URL";
+
     public static final CorpuserUrn DEFAULT_ACTOR_URN = new CorpuserUrn("datahub");
 
     public static final String LOGIN_ROUTE = "/login";
@@ -77,7 +82,9 @@ public static boolean isEligibleForForwarding(Http.Request req) {
      * as well as their agreement to determine authentication status.
      */
     public static boolean hasValidSessionCookie(final Http.Request req) {
-        return req.session().data().containsKey(ACTOR)
+        Map<String, String> sessionCookie = req.session().data();
+        return sessionCookie.containsKey(ACCESS_TOKEN)
+                && sessionCookie.containsKey(ACTOR)
                 && req.getCookie(ACTOR).isPresent()
                 && req.session().data().get(ACTOR).equals(req.getCookie(ACTOR).get().value());
     }
diff --git a/datahub-frontend/app/auth/cookie/CustomCookiesModule.java b/datahub-frontend/app/auth/cookie/CustomCookiesModule.java
new file mode 100644
index 0000000000000..a6dbd69a93889
--- /dev/null
+++ b/datahub-frontend/app/auth/cookie/CustomCookiesModule.java
@@ -0,0 +1,22 @@
+package auth.cookie;
+
+import com.google.inject.AbstractModule;
+import play.api.libs.crypto.CookieSigner;
+import play.api.libs.crypto.CookieSignerProvider;
+import play.api.mvc.DefaultFlashCookieBaker;
+import play.api.mvc.FlashCookieBaker;
+import play.api.mvc.SessionCookieBaker;
+
+
+public class CustomCookiesModule extends AbstractModule {
+
+  @Override
+  public void configure() {
+    bind(CookieSigner.class).toProvider(CookieSignerProvider.class);
+    // We override the session cookie baker to not use a fallback, this prevents using an old URL Encoded cookie
+    bind(SessionCookieBaker.class).to(CustomSessionCookieBaker.class);
+    // We don't care about flash cookies, we don't use them
+    bind(FlashCookieBaker.class).to(DefaultFlashCookieBaker.class);
+  }
+
+}
diff --git a/datahub-frontend/app/auth/cookie/CustomSessionCookieBaker.scala b/datahub-frontend/app/auth/cookie/CustomSessionCookieBaker.scala
new file mode 100644
index 0000000000000..6f0a6604fa64b
--- /dev/null
+++ b/datahub-frontend/app/auth/cookie/CustomSessionCookieBaker.scala
@@ -0,0 +1,25 @@
+package auth.cookie
+
+import com.google.inject.Inject
+import play.api.http.{SecretConfiguration, SessionConfiguration}
+import play.api.libs.crypto.CookieSigner
+import play.api.mvc.DefaultSessionCookieBaker
+
+import scala.collection.immutable.Map
+
+/**
+ * Overrides default fallback to URL Encoding behavior, prevents usage of old URL encoded session cookies
+ * @param config
+ * @param secretConfiguration
+ * @param cookieSigner
+ */
+class CustomSessionCookieBaker @Inject() (
+  override val config: SessionConfiguration,
+  override val secretConfiguration: SecretConfiguration,
+  cookieSigner: CookieSigner
+) extends DefaultSessionCookieBaker(config, secretConfiguration, cookieSigner) {
+  // Has to be a Scala class because it extends a trait with concrete implementations, Scala does compilation tricks
+
+  // Forces use of jwt encoding and disallows fallback to legacy url encoding
+  override def decode(encodedData: String): Map[String, String] = jwtCodec.decode(encodedData)
+}
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java
index 3f864ed5abddf..baca144610ec4 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java
@@ -1,19 +1,9 @@
 package auth.sso.oidc;
 
-import java.text.ParseException;
 import java.util.Map.Entry;
 import java.util.Optional;
 
-import com.nimbusds.jose.Algorithm;
-import com.nimbusds.jose.Header;
-import com.nimbusds.jose.JWEAlgorithm;
-import com.nimbusds.jose.JWSAlgorithm;
-import com.nimbusds.jose.util.Base64URL;
-import com.nimbusds.jose.util.JSONObjectUtils;
-import com.nimbusds.jwt.EncryptedJWT;
 import com.nimbusds.jwt.JWTParser;
-import com.nimbusds.jwt.SignedJWT;
-import net.minidev.json.JSONObject;
 import org.pac4j.core.authorization.generator.AuthorizationGenerator;
 import org.pac4j.core.context.WebContext;
 import org.pac4j.core.profile.AttributeLocation;
@@ -63,32 +53,5 @@ public Optional<UserProfile> generate(WebContext context, UserProfile profile) {
         
         return Optional.ofNullable(profile);
     }
-
-    private static JWT parse(final String s) throws ParseException {
-        final int firstDotPos = s.indexOf(".");
-
-        if (firstDotPos == -1) {
-            throw new ParseException("Invalid JWT serialization: Missing dot delimiter(s)", 0);
-        }
-
-        Base64URL header = new Base64URL(s.substring(0, firstDotPos));
-        JSONObject jsonObject;
-
-        try {
-            jsonObject = JSONObjectUtils.parse(header.decodeToString());
-        } catch (ParseException e) {
-            throw new ParseException("Invalid unsecured/JWS/JWE header: " + e.getMessage(), 0);
-        }
-
-        Algorithm alg = Header.parseAlgorithm(jsonObject);
-
-        if (alg instanceof JWSAlgorithm) {
-            return SignedJWT.parse(s);
-        } else if (alg instanceof JWEAlgorithm) {
-            return EncryptedJWT.parse(s);
-        } else {
-            throw new AssertionError("Unexpected algorithm type: " + alg);
-        }
-    }
     
 }
diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java
index 4bde0872fc082..7164710f4e0de 100644
--- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java
+++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java
@@ -38,6 +38,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -49,19 +50,21 @@
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.pac4j.core.config.Config;
+import org.pac4j.core.context.Cookie;
 import org.pac4j.core.engine.DefaultCallbackLogic;
 import org.pac4j.core.http.adapter.HttpActionAdapter;
 import org.pac4j.core.profile.CommonProfile;
 import org.pac4j.core.profile.ProfileManager;
 import org.pac4j.core.profile.UserProfile;
+import org.pac4j.core.util.Pac4jConstants;
 import org.pac4j.play.PlayWebContext;
 import play.mvc.Result;
 import auth.sso.SsoManager;
 
-import static auth.AuthUtils.createActorCookie;
-import static auth.AuthUtils.createSessionMap;
+import static auth.AuthUtils.*;
 import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME;
+import static org.pac4j.play.store.PlayCookieSessionStore.*;
 import static play.mvc.Results.internalServerError;
 
 
@@ -97,6 +100,9 @@ public OidcCallbackLogic(final SsoManager ssoManager, final Authentication syste
   public Result perform(PlayWebContext context, Config config,
       HttpActionAdapter<Result, PlayWebContext> httpActionAdapter, String defaultUrl, Boolean saveInSession,
       Boolean multiProfile, Boolean renewSession, String defaultClient) {
+
+    setContextRedirectUrl(context);
+
     final Result result =
         super.perform(context, config, httpActionAdapter, defaultUrl, saveInSession, multiProfile, renewSession,
             defaultClient);
@@ -111,6 +117,15 @@ public Result perform(PlayWebContext context, Config config,
     return handleOidcCallback(oidcConfigs, result, context, getProfileManager(context));
   }
 
+  @SuppressWarnings("unchecked")
+  private void setContextRedirectUrl(PlayWebContext context) {
+    Optional<Cookie> redirectUrl = context.getRequestCookies().stream()
+        .filter(cookie -> REDIRECT_URL_COOKIE_NAME.equals(cookie.getName())).findFirst();
+    redirectUrl.ifPresent(
+        cookie -> context.getSessionStore().set(context, Pac4jConstants.REQUESTED_URL,
+            JAVA_SER_HELPER.deserializeFromBytes(uncompressBytes(Base64.getDecoder().decode(cookie.getValue())))));
+  }
+
   private Result handleOidcCallback(final OidcConfigs oidcConfigs, final Result result, final PlayWebContext context,
       final ProfileManager<UserProfile> profileManager) {
 
diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java
index e9ddfb2611ceb..4f89f4f67e149 100644
--- a/datahub-frontend/app/controllers/AuthenticationController.java
+++ b/datahub-frontend/app/controllers/AuthenticationController.java
@@ -13,14 +13,15 @@
 import com.typesafe.config.Config;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.Optional;
 import javax.annotation.Nonnull;
 import javax.inject.Inject;
 import org.apache.commons.lang3.StringUtils;
 import org.pac4j.core.client.Client;
+import org.pac4j.core.context.Cookie;
 import org.pac4j.core.exception.http.FoundAction;
 import org.pac4j.core.exception.http.RedirectionAction;
-import org.pac4j.core.util.Pac4jConstants;
 import org.pac4j.play.PlayWebContext;
 import org.pac4j.play.http.PlayHttpActionAdapter;
 import org.pac4j.play.store.PlaySessionStore;
@@ -33,18 +34,9 @@
 import play.mvc.Results;
 import security.AuthenticationManager;
 
-import static auth.AuthUtils.DEFAULT_ACTOR_URN;
-import static auth.AuthUtils.EMAIL;
-import static auth.AuthUtils.FULL_NAME;
-import static auth.AuthUtils.INVITE_TOKEN;
-import static auth.AuthUtils.LOGIN_ROUTE;
-import static auth.AuthUtils.PASSWORD;
-import static auth.AuthUtils.RESET_TOKEN;
-import static auth.AuthUtils.TITLE;
-import static auth.AuthUtils.USER_NAME;
-import static auth.AuthUtils.createActorCookie;
-import static auth.AuthUtils.createSessionMap;
+import static auth.AuthUtils.*;
 import static org.pac4j.core.client.IndirectClient.ATTEMPTED_AUTHENTICATION_SUFFIX;
+import static org.pac4j.play.store.PlayCookieSessionStore.*;
 
 
 // TODO add logging.
@@ -297,8 +289,12 @@ private Optional<Result> redirectToIdentityProvider(Http.RequestHeader request,
     }
 
     private void configurePac4jSessionStore(PlayWebContext context, Client client, String redirectPath) {
-        // Set the originally requested path for post-auth redirection.
-        _playSessionStore.set(context, Pac4jConstants.REQUESTED_URL, new FoundAction(redirectPath));
+        // Set the originally requested path for post-auth redirection. We split off into a separate cookie from the session
+        // to reduce size of the session cookie
+        FoundAction foundAction = new FoundAction(redirectPath);
+        byte[] javaSerBytes = JAVA_SER_HELPER.serializeToBytes(foundAction);
+        String serialized = Base64.getEncoder().encodeToString(compressBytes(javaSerBytes));
+        context.addResponseCookie(new Cookie(REDIRECT_URL_COOKIE_NAME, serialized));
         // This is to prevent previous login attempts from being cached.
         // We replicate the logic here, which is buried in the Pac4j client.
         if (_playSessionStore.get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX) != null) {
diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf
index 18d901d5ee7dd..1a62c8547e721 100644
--- a/datahub-frontend/conf/application.conf
+++ b/datahub-frontend/conf/application.conf
@@ -22,11 +22,16 @@ play.application.loader = play.inject.guice.GuiceApplicationLoader
 play.http.parser.maxMemoryBuffer = 10MB
 play.http.parser.maxMemoryBuffer = ${?DATAHUB_PLAY_MEM_BUFFER_SIZE}
 
-# TODO: Disable legacy URL encoding eventually
+play.modules.disabled += "play.api.mvc.LegacyCookiesModule"
 play.modules.disabled += "play.api.mvc.CookiesModule"
-play.modules.enabled += "play.api.mvc.LegacyCookiesModule"
+play.modules.enabled += "auth.cookie.CustomCookiesModule"
 play.modules.enabled += "auth.AuthModule"
 
+jwt {
+  # 'alg' https://tools.ietf.org/html/rfc7515#section-4.1.1
+      signatureAlgorithm = "HS256"
+}
+
 # We override the Akka server provider to allow setting the max header count to a higher value
 # This is useful while using proxies like Envoy that result in the frontend server rejecting GMS
 # responses as there's more than the max of 64 allowed headers
@@ -199,10 +204,14 @@ auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
 # auth.native.enabled = false
 # auth.oidc.enabled = false # (or simply omit oidc configurations)
 
-# Login session expiration time
+# Login session expiration time, controls when the actor cookie is expired on the browser side
 auth.session.ttlInHours = 24
 auth.session.ttlInHours = ${?AUTH_SESSION_TTL_HOURS}
 
+# Control the length of time a session token is valid
+play.http.session.maxAge = 24h
+play.http.session.maxAge = ${?MAX_SESSION_TOKEN_AGE}
+
 analytics.enabled = true
 analytics.enabled = ${?DATAHUB_ANALYTICS_ENABLED}
 
diff --git a/datahub-frontend/test/app/ApplicationTest.java b/datahub-frontend/test/app/ApplicationTest.java
index 417fd79e76bbd..f27fefdb79669 100644
--- a/datahub-frontend/test/app/ApplicationTest.java
+++ b/datahub-frontend/test/app/ApplicationTest.java
@@ -1,6 +1,11 @@
 package app;
 
+import com.nimbusds.jwt.JWT;
+import com.nimbusds.jwt.JWTClaimsSet;
+import com.nimbusds.jwt.JWTParser;
 import controllers.routes;
+import java.text.ParseException;
+import java.util.Date;
 import no.nav.security.mock.oauth2.MockOAuth2Server;
 import no.nav.security.mock.oauth2.token.DefaultOAuth2TokenCallback;
 import okhttp3.mockwebserver.MockResponse;
@@ -27,8 +32,6 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 
@@ -149,7 +152,7 @@ public void testOpenIdConfig() {
   }
 
   @Test
-  public void testHappyPathOidc() throws InterruptedException {
+  public void testHappyPathOidc() throws ParseException {
     browser.goTo("/authenticate");
     assertEquals("", browser.url());
 
@@ -157,8 +160,23 @@ public void testHappyPathOidc() throws InterruptedException {
     assertEquals(TEST_USER, actorCookie.getValue());
 
     Cookie sessionCookie = browser.getCookie("PLAY_SESSION");
-    assertTrue(sessionCookie.getValue().contains("token=" + TEST_TOKEN));
-    assertTrue(sessionCookie.getValue().contains("actor=" + URLEncoder.encode(TEST_USER, StandardCharsets.UTF_8)));
+    String jwtStr = sessionCookie.getValue();
+    JWT jwt = JWTParser.parse(jwtStr);
+    JWTClaimsSet claims = jwt.getJWTClaimsSet();
+    Map<String, String> data = (Map<String, String>) claims.getClaim("data");
+    assertEquals(TEST_TOKEN, data.get("token"));
+    assertEquals(TEST_USER, data.get("actor"));
+    // Default expiration is 24h, so should always be less than current time + 1 day since it stamps the time before this executes
+    assertTrue(claims.getExpirationTime().compareTo(new Date(System.currentTimeMillis() + (24 * 60 * 60 * 1000))) < 0);
+  }
+
+  @Test
+  public void testAPI() throws ParseException {
+    testHappyPathOidc();
+    int requestCount = _gmsServer.getRequestCount();
+
+    browser.goTo("/api/v2/graphql/");
+    assertEquals(++requestCount, _gmsServer.getRequestCount());
   }
 
   @Test
diff --git a/docs/authentication/README.md b/docs/authentication/README.md
index f6eda88784486..ff4a3d83cfde3 100644
--- a/docs/authentication/README.md
+++ b/docs/authentication/README.md
@@ -31,8 +31,9 @@ When a user makes a request for Data within DataHub, the request is authenticate
 and programmatic calls to DataHub APIs. There are two types of tokens that are important:
 
 1. **Session Tokens**: Generated for users of the DataHub web application. By default, having a duration of 24 hours. 
-These tokens are encoded and stored inside browser-side session cookies. The duration a session token is valid for is configurable via the `AUTH_SESSION_TTL_HOURS` environment variable
-on the datahub-frontend deployment.
+These tokens are encoded and stored inside browser-side session cookies. The duration a session token is valid for is configurable via the `MAX_SESSION_TOKEN_AGE` environment variable
+on the datahub-frontend deployment. Additionally, the `AUTH_SESSION_TTL_HOURS` configures the expiration time of the actor cookie on the user's browser which will also prompt a user login. The difference between these is that the actor cookie expiration only affects the browser session and can still be used programmatically,
+but when the session expires it can no longer be used programmatically either as it is created as a JWT with an expiration claim.
 2. **Personal Access Tokens**: These are tokens generated via the DataHub settings panel useful for interacting
 with DataHub APIs. They can be used to automate processes like enriching documentation, ownership, tags, and more on DataHub. Learn
 more about Personal Access Tokens [here](personal-access-tokens.md). 
diff --git a/docs/authentication/guides/sso/configure-oidc-react.md b/docs/authentication/guides/sso/configure-oidc-react.md
index 512d6adbf916f..1671673c09318 100644
--- a/docs/authentication/guides/sso/configure-oidc-react.md
+++ b/docs/authentication/guides/sso/configure-oidc-react.md
@@ -72,7 +72,8 @@ AUTH_OIDC_BASE_URL=your-datahub-url
 - `AUTH_OIDC_CLIENT_SECRET`: Unique client secret received from identity provider
 - `AUTH_OIDC_DISCOVERY_URI`: Location of the identity provider OIDC discovery API. Suffixed with `.well-known/openid-configuration`
 - `AUTH_OIDC_BASE_URL`: The base URL of your DataHub deployment, e.g. https://yourorgdatahub.com (prod) or http://localhost:9002 (testing)
-- `AUTH_SESSION_TTL_HOURS`: The length of time in hours before a user will be prompted to login again. Session tokens are stateless so this determines at what time a session token may no longer be used and a valid session token can be used until this time has passed.
+- `AUTH_SESSION_TTL_HOURS`: The length of time in hours before a user will be prompted to login again. Controls the actor cookie expiration time in the browser. Numeric value converted to hours, default 24.
+- `MAX_SESSION_TOKEN_AGE`: Determines the expiration time of a session token. Session tokens are stateless so this determines at what time a session token may no longer be used and a valid session token can be used until this time has passed. Accepts a valid relative Java date style String, default 24h.
 
 Providing these configs will cause DataHub to delegate authentication to your identity
 provider, requesting the "oidc email profile" scopes and parsing the "preferred_username" claim from
diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md
index 0689db9b17331..779c3d3d7c432 100644
--- a/docs/deploy/environment-vars.md
+++ b/docs/deploy/environment-vars.md
@@ -79,9 +79,10 @@ Simply replace the dot, `.`, with an underscore, `_`, and convert to uppercase.
 
 ## Frontend
 
-| Variable                           | Default  | Unit/Type | Components   | Description                                                                                                                                                                                                                                               |
-|------------------------------------|----------|-----------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `AUTH_VERBOSE_LOGGING`             | `false`  | boolean   | [`Frontend`] | Enable verbose authentication logging. Enabling this will leak sensisitve information in the logs. Disable when finished debugging.                                                                                                                       |
-| `AUTH_OIDC_GROUPS_CLAIM`           | `groups` | string    | [`Frontend`] | Claim to use as the user's group.                                                                                                                                                                                                                         |
-| `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` | `false`  | boolean   | [`Frontend`] | Auto-provision the group from the user's group claim.                                                                                                                                                                                                     |
-| `AUTH_SESSION_TTL_HOURS`           | `24`     | string    | [`Frontend`] | The number of hours a user session is valid. [User session tokens are stateless and will become invalid after this time](https://www.playframework.com/documentation/2.8.x/SettingsSession#Session-Timeout-/-Expiration) requiring a user to login again. |
\ No newline at end of file
+| Variable                           | Default  | Unit/Type | Components    | Description                                                                                                                                                                                                                                        |
+|------------------------------------|----------|-----------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `AUTH_VERBOSE_LOGGING`             | `false`  | boolean   | [`Frontend`]  | Enable verbose authentication logging. Enabling this will leak sensisitve information in the logs. Disable when finished debugging.                                                                                                                |
+| `AUTH_OIDC_GROUPS_CLAIM`           | `groups` | string    | [`Frontend`]  | Claim to use as the user's group.                                                                                                                                                                                                                  |
+| `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` | `false`  | boolean   | [`Frontend`]  | Auto-provision the group from the user's group claim.                                                                                                                                                                                              |
+| `AUTH_SESSION_TTL_HOURS`           | `24`     | string    | [`Frontend`]  | The number of hours a user session is valid. After this many hours the actor cookie will be expired by the browser and the user will be prompted to login again.                                                                                   |
+| `MAX_SESSION_TOKEN_AGE`            | `24h`    | string    | [`Frontend`]  | The maximum age of the session token. [User session tokens are stateless and will become invalid after this time](https://www.playframework.com/documentation/2.8.x/SettingsSession#Session-Timeout-/-Expiration) requiring a user to login again. | 
\ No newline at end of file
diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 9cd4ad5c6f02d..3af3b2bdda215 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -22,6 +22,8 @@ Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
 ### Deprecations
 
 ### Other Notable Changes
+- Session token configuration has changed, all previously created session tokens will be invalid and users will be prompted to log in. Expiration time has also been shortened which may result in more login prompts with the default settings.
+  There should be no other interruption due to this change.
 
 ## 0.11.0
 
diff --git a/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java b/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java
index f9cf1b01e1762..d3c5ba822ac04 100644
--- a/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java
+++ b/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java
@@ -29,4 +29,6 @@ public class AuthenticationConfiguration {
    * The lifespan of a UI session token.
    */
   private long sessionTokenDurationMs;
+
+  private TokenServiceConfiguration tokenService;
 }
diff --git a/metadata-service/auth-config/src/main/java/com/datahub/authentication/TokenServiceConfiguration.java b/metadata-service/auth-config/src/main/java/com/datahub/authentication/TokenServiceConfiguration.java
new file mode 100644
index 0000000000000..0a606f0f06d92
--- /dev/null
+++ b/metadata-service/auth-config/src/main/java/com/datahub/authentication/TokenServiceConfiguration.java
@@ -0,0 +1,15 @@
+package com.datahub.authentication;
+
+import lombok.Data;
+
+
+@Data
+/**
+ * Configurations for DataHub token service
+ */
+public class TokenServiceConfiguration {
+  private String signingKey;
+  private String salt;
+  private String issuer;
+  private String signingAlgorithm;
+}
diff --git a/metadata-service/auth-filter/build.gradle b/metadata-service/auth-filter/build.gradle
index 2dd07ef10274c..61e9015adc942 100644
--- a/metadata-service/auth-filter/build.gradle
+++ b/metadata-service/auth-filter/build.gradle
@@ -14,4 +14,6 @@ dependencies {
 
   annotationProcessor externalDependency.lombok
   testImplementation externalDependency.mockito
+  testImplementation externalDependency.testng
+  testImplementation externalDependency.springBootTest
 }
\ No newline at end of file
diff --git a/metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthTestConfiguration.java b/metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthTestConfiguration.java
new file mode 100644
index 0000000000000..05ca428283a6c
--- /dev/null
+++ b/metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthTestConfiguration.java
@@ -0,0 +1,79 @@
+package com.datahub.auth.authentication;
+
+import com.datahub.auth.authentication.filter.AuthenticationFilter;
+import com.datahub.authentication.AuthenticationConfiguration;
+import com.datahub.authentication.AuthenticatorConfiguration;
+import com.datahub.authentication.TokenServiceConfiguration;
+import com.datahub.authentication.token.StatefulTokenService;
+import com.linkedin.gms.factory.config.ConfigurationProvider;
+import com.linkedin.metadata.config.AuthPluginConfiguration;
+import com.linkedin.metadata.config.DataHubConfiguration;
+import com.linkedin.metadata.config.PluginConfiguration;
+import com.linkedin.metadata.entity.EntityService;
+import java.util.List;
+import java.util.Map;
+import javax.servlet.ServletException;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.DependsOn;
+
+import static org.mockito.Mockito.*;
+
+@Configuration
+public class AuthTestConfiguration {
+
+
+  @Bean
+  public EntityService entityService() {
+    return mock(EntityService.class);
+  }
+
+  @Bean("dataHubTokenService")
+  public StatefulTokenService statefulTokenService(ConfigurationProvider configurationProvider, EntityService entityService) {
+    TokenServiceConfiguration tokenServiceConfiguration = configurationProvider.getAuthentication().getTokenService();
+    return new StatefulTokenService(
+        tokenServiceConfiguration.getSigningKey(),
+        tokenServiceConfiguration.getSigningAlgorithm(),
+        tokenServiceConfiguration.getIssuer(),
+        entityService,
+        tokenServiceConfiguration.getSalt()
+    );
+  }
+
+  @Bean
+  public ConfigurationProvider configurationProvider() {
+    ConfigurationProvider configurationProvider = new ConfigurationProvider();
+    AuthenticationConfiguration authenticationConfiguration = new AuthenticationConfiguration();
+    authenticationConfiguration.setEnabled(true);
+    configurationProvider.setAuthentication(authenticationConfiguration);
+    DataHubConfiguration dataHubConfiguration = new DataHubConfiguration();
+    PluginConfiguration pluginConfiguration = new PluginConfiguration();
+    AuthPluginConfiguration authPluginConfiguration = new AuthPluginConfiguration();
+    authenticationConfiguration.setSystemClientId("__datahub_system");
+    authenticationConfiguration.setSystemClientSecret("JohnSnowKnowsNothing");
+    TokenServiceConfiguration tokenServiceConfiguration = new TokenServiceConfiguration();
+    tokenServiceConfiguration.setIssuer("datahub-metadata-service");
+    tokenServiceConfiguration.setSigningKey("WnEdIeTG/VVCLQqGwC/BAkqyY0k+H8NEAtWGejrBI94=");
+    tokenServiceConfiguration.setSalt("ohDVbJBvHHVJh9S/UA4BYF9COuNnqqVhr9MLKEGXk1O=");
+    tokenServiceConfiguration.setSigningAlgorithm("HS256");
+    authenticationConfiguration.setTokenService(tokenServiceConfiguration);
+    AuthenticatorConfiguration authenticator = new AuthenticatorConfiguration();
+    authenticator.setType("com.datahub.authentication.authenticator.DataHubTokenAuthenticator");
+    authenticator.setConfigs(Map.of("signingKey", "WnEdIeTG/VVCLQqGwC/BAkqyY0k+H8NEAtWGejrBI94=",
+        "salt", "ohDVbJBvHHVJh9S/UA4BYF9COuNnqqVhr9MLKEGXk1O="));
+    List<AuthenticatorConfiguration> authenticators = List.of(authenticator);
+    authenticationConfiguration.setAuthenticators(authenticators);
+    authPluginConfiguration.setPath("");
+    pluginConfiguration.setAuth(authPluginConfiguration);
+    dataHubConfiguration.setPlugin(pluginConfiguration);
+    configurationProvider.setDatahub(dataHubConfiguration);
+    return configurationProvider;
+  }
+
+  @Bean
+  // TODO: Constructor injection
+  @DependsOn({"configurationProvider", "dataHubTokenService", "entityService"})
+  public AuthenticationFilter authenticationFilter() throws ServletException {
+    return new AuthenticationFilter();
+  }
+}
diff --git a/metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthenticationFilterTest.java b/metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthenticationFilterTest.java
new file mode 100644
index 0000000000000..2ac65bf09c912
--- /dev/null
+++ b/metadata-service/auth-filter/src/test/java/com/datahub/auth/authentication/AuthenticationFilterTest.java
@@ -0,0 +1,53 @@
+package com.datahub.auth.authentication;
+
+import com.datahub.auth.authentication.filter.AuthenticationFilter;
+import com.datahub.authentication.Actor;
+import com.datahub.authentication.ActorType;
+import com.datahub.authentication.token.StatefulTokenService;
+import com.datahub.authentication.token.TokenException;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
+import org.testng.annotations.Test;
+
+import static com.datahub.authentication.AuthenticationConstants.*;
+import static org.mockito.Mockito.*;
+
+
+@ContextConfiguration(classes = { AuthTestConfiguration.class })
+public class AuthenticationFilterTest extends AbstractTestNGSpringContextTests {
+
+  @Autowired
+  AuthenticationFilter _authenticationFilter;
+
+  @Autowired
+  StatefulTokenService _statefulTokenService;
+
+  @Test
+  public void testExpiredToken() throws ServletException, IOException, TokenException {
+    _authenticationFilter.init(null);
+    HttpServletRequest servletRequest = mock(HttpServletRequest.class);
+    HttpServletResponse servletResponse = mock(HttpServletResponse.class);
+    FilterChain filterChain = mock(FilterChain.class);
+    Actor actor = new Actor(ActorType.USER, "datahub");
+//    String token = _statefulTokenService.generateAccessToken(TokenType.SESSION, actor, 0L, System.currentTimeMillis(), "token",
+//        "token", actor.toUrnStr());
+    // Token generated 9/11/23, invalid for all future dates
+    String token = "eyJhbGciOiJIUzI1NiJ9.eyJhY3RvclR5cGUiOiJVU0VSIZCI6ImRhdGFodWIiLCJ0eXBlIjoiU0VTU0lPTiIsInZlcnNpb24iOiIxIiwian"
+        + "RpIjoiMmI0MzZkZDAtYjEwOS00N2UwLWJmYTEtMzM2ZmU4MTU4MDE1Iiwic3ViIjoiZGF0YWh1YiIsImV4cCI6MTY5NDU0NzA2OCwiaXNzIjoiZGF"
+        + "0YWh1Yi1tZXRhZGF0YS1zZXJ2aWNlIn0.giqx7J5a9mxuubG6rXdAMoaGlcII-fqY-W82Wm7OlLI";
+    when(servletRequest.getHeaderNames()).thenReturn(Collections.enumeration(List.of(AUTHORIZATION_HEADER_NAME)));
+    when(servletRequest.getHeader(AUTHORIZATION_HEADER_NAME))
+        .thenReturn("Bearer " + token);
+
+    _authenticationFilter.doFilter(servletRequest, servletResponse, filterChain);
+    verify(servletResponse, times(1)).sendError(eq(HttpServletResponse.SC_UNAUTHORIZED), anyString());
+  }
+}
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index d22f92adca8f9..5d72e24748072 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -25,6 +25,8 @@ authentication:
     # Key used to sign new tokens.
     signingKey: ${DATAHUB_TOKEN_SERVICE_SIGNING_KEY:WnEdIeTG/VVCLQqGwC/BAkqyY0k+H8NEAtWGejrBI94=}
     salt: ${DATAHUB_TOKEN_SERVICE_SALT:ohDVbJBvHHVJh9S/UA4BYF9COuNnqqVhr9MLKEGXk1O=}
+    issuer: ${DATAHUB_TOKEN_SERVICE_ISSUER:datahub-metadata-service}
+    signingAlgorithm: ${DATAHUB_TOKEN_SERVICE_SIGNING_ALGORITHM:HS256}
 
   # The max duration of a UI session in milliseconds. Defaults to 1 day.
   sessionTokenDurationMs: ${SESSION_TOKEN_DURATION_MS:86400000}
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java
index 6b2a61882be90..d47e1a0a73401 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java
@@ -23,10 +23,10 @@ public class DataHubTokenServiceFactory {
   @Value("${authentication.tokenService.salt:}")
   private String saltingKey;
 
-  @Value("${elasticsearch.tokenService.signingAlgorithm:HS256}")
+  @Value("${authentication.tokenService.signingAlgorithm:HS256}")
   private String signingAlgorithm;
 
-  @Value("${elasticsearch.tokenService.issuer:datahub-metadata-service}")
+  @Value("${authentication.tokenService.issuer:datahub-metadata-service}")
   private String issuer;
 
   /**

From 79f65563126c478fc4e254ea52d60fb645e26aad Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 17 Oct 2023 18:41:45 -0400
Subject: [PATCH 060/259] docs(datahub-lite): Fix recipe (#9023)

---
 docs/datahub_lite.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/datahub_lite.md b/docs/datahub_lite.md
index de0a20eed1d01..55491e3b998cf 100644
--- a/docs/datahub_lite.md
+++ b/docs/datahub_lite.md
@@ -85,9 +85,10 @@ source:
 
 sink:
   type: datahub-lite
-  forward_to:
-    type: datahub-rest
-    config:
+  config:
+    forward_to:
+      type: datahub-rest
+      config:
         server: "http://datahub-gms:8080"
 ```
 

From 9bb0bf25ae338c552d9d04620b4a7bdb607924c3 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Wed, 18 Oct 2023 14:09:27 +0530
Subject: [PATCH 061/259] fix(ingest): fix typo in parsing list of groups
 (#9037)

---
 metadata-ingestion/src/datahub/cli/specific/group_cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/cli/specific/group_cli.py b/metadata-ingestion/src/datahub/cli/specific/group_cli.py
index 9baa8ee68d975..e313fce33d4d5 100644
--- a/metadata-ingestion/src/datahub/cli/specific/group_cli.py
+++ b/metadata-ingestion/src/datahub/cli/specific/group_cli.py
@@ -43,7 +43,7 @@ def upsert(file: Path, override_editable: bool) -> None:
     with get_default_graph() as emitter:
         for group_config in group_configs:
             try:
-                datahub_group = CorpGroup.parse_obj(config_dict)
+                datahub_group = CorpGroup.parse_obj(group_config)
                 for mcp in datahub_group.generate_mcp(
                     generation_config=CorpGroupGenerationConfig(
                         override_editable=override_editable, datahub_graph=emitter

From 5937937819c00afcef2f3702f51778b4452bb47d Mon Sep 17 00:00:00 2001
From: VISHAL KUMAR <110387730+vishalkSimplify@users.noreply.github.com>
Date: Wed, 18 Oct 2023 14:52:07 +0530
Subject: [PATCH 062/259] feat(ingestion/Vertica): Fixed vertica integration
 test Updated vertica dialect (#9011)

---
 datahub-web-react/src/images/verticalogo.png  |  Bin 11194 -> 25763 bytes
 metadata-ingestion/setup.py                   |    2 +-
 .../tests/integration/vertica/ddl.sql         |   19 +-
 .../integration/vertica/docker-compose.yml    |    2 +-
 .../tests/integration/vertica/test_vertica.py |   30 +-
 .../vertica/vertica_mces_with_db_golden.json  | 3840 ++++++++++++-----
 .../integration/vertica/vertica_to_file.yml   |    7 +
 7 files changed, 2699 insertions(+), 1201 deletions(-)

diff --git a/datahub-web-react/src/images/verticalogo.png b/datahub-web-react/src/images/verticalogo.png
index a81047fd43edbfc20d51263799454b96d3c565d8..5da38f4e67c7d4ea72c02983bc8cf01c9b10752f 100644
GIT binary patch
literal 25763
zcmeEug;$%+)^7+Y1Sdd?1$QV~io3fMDOR*tvEWcNxJz-jwou%qKylZW7KbKyacgn7
z>3i<|zH{!n|H1v%nyfsTJkQKz&+NT_+Y+s%p@@e~feiov@RXJ0UH|~-Pyhf83dTV7
zsAtyGq2AEkUnt4|swbhpP#@&14U}!v)dB3NYcK#U(jEZ%dkgACfqJ1HFGK^Nqu$Z}
z_7wvE&$sB%LiGQ0jfR>50Fbx6wgv#C0LpUGx;|(Jd5>D5KduHt!=+$zj3Doj$1J*q
zoF6z(hv-_Q*alXZd|;(Yu1+|dS|+lEt;A3wlNi6_hnwD&dhf&BtY@~9!lDf~fm3HI
zoqiiPU(RwC7K8<sN)zQka#H`@L=w9J{8!zR4$VeSf%LTh)(AtpAHz<WJid@kZpi-k
z1AiYbbO%>Pd<>kc67<$+`0)Zli1u#{J`K9=?P-`+hzzZvjPP0<`7^Y)F<5c%--9ER
z%>BPpg7aE#Jl#4z+6TQ)3=LeaqG7rZqB$xEl|a8EVaL)m4{B99T=BuPKa=&%Wz5H_
z?c)WCl-jm4ySLD<bAMF(Fc;J9LU>SXB)=@&lOOSM)$;2ym6nZgtQ-5clx{(v8?00_
z75_WHH88cr+W!ul!YCm!)KEq=`7WIpq5gG7Si3Uh*2!%&b>yCu-As2-D(Cwv!UF@~
z=%l>t_TU4tYyg(xf*JfYQ>S1zFo3LvMusU6j2n{0Qk3#OQ}l^_`}_ACBk1rZ^H3IF
z+&kPrn+w7QwZ?QQ$<`nU#d>aX;#%&5o$p(da;u#?sEuu?Aa+o=GYIbon*pHNEPVIu
z>ng5(H}7N$7YW#x8m3iGY|X|0R&|GITCj+Ye0$-Z;{GUv8uSWF1fsj9fl$$X&b$3Q
zw*mT23Z&3ejrHZ|JQlr4+GZUx&$4#4c5}N2^4o>=x92b9YC@Hl2t*>c6L)Cm*DGVx
zaoKe=HFjY>_tUR4*RuohyYn#v$c;9qaBcN#;D?|*O!hjKV+WE6(PoB&iBGS!fvUy5
zEz=#L2!?{dbS{d*(U%e3YEfZ(ngO{u^^k}X5vWCLy11F|!8$^N^!prmGp=@E^NF>U
z&fOb6-AHs6Z*#*Zf7-s^ys~>GiJ5yh4;;N?AbN(z734?##ywY9mh(i({MOl+0P)*D
z(W^nt^(T04Sb0|&p%qpYYb2hSSM1D*W_qI^mXE%fvD&P~rTLV>rRY>^C(k;rtQzb=
zhBhg1NDzQOn)AD=a7Jp6eV;!MYx5iT<a8zO=5Gbm-c$QHwA)^RQ}ZfqJv-0j&GpIF
zW+0Al10_Z)w>e-7(F*3gKGzt#>z88+LJ8SO@X+jq?-*qmXOXoFe(TkVLV&>1;o^I8
z{M_%<)?mk181YTTeHXurqr~yw>|^JVuwPjyb6h}3Ey{Xa6KyrDB0`zXLaBd!^~(R$
zryV?F{O+1ih@wcK;MwnJljInw=U6c1@4u$~3S%?CC+^>0VUFVM5kUyy`JnW^C5Ivc
z>A8sX14N(PW;nx@qV$lAcZ8c(ampr${|hVnq|6x_6%hgGonWGv@XurOxxs!YYd@am
zySIdwRK2%`;xc-v^jf08<vB0ORE9SJR3+Q1?136rnWoX;D5(MoNzAaN!Uy<$xh#A<
z*oy|83Vj7_kzmojH-61Qi2p79cj1Ae{POpNN?t-jD~+})ZCc8HJ;&!oabwkjz}xA8
z+`H2!t+>%r@!;dP<I-fSP&}`=9L=glz$g;~5oAXfcvg%Q^xT`)2v4#Tq`B<<dE|>W
zntX#*lt;U7?&{U27g4-6Se>SKqzD|r$8_fkb6ez%P(x$E9D*^@d;lM>@oTc-l%wR1
zxoWT`-o|qu^(E|Rx4HFzad=6XU*}_ZIwX@MpCDWUV{<>^y&&8$$%E)znziQn0oO9C
z#LE!{YWQ`BSz0P>(kv1&Og48j@QxFq<gj7+mFv)!LU*CX=lKtmY+{u*SNYxv{hU{V
z$CcPQUsqioEk%WO`$Sq2YlU+6^V7P71Fv{ri0;N}omJs}SyKtFT**9=m1F-smBFl1
z>73>sENSFGD{x^T2@A0?PnNLJ7u^l}Zdocz52Po|00=hcgEyn9VqrXyn&rt+Qli9}
zc=0!a=PIl&JWPK;J`V$ETVJQWav>t*2kf7<p?WXEImDl6OlTKBM$?H5O<Ul&{Whv{
zB~#HOSL(dOuF0Jner5i)0=8vs2$Jg#0Vq1Y65u)=(OWW}e+hw<aln5-E*YkOes4m1
z9GydOg3QlwFU)=Ago$iJo^@mr{xPZUJLvwVN?9)gT{h6QDN+9lXafazIw^aCq48R+
z7AB}=Pa6S^`|hOi=}aO;9^A@G+3B^cfgM+(M59^p_7zaO3Clg1jtkknFXMTVZQXdM
zU&1AqKS#Ti<XLi8^hskxO8<-pSxH}B=`fnPrD!5-S$iSCzlZKQct4V-*%fT9_#H_M
zW}=_Q2VoRKN($Sh+xE5AGP;*}ytzF%yKG8~)`_*Yu{V+_hgk?-w^vPUc?wBL?OidE
zsz&sEC8^6DNlXUVW+yz`Vn->TLOoDxX~_;BJVSdMq5tS<7d;}mo1*qNkB4ERfOWYb
zd?d+3IQ3@e^pMzZ&e>^{w)LV;d-;1zR&vqar)Uj@&lAxe%)63l7Jar)f>LBP<xZ8q
zPt0zWHP;&J{UAbvfR%#h_Rk6%NY8sP-NjzjCd9-_rGXt|&8veobUqX=iaj}`X~z)p
zj9(o2pnH1|pVXVR6)q!Ur?@0Ql~D8b_!YfCeh#z^uKbMxn9Htv)cJgWQxD__K91OX
z!8%vBgiQmeZ*a0mBZkL7gg_^`pr$t4PX;?p1lt*QJpt1qVrX&C%)4Txa_3fc7%uHO
zNSkR)oy9g1l?Mo&{~&F6lpVnP=q=A02b(%#g24tv2Kbv1#_3KZ>z=t?I<ye@XTnSX
z+%4@<R|aDh0}rXmPkgH7&xCH`k&CR(ZXNV|0?)14I|b2}@G%|SLT3d#)j`UJnEF@Y
z?9r;h3}mAHo@_2C^_&3mQtZ6KncZZqlLPHa1&>*Bg*t0Tnv2pZtkTrZ^r&y@kvNki
zuFz4S6)bu?`U;Tzf*nU%XtXr$m5|LL<G1^g91Vztu$g?5GopgQx&;6l!vK{Yo+-tv
z3!GVPS4fn4p(Jz_a2Mb>bef?YELkr^Bw}Xs=`w5I4hCRuwnO(tTay=o^83-AmjM5a
zk-WlA6J)oN^2C84;=VmRE-&gut+I-_78^KUD@9mPv~T!mw4w~$&)l8BZ!4f5b6{{^
z2_6r5!BB)_l$M~gBiEY6m{4JA+N=;mT|a}PdK6YnmGICjH;y-OOO>RbC)a7;<brMc
zPN;+tf(9w0fy)w}hCS-tsvK_l*TD4MNA&lLzqzf1aW^6fmXK;|99oAtYabn|iF_Yl
z^EMvxF;Ku<<ta{#`|IA~SRtD(xH-?wDCgwAyr+m?qcH!ovt`E!^4~0>@rF|qO-HId
z@TByUGyP?hJMghl2iiGsY}!n&t)kGV+7e4`pco>Ak-ytmk7Vn=Av@1<-Yh9C2;K5z
zZzT^@+0oTHj5erL-tKvj#|+hG`V1vsv2uwt0S1~JZS1g(^{w;l_<j|i8>UkvHhJx&
zyOwH;dj-<CMDu5eXuv=3A%AaMWRgQJI~hrwV6NN?L1Yw6Y;V3uHRd)U3-Ag#h(up>
zN&EE$CPtjP=WQE)YR@EFSVgjW35)LYPM7BSqwjJrhR!?gKwhKD%LUo2^v~ObfC^gT
z+HkC8Omz~2@5GCWtO<n3AdAY%H{6ByRV!~PO@Ff6b5~uzS26*8rtnbTDV{DLVFfk1
zJqi&a<lQ%e3y}e*9+^%_Y*0p<0lmASUCWkbv(OuHuZh`9i+ESuU+V!ojT4i<gH%!Y
z4{|zB;AFqICPD-7UXl@BB!6e0e(-a^$<5y70@G%I{8S@}i=S_=gEfb=gqY_kjB=F6
z2d~bRKZwqb<26sN+lQf`!k;o@jRT4+HH5N{|2!R!7Lup|B-(srp`%s?09CtfDpMXP
zFUu{tB>}T#%)+B1<nOl#)P^6t6Bb2;#~2Fn6{(BXu1v7MJ_;bGIBiI^xxv1yOeyO_
zZrgeGs4ZS0I5<5bL&u^AA34ivKk4|iMICM?b(K{e12bPYV72XxX&!lbwayM)-+jWz
zQ@627HL0NtBorbQP)7um{)E!rYY6&k^sf(nvg49Z){P7nfg%&+B{L=Pc5g&7t&dbM
zD`s$uMWBI`UDe&&hQs(n8&P?xZyP4}GJVZxp9DI$ahH$sJq}stYndzJz3RXU(vdRe
z?2-rYF1_j}efgjSBt%?qY8J`kD`eG+jW7m+pW;8u6=ruAi}}83q^sNnF6)Eb`#7SL
zLOD544cT<O_Kww0x1_1?C0X0eKaLBhycB#Lx&E%ZZo8=}c`5-z@z7UOhXqNyx;1S;
zm8i}Rh`iAllA#=`j;VR|81X%w4oY~^Es1Bwr8)aX^_YqBIni&c04*18@{00EIutd@
zs+B9cq=BW2R{pEI=Y~3w489**JCCsaox^uwj^sklOH)hgQC8G;dEpKCEK;(jw&i?o
zL4=F;evZa^36#U7VJc$8I|P2qx{vp|4-vxm8vHH#{p$k(5~^PG8x)O(R0%$q_KK}B
zuJId6oCR(=nA(wjQ`);BSxMXaQTRjjQEfjc1ATwiQ#a<l22;2be80HrP8ctK40uk)
z+j!kkXYJ?q$fwB~oKvjVj9KK^=8bgNZGKNys+}7WKyCC#Xnt0bMWbP30FK%!{Am?k
zVM<RYNkQvT*AfRRITYCxKjGMn=5qUb26ziyQuWjHjh#{QJpeDa?~}bqoZWt6@4C0$
zL_bFJ#9IUVnqD?eXi(5Z(VnOUvs{L16<~6VJ()97Sq2}Uyft(T*g6Rj%rVX^fBbhg
z2(4fn0&g;?NJ7%ez8T-))NnKC2lCk6<Fh&Qf!J}{R(oHaH^Qf3@s8ZzeM*)EY|D6A
zoBotqz*o<Z`dpO2sm{J;iq9lMfLeDgt9!I|`%KDuVQt1>>l7NqlDSUoxEg6w_dBa4
zh()T(mGTEND(JX5_V|jDyvtC7`Ueu$J6rLB^o74^OG>l9mk<WM{^~hPK)=tFVVeG3
zI)+J$Tr0(fksbM$SDeuur89v(qEC!IQdO1TMCPl16*U8K#rnvYAUSsM>)loQ*ZY3g
z-HApyQRS55E8dm}Kui+g5x1v)&jK5A#mf7TVO1|baJ>%`D?rUujSH=aMi-1|*&|}-
z=hSHhZkq43Xc5!sTnA~DVl$k_{UovaG2<(~WfjMik2Q|vx7<j!2O`@QzV{Lkl%ya%
zJIU4a@i!Hv(WG(GiR{B1SQEgPOPih>W{Zt{+!=vlt|adswc)domyw~G4mjkcBLX$<
z%Yxs2)uRDmX7FeozD{;O!d7j66L@RNL0PXK{vQ<-3lj-gQJEi7ev&y71t=Ub*Tnjm
zP=|0Qlq;31OHn1vLTT<XG<F;|UR2o|(beq2NP4ZGE{EqPZKovSV+M)RosXMRgr*=~
z{*(`%=G<HV(LAG_R{B)$gmhqLvZ<1vZ#$nN+7<{k^ux@4^wTzfc4(8P`v*NW0rgVa
zsqBkrY(lNb5>HH|b?YBrDz!zU7EkqtKb82A&&$U{UJa{L2NfIzcnJN$gIXE+3171P
zQt$pa(-Uc|i?lE7QiR$MH7D5X8RpO4#6_hN6>mS#S^ngHaqHm@ZVZwO@^pWW29#j7
zoX+T*D~r0{Pyy{KAy&Jk=PZ=<%t3r+5MT;jLLU8)8};o0^@8oIW!FM`s~tgdw4>;b
zi)W`+uU+-`TgejWuMF@5McxLB$wh_^9jdVwfFZIr@Dy=Nq4H0OoREu<WXo-82vw|7
z?dHc>kSPIF+~t&!Uh5Iim@dkm9>WdK-wE>7jOtb0%!#5b;)3=LDr-9aT0+8CODz)O
zA0M2Uup&iW-q-W{&249-fs9$hpqoB}By>}Z$FhT^o3!AD6Q$vYnF&Alef^}lG4io`
zc^-7#V=?ZBw))gvnC?iLaRD^{N$GF~ky6<VcN=-Z5iaRhYs`TT-tc=nFa9*?U^EBs
z2hzN+=4h6TkU2c1&q}B~i=aAZ6N_55)n$7cWfjPR7J~tj+;s9|YO>$boQ#1vMF_R#
zaMkp6+c#7@woC^eFMj8JH8|w78z#SuBnYr?I?kA2Ngx5WpqY@2kgnWfi7*y-jkaPh
zAk7&TWG9SRc8Ec9j9kbW^9iR3_+?A%d;~gTFZ5r#phkO6L@bey0`pajJdoU>Ma3S-
zm3mQ4Q+?0#^9k$e{r8<{){#y$b`{p@pyP@t>3ssT+#g7VE3)1j=gFt>q<w~I{$Yo(
zI0Y-s#P8<R9bSv(x=)t0c?ccC5cZdA!%-+tQFj91B0roY{xsq}$C3a@K&}Wb@JKur
zN&M+@$6MJI=u(U=O4Tm&lf!hy0k2wLc(!7|(JGPdhJ@8tSV)RmH9eGJ@?KetI1~Sm
z(CGvuitKaq_T`?P^MFz%<<PS&J2|REMLnx8{Kvk0jpcYXQYJw-;}To@A2}v((4S?l
zJg?3Ev4=hG8pe<)iazw?^1h8Ukl7<n`$xGvf-r1GsG8UGrQI6|%&McPod>7O?9U@X
zOAnN9BSHbf>O(24=}<Jq04w2&LYhE3%LKw@KoEstIU>3+iCvxc{8Tf+Nae3(u@$zy
z$sn}-&Z#<jj`;*!i{74z^GA4$NR=zNL^LP2F0_G@e^vm1Kj{4(<CKgFpL!~#(CSJP
zgba7S0SA`vhU^(+#F0P?>>%zGd@mX{6hI+F286S|dDv{+k^Eds^hvV8e(YA@j6N}<
zzGQz83!!_-rJ|ks;(f7V@}xU^x+9jNtflZCqom0$iB(LT#AWF7884#c!)CX;^@?eO
z+=p-QxTQ}cMlXJ3m$Z3@P1#klpr_H5G3i<F+!?gsJVbPCwLkszRsyEx$So+MiI(=g
zpxek!X4qZG-Y*jO^{p`4Bq7UQ*xM;4I8_~7Y!(+YkuwN>jek!pec8v3sc@|GoOHi#
z=vtg8fe@<o91x=jV%G?sTb?fIx~+eYqS0eeYbyIWF7?vDk!T=+@~*l$Q5ek`Tr#oG
zZ1$<bKHi2ZL;l2G0QjId5p{Is+NBm$<LFH0gw1djAD{FjD2#8aNA{Xk4h`#p>Us&`
z-$0+uE$+YR*W>Zms7`WyGJzaxniD@T`x*M<v+)Lr7QF<h94nZc8Bic5jJ^~(X>IJ~
zT9VRBZh5Bq2`>hs`|jgomQHgI3r~NkX9+8^@+Yr==k-AtGIR%$>R@h`xQeHAKqt9i
z247wqQ>zKamaj$HI%1GafY<Er*BoQGT}Q2vS}$CH?hO9;azjL3jE>cbP%_&g##xNh
zU&z?)#{OsU1@j{vRsuWRyYTJH3cG}ye&3V$o)n4unixV79SdMavBI#sjU6&-u^m`I
zP5REe@6X!vo-%Q0AinNP#Mv7;I8O#P6ldy$w1`-m={HL^i-|zBagkY@Zu!n?^E$)e
z(4=}L=6$!Gf3$dC&yz3M;(V4m!ec$!8D3ASgW{Tg?KodvD~k5%F3HgX;1;!QI)HZN
zlmc4h*Vo0Xu~Lk{3b|u<5`X{4J&7h^OtnwOANVR`sb>kg+XemJZ**DYwVcNKvi01Y
zq!iU1ZgzlPVKUr7HXNp>#TO$2B;HoOvZms%{5nZivy5Q0Cje$6quik{gL7P`$qF>@
zxintMJ%a`&=#q|yiM7Ox!%S#|o>V7f;<A4dMB{o0gixGzI5ppq`MYLy#=<mTG*nGL
zJbYyu7Wr;JfQ#97TwM23P71SUZUb^R;P#vsN(C7szcm4Mau-Xs5y3wb8xY{NLPn?F
z;9eQyvY@t#4b(awowt{Cs!6u}H2?i7=#Ts*I)eaQa_6F{qzgyuNMq<T*AEd(^fsJK
zK2IU1b6S26mpY4wW4<2!MlKqhc{{V6xhW*aRth6~Oh;AkH(bTmzLB2X(4r#AjEO&O
z3c|CY4E{+oxRC@g0oNWqYG=NPo1i6fd<p!ymZSHbdUv!bX-|kUDZg|DGbcKgUF`Fu
zrpcQJQ>AP1G9Tod3Yh;mQa9g>!mM3!&Md`jC!jxFpGbM%`4=v4`P@_32!|Kk5@Frl
zr}9i-+32BgpIy5-^G2KX5^|m!2CeA{tkAdaZt5q7PmdjDOX5JovKDlf>jaayk!**f
znfAXUFT9dk#fahqkBQx{w0CUk(7Vn0<LF$vl|}1fd(<REVgW%XnPa62GBt26+;~ok
z$Fb@r_t{vL^S8|14?&UY;@E<K`6%YOh?*bt*P?(gJ`ZcMe<G6w1IX%Yd;HG2q?X>G
zx9u@g;F!+Jw?4JQvoqVHIDF(Q3j>~0dlRRXwGkH2(_1o{xP0<#fWTX#E2Qm)fKK1i
zw`^clA~ZEZIH{7YZ9tF5c5x15%kQyk>~r-{ey$4eIL-XY1pHvk-vDTGNPb2-pxF0w
zggWOO;&_1ZyH0M5x$=Y7PU7@4N9>9?aX5o^qc6qNjR)_gNWF!3f^r+b!TwEenJeot
zi?`sfC{Dz%ud>n)BmxA=k!>jH7Km^35;1iGj^(*2Q5*glvStNiX^E1RFIPpt!k(NN
zb-th_DWPv(r@G7*&=M`qsTz&nu&@*E8BDT9yuKq#0A5`_->;?^_R2QA$1eL=H#5wt
z4l0)p7Ca<<$v`6iwFtB&-7d@Z-~;$0d8pNVFzuL6BdX{)EPp(ZAPru4qHx6!z#Eo_
z=k<M2ro8rj3~k$0*510tW)8R)lEYEBPJem&D`aTTt^mW`@m2J#G2bXG1z}2lHTnh*
z&6IG;Io`oHUM-j*VoCan9Rk;fWD3@<8>25}T-Aa!Lax48yt<td5&dX#O`_83k#?}a
zn;IInGI5eWRzCl$x=Hs#pLCS1%+&z-TPoH7ys$%1mGP7-M?W-_VA=lx+$w&l83E(G
zVf<%0ID0^~v23AxM}FeR)g|ExJyi}BT93%}{Fr8Ai%nt!OnLnY?yahRwA%9sL5E)v
zJE0ubHY!tpvPaXDpmAmxMteV)^$cGOl=Rs`k9vdXu@tv#qc+AOb3U1evG_hk+EmzD
z3a8MZyw<%Sp?58|NTYIE;5`;>L`~_dx3<z}q$5hB@gm1B>Tf|Wgl-6pC@6<zZ|vt?
z9BExTYBn9xM4%uas`G0l#x}GnjV{ifP1PBB%QMr7dAVMCL3%AVU>{^6CoM*)$AcZ7
zM1}86Ml<ut++o#uwFNh7KEeqR8#>3kAtThr5K-bx6S?NO7<J##UTzc_inp=Y?$Xi1
zzd}1V$eEh2_vw2ZA>Q2P&^vTwR38364oR$AN{a#i{@2ghSj)=^Gnvh4_Z|B`N8`@N
zB|MwZup5tGBklWnkjyr;kEm_vxf^OqWA_qL2(_u)#@7Ux@s{40Xr`8E)zvs`hWyT#
z^}R>Lp>R=QHmJ4ZwbhRI4_r-cPg?j0u-S0z6R;Q0E?YIna;3!|*Q_1GF?Qs9eHd|i
zW>8NKb@kQb{18KhAF^Tks5&{8uA&!<=DD$+Avyd`!V=crO<G#Zgz|3vVCd%7Hzlxq
zMN^?D1<^|eWUxlQo0B0=Uoi*G!mU@02i_rJByUSvKXG=L#WyvkUF<JQnf~zeCN4Re
zsh~JtisIV3k10Xc#AyCOo<z4fS~A|=QL_JaZ&9O7+BpIPKEEwuE5>K;P6N^v=nrc&
zzn1`AGkmvVh!~4sV^KKI@EGW!aefrVq}uy6y)O6aT;~Tq=8_K)|EvP9#G5~w9!9+6
z5k3)eF~5v2ktHGjteLTUNcXJ86@}NCfONKk^BSH)<sOY^R!D`Dg|k>+j-I=!s@3&A
zyp58MCf1+R7OZ?ufpu@v%G@H?aS9j3>8p{)&#mQxe5-|2x=V@r-?<}pk_{#|6jAsy
z8(=GN`C+uvga}ebfrc8_OU{UtASjV(t9z5_Sg_u6;s7aZGi3Y5N1{0Lf}-P-TV&`Z
zSL7;r^t#7!U%szi*wkEs@C%-TUd^;c>Q4)948Sy&?&Nm$!ctv{7mwyYfxaVmzBBx6
zUopx0*5d7!L^A1rK+#{H^L0o&{veSlr|0W86cvtjexs-`yvnC|X*ui#;}eP~fzsqA
zEHjnHP3`wt?Bf++SgWFr1wY)iw)V++@wfEXQlMT4IDn>S^UJ$k;!DY}%h)K3)04Xz
z#dF*BH$vD0i&7HQ{y*NJ`|jX`PoRx*oB^;+l3ot$Nce3jzOC0A#4Tuox)GbYJJul7
z$MH-jzWnY&xeZ0&ElIbkJ6$skY3#r>x?@*_lFU>g2)~PAd}x*Rp0O-nOS*npirRhF
z8tZbYJOuhewp=zc6n=z$_q6HJJ@H(`$m83brsiR`db~>24DR6%#swLI3^=Hxmk++h
z@p=+6cn{&wM2?}%DwI*#hwr1}exbPN$o8w~o#kN$>mtUaQX;$2Cplk<uW80Wc5csK
zunz2ZytdSrtVS9yn`A7nztG@qeB}7cdwW<gk4l_TYuzI$CVDMQ;4bVtk~KWmRU$K?
zP5bu6Wjm*ddxt^70$nx+WX=-Z_mv1#3_T%Y(5R+`(cej}G}&7G(wM7wD)YvQUO=^4
zuFwmlK))tmxc!66?a}ft(4}mhfB9?HukXsWUf<1&W7#mDTCvZKvar^vI)9oa^?}kQ
zsXo0zhu$%!KYJ<ZxeGg1R`F{S_+GSXQAX@Q^QdnRld30(e&i=&TQb>yB>o@zVkX^+
z?yyg1e2Zn(1F>r&)`KCFWYf)R84R^t<qsXX(K1a0a=#4Fr^11%Y+_J*myhowl_&@l
zG{g0;^S^xS`2c<FOdZ6^BrZoZ$8h+hx3-7bmJ);f)StzjpWSAA!uz(K1?b5F<EE$;
z@nGB0^K^+@JMsZSees#jBKV%Re*nW!!t}#al8Rf#n3t|7PJe7d-(JD?OzL{A_G%m8
z)tOak92uH86yK%~^S;bkpmb`WOF;DR17HQi2JS+mC|(E41q<3#NN2!JDTgD+W278`
zG$0QlLC3iRQzqJk(M$<Ia{nsp<ffnJJAFsmf-mL4yO(s-k1{h9fuq#7MG*fJHuSpL
zPharNA{FgYOi<4Jb7mg$cjxWE&NkK$tDt-WGKmaI0<fLg$5`S_0zI#%rW0+0AIc1W
z$FF@HbPA2DQP0YsDUIl45L-I8sZOB67YHQS$yjB5O-T{4KTsc=I=ZRwbwbsURo~Z)
zWLvl3uSk6ky6hLPZcB@jYA5tzhRHTovhyH9wlo#4qOoUPIVTN~nW`x!>-;|`<Kr?Z
zhuaZ}&vw#MusBGQUrBKn&PPv^vCxTHn}jrC^M@TPURZkG`m7Z^E{4>zp%5mQECr4I
zD@7#UWDW3<1jtJ;#FT9K%Ri)<!tiQk-2X{9A344RF(4QSw0Zem^;4+%=5y|JBiq+P
zCN%`{b#h&G@Z19OiI3<Z1X<V$tztXvN~0_zm+8;VdGv43`<m7z!bw9iPsTE+jyO39
z_)0&?@=?7ZS9~1n`fJ8`TFc)w>>#=*@99O!NzvuE!+k1X3kS~-<OwtbAKJ(Hgttt|
z2iU>oM##3Y-Mk<m7&DZujHT|<7qvVLTbjmWf74a{4C=@t_~~ShQxk<i+(`UkDWx^)
zs<(t+ZJt`}ds5P^)wdZq&{$DLk_Glxifw9|uSxB&ZK{7TB8$=ur0e$|ReeLNLV`vB
zl9_@%jU-!;e(060<U?u%Uzjv}e#vc;GN)vAVG^F_R}JIccjbXuPkE_t3DMpPvSMyA
zQG;vy`l(^=i9#C>5`xN3G`!BM!CZjfoJF=m_xqye3wzR1&!FP4^S6C_I4<vHUg@lc
zI!%7}Dz}U1n@T<k$<o?pJy;xv)iTl9g2;!oM3Nihx^IuirMr{1jT)Q+C7W`dhGEp9
z|5%<(dMMCg3SNK7Miyz{!_e6vD?Z1*zoNNr)b@$GfDtA&b2I&rkAb<Lsf<cR*~dD*
zuFnWK>1FbznlFWM>Jh@<5c!C}40tc_K+Y<h69sW)R-7(txMy5V?^CJH?OwY-xIQJv
zP#ha{OW!U%q{44Lx^|v$s7UlTqk7~e;O}ALxOh&7jn~3f#)p!Lo#120^2NEz?~HCH
zboAS-6efrrIlLX{XDV15kR?crP&g^bZs^>Gf?_>?vO0(GLCIY*`NT`z(S<V}X(AS?
zQ`)?CV7170nJ!m)tq!7$t8tx<#fLYwhqDiUr1u4($hEv<=M8DlVtV%>n(-__GXWNg
zspM&&z?XocYu?+Mz?-D2RS>i6i}6mITsF}!-egWGr@ThJezOLApajcRcZ#JSTvQ1x
z)4k8)s?Kw?y&5jQf>~x8H^?qyu5N9+-lM!f<<bcJFjwntErE~&dGy5$dq|WtLPH2i
z@sLz+ue4sDrxMJ;XQMuWl`?E-cIa+r+eQ}OE^dh~Y{f4t7xbxm@k3wF;EC*s_8X2n
zmis5nFI{<VbiN}?U{)i}Fp_H=T76@bvNlo<_7j$wC_WV#r$JARlFGyKk3byZEd#g?
zeo>;B>p>})AZGzN)Y9{CBP!-G%8=9|ZEPj3{f>ae{>ocT)KlnkhntHghDkoCHnGx>
zB|R>rBxul$S(1Kw3zRQ&Gl@MvsAq9Qigqy82=1pPd~5il(XQW!mx7+=C$SKDkX6<2
zCCQgXl*Ig{yEdAO5)s0ptsQKcWWNr+BWY11Q&6Hq{hwkSup=CvrN%cou8Cgmz9(>o
z_Z1e?Bh<`=-nfwnrb5yBmN!^XAyuN*=Q4;~r|;X0VmrT)z1B>T4zDqb)+(yg9E@Cq
z))Ly_xm3egMg!$5B2Y5`V+Le^$obDkM7dZN^pT}At~kv-^)Qag73vU$gO6kBHl8vG
zUbsp|q9Us*!I~a83v4VI@`bz^%rmy7SVx8l(erlBB$8akUFEIUWbl3{bW2J&!>vhf
zu?l4fWXQJpaz>2fviKPYtwoK;J=ydpK@J3((L}ud8yzP_s}9{fozUuqW8D|mqh?SF
z4h?)M5B4trONcByy6B%If^>Tw|2U)^P#5Bs9#qn?3HF#|oFMFWt{(B)ht+-+m2RtK
zji~h&@rx*d4_k@Vg$c^l{EPM$AR+nz94}Vn?+{IUl-2hNE0>1u_yf|a2^=W$bpUAR
zfr8Kv!Cg6`5D%Rk--*N~xWCG!Nco6&W6)AlU(co-sa&xT)-5z0=>R&tth^&T70HEC
zp*&$xQkcX-R8Bz><~PD`$4X<A<O&l&&T`W(^59#a%N@!d&C)iZM|k->FG||9{kN6A
zl<!AS4r@k{A()Sw{N;jvw!fLf$E0i#<Ucq1c0{)}r7-=0!X?`@W8h)g=do(5M=Xu`
zUl<|;%)Q?J&6KK!*SGKS&m>=@r%4sT{t}l_ZTI3bDMp8uo^m|=mL&f5W%{pg0k={e
zuxX))<070=nG$DOmu>tZhhTDt#VAbfwyacpUO{PbP*9zXCqjiDNTnV;1~|k~3f9?2
z2mWO(urlqeH{^oDjlB517@W;~E%nr<l2a`7-N+5|-l&s7(wL{!J@AVDc<3n0?%O04
zzRc^$P^v_-dSGtl%!VEM?L*4Pi=v14HWwxj<JV@WK)iWNu(TUFvtW|yUjk7pOqVc#
zB-s3ONqRI{S`jWX{rep!HW>{A?`W|4SEVKm(+mjscV|$tjf)$*?HoBYvl01l2@R02
zh`15Kf1qO<{Tb9dR>qpY@mK3g2N$$eP`Us++&*m;qwSBrN3j;n3zy}%4#Q(l>C57e
ziEO+1+Nk(;+%L`vJ2e{B^N<^n7v0)|NiBXTSALvMJAu52PbkvXk5y25zzF|@$$vpC
zn{<HjFQUZS$MQ32pqB9okl+imSRjFeL6c#L7xb4!u~tR0^jB{L1))d@amx}R!pitH
z8-yn99>~4<0_se=jh%=1hDDWdB$YfCZ;2)7cgoC|4imY$`j=f$h=~>`oV&y@aFnBG
zaY!1dKAH`dqdUgx=>P1xHX70;%aZu5{*P;~pVXwBI!&Fj3H!AQDwg3f@Ij+aASp#N
z6>lSN8c92@t#wapj*BGu@tPOfGt{nCb(R1!c9!dwG7owm<UD+qJpLiu9~_QiDTQEk
z%Vk_j*{zTi_o6&Ev-MCm(Xg&R7Q^2n1Y(xpYqmC|?Zu`loVXlIzICY<?^7f)O}zRy
z6%QCO3wdc8bwa;Cft1mqCCE{c@*L9DPTWCJ77Zo*0&>g-2{py}VmVPR=crHk(j>a)
z4#*oW*mU4U^Cafy2OcrL)Dx2ST)iwdrWNBCES^H9faen7MXxsO`5Cu9k9i?KdPPZr
zfw|#6EMZ9JoZOL(dPp?*dptmNoz51JBIZNf>B6Ud<VQG9a%3qnA2G=ty4oMaI16f)
z9x*`0P=jkqsu<i^=as4O*N_=r!#Ti6S%gXWj8mP4+%~PAHb^9_ThVU22W@F|Nbghu
zb>ux6j?F;hcpJRo1p|_U9tI3%6?C%=^g}mnxArAIt~;8=qbCCu!gC0<>zK(VcXltM
z$aUFc#M`~kIEq-+RelU4UEoG?cvqpg6|mW!W<H<2Fd*v*TnmIBVsR)O$5z@yt{BB6
zNraGP<%-m}w0e>HfFDlkMB*(wF{Oc;*91tual6xqk4Ye?Xf*FoS>YerZRN~d&cQOk
z39Q5Ne-ud<bYZ*O{m1I;UDqUC_vD~ya?aRIPD)XSMaucF24Pwg^aC35!G>dRwbum)
z8`Vw%N#2&Iz<tuLf8YJf{s$k?fNa!u*)ah9`M1&0mKvxW|G~QQv?OSO-YdkTJ`pQ-
zNkexD#IIBBx1UNN6Q1F2(7fId!KBUzc~jzp|325znYN4buW^Xv5opj1ZTu09f*Lp1
z&YKUOM1MS%ER+rMGCA}itTMq}Hw}Kt*pJ5k$od}2r$kAufR7vMo~JgP*IBCyZS35i
z*HN|1)6np{Lijhv9rTH(WSKL9RY!@<hvFM{co~N=PBji-$H52KP6OdJ<0HM=*v{Q4
zf75R=0E)#jhjokKKcrR8XOFYMWLRQxxjOM}7SCD@uOT}s7Av}g*XD9=93G)h<j<LJ
z55Lp@JlIJ$MzfSS^_9IeUSJv4IE0CzCD~qvk<ZV@*3z84##x4Dk;0pqyMNwY325Q?
zu1N4DIb}UYCH=PcM+=rzB^e@81(7#WOTX_URm)lp?|FCJruDI`5J$C2w-AJjb^o%|
zq`1~<I<jdGbMntIb#RD(t~FJdAkX?PVVNZf8f51g1#W9DZR)Z8Gf4t#6EAsMS~!7)
zAK7_(utdF@J2z$q)9NFtcY3uEzV5U(US&QdWPa{G&#JqS6OuW^=Ut@0f^L;@@nM{v
zpMOm0Ba=0bU>$fEo(2Q~f@>Q#x_=cy0C+!9OZFZ?jtWv`v!Qf7q!ofjGrLw81bn!V
z)eO;CSTh*?Y|9e!4BNkOT+5Eb8aMahQyN-h+tH#k?a?_t=F}K*(QUA<o)C7Uj$Wva
z2};;00Bh)3Hu#hhCf)wg@=C!RxC%`|u+=j6XNS+URv9nCXO+nwn6;{pTz*$#v;|<j
zBE6>jsT2Z<YJ>Z`HTI5K#s5I=%xvRyFN@FHYp)(N@wmG9KC4G1WlsZv42ZLp;6Pzn
zQCF!OK2Bf}__&C3d3rnzqbi*o0tjMF3QZK1VF}kaoc}sca!qpXzauuyKI{x$>tP3d
z+v5GJ7^Z%WT4$B_ouK!Oowz9)W}@x7PYve(BlDxApg`y^oHj>y+I{y%RTPgi9*cx*
z#Xqw>t)Ok7k6JI}mT7AX05qzg@=F$ouK82_`nMwV<+-G2O?c^}Wf4-XzMKgaN1eGL
zUBu}prT9aM=U#(ddTl$;i`V#PIpPK1K1R8qnBY2)-}AxAXVIb^5EPc6!ZJxLBq%pP
z+;-v*p3|QT?9ddZ$Qsk#AyXN~U&{Op$;Z#bF=9gUE?26cc}hL~>oW&`6!iei*{N?t
zp@~xIO$!nZz5Z%+xD`dZkyLrV|0Mm5ArWG$A!*2^%o>`f6(@*GXE=Al+lERJ;`3w4
zHL$aKX`W#6giLlaEG4NRxxQIa*8iofuuvPBhnO8Ja(wi&mc+p?9lM2j+1c%PE?zN=
zRdg_%<p{Aa6U~w#+!98M$|~xKLcWz=A37ncBGc@ZKbJ{eYcZEb>Yq?Zwx*y7HRq{<
zj$A7%-dKQZ-CH<3PI$I%95LHjWX<qjo7u?I^24hT<{~>svZ8K{l32S{t5y6bT<2ap
zR@1hhe;anxhnjP&m(aQ((u27cs%{IOJ@q7P&QObKY{Ne&*NHQEal`6S43>6JKO$qr
zoW_n7fr=-)iZ5$VqrYWV3>r$=&fPJs*goC{N3*jjAcRR?MwXB@On(qoX%s!*l1jMl
zIi%C7j?DN=e13uCpHYaHmD1=R<>R0vB?4gu?$sJT1lO)9rxCg6SvuZ({3|s&i02z>
zSmT()&F;^4M0>=BYw^$|gb+SE&H~}$sfIKHlfkQ7J^DjAi0`zb)E%%!l_33;62iC2
zdKH&*E8K#Q7r}}vvdx}l>V#DKTc3l+efyg2;(b<Wk{(U%EjaWStChVfgPyvJ6|2GU
zJ+E@epeS_AyudYdAqy=KdF24r=rhtop5DAHs+-(nK{)glhY97e{8A2PfcX7Zi@82z
z_i1M;R}>~jgG9-j69`=kBt0{o+H1{VAjHqAVnbUUzbCr(S6_FzsO^$M55lxA)p~&k
zETD~hMITWQ@INs4ULXSf)qm;Jf;rO=@-gtyo{1M7%86p92}~?9?D!J)PS3(%j8OJ#
z+KK_F)x`GEj%*!tH1RQ-7e){|mA#%(xFrz+9m6?(=NNFgcF*9T9mzoTxb%aU#uRq;
zTlPx#v#h^BD*$Uh@?$*Cyx64Mch6ToQcw9_Jp;PXtrksZ3Ti)>C3(J_fpGG*D+{pG
zrkwdW*$U4^pW_W1Yfa!>Q=-CuR;4}sc{%q$d^^BEY`9&I93?5sDBv#%r=!tADS%6q
zj~cmW=m}#1f~F7H*NLbEQI@oBbfs4qqfIyOc8Tm9X%inK-@?vyjBR&euT$3E?r}H7
z`izMareD6s`70A|h=ri)xL3a0#1+iWL2+*^FbNUfJOy!mj{)y4LDgYnX+<uB-kgc}
zAR!pl8R_tqkXMD`X?b6A0Kb`(*KqQ1#D(oiBG*Seinov9e^`#uvVH8jZScdKG-^<=
zl?r?;jk6@>9Ui!Xk5{ftdD;4-sj%P&qiOh6-%-oJG7*ntp|KO~%SeWY^9?1W=+%Hq
z8saR)s;ZsiaT$ATFxZCyhW*s_l?UqrW<)9WU$s#92y|Iu#UX9x3_sFvyTYSrr}AK7
z74>`f7IG=t8-LX}bBA=96_eYAZ9sNsA~mfM&bL}_l(=S(M_cz%S+u(802=NQ4QbXa
ziiivRe@2uC9Y*}5n+|Fj4r*`Tbc0G9B&b?5{$}dNx2l9YR^V;y!sheTYzya8yyS7x
z)`q3WKh?<ee5yr}4+bRd>+3`7JXW=p`bR)l(RSZyc7>m|U|O!|HWla91(@PT#!9)w
zqmooRh!Gg4>mA0=DNRTXvm}kG@FU4h-SOw*A}+Pl+$}DWZ4|G~P)DW&>>oLMiX<s|
zPlHPX{kv?bSS%>c1BZ;SJCiF*cPZQnA5~B=nfqQGe7tCu_0IwebmvNT@cs<As|JP;
z-E5`5Ko#e}y;xwx6h2nn3eLDou^4}k%CrjbqE~q8NW7?^nJ+2(daDvc@IL?5%;n?e
zEWpt-tLow`g8SAZ7QOFCr2$K|ji;D`I#)>y=3!kQ|3(uC(I(ws%?x43B2)hu`|U)C
zdiKE5)4?mVgGHsOgSU`B`jMy<oGn(NWo3FIy8(>Tg>Q?c_=*&d-6?|3BW{F8M>%+4
zYK9Eu+$jAf0BUVK?G3I%GEWy^{#_m_^XJz270$K84vrqUEQHrTEb~t&7r3)I<b;j!
z0dyI4Zs3=9-A9@s@`uP%=q=rdI)b?3a-*EDo|HYvbTD`qwZn~6Xse*(5xt_Y2IzV?
zA6GlQ79DVoE<L<(Np=A%`4;AcWL^_4@?Y)>G%j(Uj4A!9OMPQ-8U*|95Ufg-uy*-V
zA&pOR-jincY=j?Gvw;!`DBQCl^hvnq?7=^EM<L&7G#9L!)-41YoFaCqZ3&hjIFO@@
zoGCh!Y0H;9KZ@xIe;)$Emr<)xh>I=gjLyFM)b~}b@BV>U+aoUEgqrCJPSB*^9?8)8
zH$FoJ?;||9K`YrB-s=Cy$Dtb!$at6gXft9Kuc!o7Q*l(B+%d9YRo4;{aht52nW38t
zpegIVnKI~|GA0Pxg|!xK*GH~UmAjoy20?c}#h{L*Eupz>XkLOAA7MmW{!4mbo&tiB
z|B`5je%ef$JG^=Q+m_$gYDwvZuLvSo^=MKQ0P7b0t-BnF3G;2&-_z8ZvR$%q$f}i>
zM<UUziK-i*1sZ}h6t>0;AE|5|#9!;AH5TpCYrO=TSir<R(0A0PCyE$~JWYL%nb#z-
zKLb3-S!tY0#3_Plf61GRzDI$-`>sxgUD&zElW?M#r->re=|fNItN-S8=P8*(DJGJO
zwD|CF8)oPra1EAY2;lgFMIVkIa){nZGrEvej&t$;*j|c5Rybx{k5p5di9SglVgLH0
zggkr2i)b+kuPaif-HJEF@Uvq{MTPqFuHk~>Pa5sFep(vs4VkRmH@R$mwyI;qajMv(
zmm?LT-}&$#@2LleO8e_NQ(MjXb1@;`#p+|b?3=8&d=m8Ipd|h3je-jd^vxgGt3S%o
zI!FnNj7=e1v~t|JUT3vH`W|cDQ<K^9G14Pc3MJXU0P%dk2rSvKlpcjue=T2q0K?uV
ziMJ|0WPH1!Nziv=l>di<0wABz2F&xfIlcJS*F}>((-!!idh(#fJOE9=!;!PKacMYy
zI2YFDOn#ZW^o@+{upiIp&%0s?P$%(dj=*ZavRvU)_#nn<SH|<#h6M!~c|#3PD=im5
z5ezo~+2pTEX1GR?Y(}!gaU+hTdM`ME-gLRPoH9n_)?2l#+eTr6V<yH=U0G_W%Qe<a
z#T%GKpOea5En;8c<(?{VrJ7*$-C8<%XA$F>NSHHRQ4QvpDr*9X5eKw(CFp#B2Y05v
zjTI5YwTGIMKxk@1tp8@)$Pa=V?>$PN&9XBctyh9*(nPu4_Q&0RfYpN2+YN`=2SZ=q
z-(&ZIR5B?^09!%RaJO^1PZjj){}2PU^Z(U8fN^CHUfZrOzIdXCDt-C4GDXz&XDpcS
zBKovKX=ox<PM4@A_kTV1zb~}1yH#qZcVM|M7PE0t2GD;rsSto5dmtc>E4jMm|M8%?
z0fJjt<^P)Te?I;X1^$Nu{~uC7B%|9WKPQlLA5X3B3i>Uu;9WK2zg0N@Q|(;KeEd`;
zFrzrp*jP*Y@0VrF@DW1&#orUmPMvQAl(iqz)>f>abX5csJVu$mgz#oYo<QSfm3UOS
z8S3u;YGoi)Rs*{$8A;fG2&_gHRQKXpYq{QkNGmNYR1jTvjfWNd-y{6RC7P)2#%))<
zivQTHLLyXlLs`Ft7U6#kGM$g=wwqcn*N^^pDJjJSR@)KA{wc}0BP^E1A#G0oeKS!+
z12gZ52vcg8<T2j4!;atd+~?oVI%Z!nOQCOtX%OTAN1MN31)yIGoEzKy_#g?~;LU-K
z{!A*Ye<fzKUMObcB5f8vTO<yiVNId8<?(pKx-@c@wv8N`CM|NbHT<;q+lh$ZpNn<M
zHh(PM9UkYi?91g33ZS~?J11wAoOzcI+I@TL*DQ1N_jjQXzrVMT(KM(H7Zu-T=?X?^
zN!&cjQA!d@B;FPJ%3DEGm+(Ne-<+}Cg=50wlpJ2v@9J@h0&!m9eWvxWw97t@2zYu<
zv6D6}T3Gc(+NyF)I%xJzguv7`d~tZ2^_=lMaK}@Nx<<25f+^NEorao)y5>EuRB!fl
zTA`S8gH%HpRia;^<I>MUu|QsHd?DlI9y-y_1f-Qi>yuMlQ{j(`Uh8exWEK1vDioCV
z4Ls{GhYIvMgpJwOTjUg!)KIu!fB7SOMSq}{ZQI_Z&b)TlF@3MKG^O(m^pdD{CWB@F
z(%N;)Hakx~B~BWHW0p-30ANJ>`(FT(W9J!zb*_%{gM@QdnQ4C&8_HiZ)q}^Ksr-XG
zeh6tci_=vr7?}{koM@4C(Xr4@axKc*g&QyuWHHQC-I)VK;EhI1D?**VO4?#4gt~c%
zH&+58x1}V*Ki4W%^yQ>C6vavhjGXbLZgFrs$<YVv9pCG-x@4d3W}hxtbYPDrctDU7
z`6D)E=UoOE!#N|KMS@aewaN}d4Yh43U!{_7Jrw)cMsOlQeUYO~R6Rc>)!(afwDE@N
z`}V%(i*z}lsFXlm24`P_dU~9UO?qNe@`H?b7NrjE%4*(F&m)up8VvrPjjC(;E#_|`
z-~mV-)2RP4?Z0udWzAt{W@pH@ghcSAv=>S&%2OW6w1lUVBjr6gu7p)wpNzcRXBK_g
zH)LKBRgaTZjmO*-e~$X^0l}h&FoACY9j6MXureo>Ow}3x%cdsYC-?OVjmdX+QTNwA
zo(IzlgS~DY5{|eNDR)|hQ7f+(f;NSfHX0Z<-UdGRugR_*X~}(lE`Is#IO9+D91;QO
zo_6jetZD2q{8F?cLS>4|b{^VO-O-(nIHq{OOZDA_@t5uBp-yTuNR2dL0;YSG<mM5?
z&8pZq^93OLQT$PX$*H!&sE1G>_7IJ<Y~zXcd|s7WAK&g|h#1!<Hl$@_%l5cM@cW-Z
zsJvhF9v-hK6us!9LoC}C{jb~u#DsNze=ZUcPiO01|C)#IDC~)SMPj{RFUyo{idt~o
zpxx(+t}69Goo|VX*%dUiHx4oC%yrL<2A#it_N7YL;vv{ea2ni_yPd)xB3|*Oe7@;4
zn>Q%tDU{v4V}_DYZ;5zTNH$Pm*hftw=d3#e)7sO#({qK;fjhx-HCDwz+hCPZVuani
z=X9}I%$Ko(2r@_KKi`b@7m=5uaVN}+-p1JnouC_eXWDOz8EWhA8odiA0xj=@g#DK)
z<uwN<E~=>#RJFXH@_UF**(3=Qxks#T<QfQ7K+5|Z`>w4o&4d^R7m|lQL<ls08-)=v
zjs^Dp#=eZqYv898us7*_Zrx`vv-j=t!J#Z3&&)c;gd0`hEXd#-7-Wj!1n!a*9_~BR
z_Y|VgJ!c2fYZ(LOMWOB`9~5PN1}~a6UNOONJjhZR&gH{(reE6bypzX!!upfIx@P#e
z(;v;)I_4Wemz`5qB$qdh<p>4u^QIq7eUQ)McsDK#$4qKHq-sC$H`^GCE!y>JE@&~l
z^kZXwEl!tvy}!B(&?vF=6cSRIP#<MpctmdPTtBmT`0fwgn!y4I<F~np&+|?wY3r|#
z>HT6OL%b&N5v4NHGh1*wUdXcZdKVTaw-4>O0gX1pY!awiWedG!e;@&%zaE)REV0<*
z8Lr7hmh9&et4A1{DSY2Xmp-n9U*kx}f?XWD!m|ByU-ldH)bH6`ZrWMTsw+sp99*kl
z*ybqvbXTRa-S#YpclSK5+*@e)-QH@cCc#gUw6RqqPm^!At8kxI<+iBQPF}UxK5q(_
z{(1~vpY^RJ+i%pOL>$(^E6-$bM>6YIyJCH|#-lFc-ks#<TjUVD8IIxPK5`5_U{GD0
zICB`iSU_GXdsEJK?}@BDa7VoK6aq-2P`y?Vu)|)hJyXD2Na#mkPLYYxN(a9e^}H_|
zJ!w(x$f50U&gZB)pmojU@l6KM%OsX7DyGUL!oqS@X<PkQ<0Fcfk*I9DK|{`$3!ul8
z{^jS{o(!3@vmzcs!`|VIi1ryv4<U`=<1WZ)IZs2f7=a=KSG_Xvq)ciaCqhm6g-`D2
z_+@9zwGLDK+Fd=<0rLWh?@>cvPr|c|&b&vP7-!o>B%VTnh{7(ET71ruG>0S$`{Qb+
zkQ`g<v>=XM>kxOJ>3F2oo$mCNT<e&lY-F3zST<qvqiiQH`MtX9fJWCNUJbj2!Oy7m
zz@Ci>JSWYearqKFD1p&@KTlqthU!n4veVkZ$0R?&5Ld^bJJBgSsEd9z$#2}<40|15
z5<UM@aRqHU0+yBbt9`I}CHI#)_JIU?Qxmd1ck1sicfWFCy`n<<P=<2WW8VGsoC~AP
zs6Vn5vI)yR^+j$CQz?!*2j>%PJ_Wf*`1Uk*){_2WJ??d=zX6^4k0tQfoqBh<*kbv0
zz`Mlz{#R9B9u8I9zONJ!6AEEumvzX_2$_WJky3WXQW#5QY}qpQeIMH(k)6qsK|+|Z
zM%l8CX%w<#-{w2&_x|4Ny{^w6GuJsY^PJB)=X0Lte(w98cHt`1H|r^pbK4F2oPa^G
z+EHgwAPRU<+>36%8PAjhEvQ#NQfH$Ce{X@nXVYp)g1_FL_?%1jr_p?j+mC*@H*1+h
z%4G1o(j)JlBHQArryX=$kwa&}ti+o7{OE1y;1#IjKG$${K~)a?<Odv@T(OcLHw3ov
z4Ryo!EPO4AFYAX#a3=^mk1q|$&Adc}5sf?d9?ZC!B0AkS(#Y-)4CxfGX92X}YsXSL
z*ch#Os!oj*HeV9M_TA;3;l=`OHZB?+NaJg_O9RnaR(@hf+%g8=G(&WxXe%R?gZd7b
z8b)PeY6Z3iw+TYsfVKR*<loubiXv?nppJ?1_eUYrkeeZgi3T?ID$C`HpM9@t%&9=>
zV&lTw0G!m&mi_MASI`x!)Rc(F?2|yk=`%Aqv)G;pdEZC;(hp{<L(!uJRcm;?sVZTg
z6zw5Cp68XX>H@y`-{2FeX1xk=3<UvX2LRl-CkjPlT-ZNK!ellDewlZv%L~gd!H1WE
z{P>_nLh?vRWDz$QhZzk|^93lA;4&|AE7aNGdps~UA(&^%r<GmVVJiV?T8!#-s3Ny4
ztx`~k-+;#4N908ma0iy&=KIIdsmORVXM8_DKW$EC_L?uj`<F<}7DlD!Dva-W;czvo
zMGZOB*SV2rK#lj{FM5u$f^c}Ozb4%b`F-VI!r)4lP>)<U>hmYOPP1}3=B-N;Fx=2^
zN_P$VIT^sF7wL&X$xeQtfVbK6T$egn$mq6j1HPG9VEu_ROnEiEt!Zs{b9r?bZTDFw
z-kt1s!J1cTVcc9X$ld=+;+R+dt3-CtRhP|UDH?DLQ7vZ56*NQ$znTS<c~gIN2ZU~}
zS|iC9&Zg=$xSZyme_a(eb?5}@;n-#q&ml);K16sG{|TwIw#x5vfk6KzX6=Vk{l|OR
zXPc^eNioH2Xf1rLhwD|af*BFrcN?6jG4;)SH$$*UEB_pJCCKZ5msEc>(|e$y_C1wL
z6V|QN3D0Nt1M~>-?&e=~AyNNhL+9r8cUQFJlaNZH_mye%EXR;@u(am{#Cd;5vf0O}
zOuEljG-uWV5skqL{AujlnCb7|R#kL)BuO<Jd|F;?^|{_Q-inJ8Fx%ylgnz5({}6LP
z@oX_Iqb;cZRwoIeFhz`u(?bXjh=W^;I8T13I7lX$HgwwH!$5H3BL<mu1$zS0?Mt6f
zxut2|b3|Bvt5{13Ccy}%Qd*u994$C)_LDT|(Zs&h?v2hRj0;w;awk96)a8uNNl~9X
zOWzGsgY^)0VzK%bWA25=-+f8KW7`4{F6W{R8tFMz#whtQL9luH15*W!p(NLogl?Qt
zZ|R&wPor?jJta1YTpLx1=TEOWO?bjvMSAW`0;pjDr-^tzasP|_K~&z(qifDo$S=Na
zO|grfQN1pDKIW$uOljs*U2HO+a|NJEz;P&>ejh%WZ4lFhe(fEUZkwWAnYl3EU#J6?
zlR|gM!PTz;SLTtN`yq<@@g6lRp9-1*j!{?ucUfnWmvx%>*?FkFq<mR(fc0RBgrRwi
zHI|$v?mwCQGL*&UqmV)loHuW2*q5YchF9E?@7<V*?AXR4H@!Vqr|r2+c_So@UVfmv
z2?Yat947{GwvJ<6Akh=iF{DiAel_RXZ=_AV77O^U*(P$!6zgFteSP5>cwlETe$ARr
znaY{TKO#(8Tu{*LXIqtM6;WXYs%>Sja%tdM*g9s;@uGd!TL3~ph?(3%Vj#D&x7<V&
z^wxD03bWtKFNo=AUM@2Kmhp5p&Lj_cS$Y+g^{CnYw3)4~eySfMegDwSgs3~ZGpzio
z<amue;(;aZg@VFjo;N;!u}YIkGguR%=i&I~%9>3X5sBk;spC_9Xdpef-U}A>F`kVC
zCXk7pz$~96kcm`xc81#r#av9dUWUNKw@Qb2wXiE$ocFdKzah*yEd<<1J?I(<${wmf
z$A~Z1k~#bcdV#bvD4xu!A!kBXoh0s|TdS1*B@)?Jk}?|mix>yR*?d6Hu6g<rB1@fr
zvW`ENRxP#{CrEkNDcK?W?5xex(+Cox>us?L<djqnSMwa%-KuU@i`?3kQ<xvMQz*B4
z=XihUsXWX*BfFo&E+xNWThqAmBy)Lcq4%ljUJ)XXR2kf8w&_5qoE#k^&36BG$NV0q
z0x|BuE!gl3oeE>MKl}ekXL<tzmRc`zjj9Lsx=AiFjasPW;3EE}{8=CO^TLST6Z{xq
zzme3RnWvA&j=mYbq%9tT_38+6<?A~6wacBpJo)s5+FQ>G1N6pUvJx`86yfM069o9x
z`tz}TB|Dg2t|-shJL4{I!W)AHJ+^#b6H{M0#TG-8ba|8Oq1#b(aI-#nD3qliFQx9|
z7fn0VSivz<@f?y~%~~%dy)`?~>owr%8Wz(&Y3S^Lh<;fVKeU471KiS$Rf>5shOcXE
zf0`<4yfjq0x?&y)KgOZWyfx0sQO^raVqI_D4`Q|5<e;fq#ARk!qHleQ;P=GaO5%J;
zKJ>Hg6uHt#gN?^qK&sp6Fj{O08)2xI%qn{SjE3)(sL{B|PJn$=SC?0Gj|cx)hWTX(
zm%rpLqf3qSd)qrtSMR=|aSS!Lw=g*rPNlrmfSqiwsOtssqq~v)r0dV1794aB+YEl_
zzKPjaXafZWe)X7Hkz+WwLF3`vBe$OU_Nxac+M1Y^o3DIumG<}owkhy3Semr$YO?ze
zB#6GcF|*LievbX2vT`~5UO{SEXNN@VAMpzJ^d<gM=Mp=eCFV7E_me+{PT9)gUDTse
z$|d6SPAi;aEx1}Y)A!&yDX*<`wbR}lDUCWVp$7_YS95zMrWP^*-oBK1$C##B`)mhY
zPxZaL+^-8-4PU=CKSy*z(?83VTql@=^TQZ{@4wbXLMgZkzTX?3My)a&BsbbbCR{l4
zjv)_|VDhzLCmCn!IT<ye8>XYM=V0<gflngK8^EMnRQQJ%=2WLXcodfeTV(X}2}2De
zUd<OiGO}-)1!lWQoe<mxMC`D5I)s4@M_`{QAY#86De~q9*wj0{gRKLpa+&R6+mPp)
zdq9d~)8awQL*+Ki1{HzX2}`V)=+IjaJjM-DQ5&4G*vNyK`y&TezU>Di0{atVOEZ$4
zG@ULoDWiXJE7L1EN^e~eXLMH=SEdq@6ccmZQw-c{*J{WKiTKlGr>i1gBOOiz;<d_m
z=g@a!&}GKheK*JGm;!A|sfw1MQwv8C^Ao5H&$ILuqWrT&w!yl=*XU;m?xgzHDoaw=
zH)}O<nqqZ4e;;EM=i9(Q!P50CDdl3X`?LJU*-~^rZ}QB%D5{g&YP$CMUXzT23Zz3P
z@wAp_$gysP4zWUy)?2M2-kBO`j#D)zuLglpqSek!=QipUG<YlBV2+hfKQhOU?<sP%
zlxfIgyzWKol-E@e<y_}0BGMmBCO%+0!<izrxJ?m7s~K7>$rS>u<hp98bL1<PV_?!O
zlzf*W9poQ_K6{EBdaSwcW7r&l16x6VkmpGT4+|>vjPhN#$+CG4iww?5wBo(24|DuI
z*7hq!zr`lupu>|bs$ZCAmwOH44aRYY`jRHbb$?MXHLdd+?F?JCw`L??n;Ycr5FVgA
zJ9cvuN3=lYQtxYx;u2(M3N{5d4<%=hzSrbP&f#6#Z>+D*Wjgq-if8J4@KFwQ6NJ97
zLPErH{<to%tD0!#X?nMJ*q2YC=*<w)@cue%`hZunPukAU`ZiqbOxq=ERPvpgQ$_*T
z(hq+_2zET1Y#x}Yp5D+0ntiN^Fy9Hw(rugGe2$+GdU)KD6+YjY1IAQc>ma|J?yq6a
zzodun0-ZwvGPL(ompn_P8Z2ORDLp;j)#_Ojf#YlI)7IUq-I%sO)7plXaB1g|`w&B)
z_49=KXtxAiOT?Lw>g-D6kwg&{bmd8HCo+iK_rGVn@19&xkn{$ehGV9)DYcGaPxN<$
zu@<R999!Zi&0-p4vjF$^Yb%HrcFZKt3k%w>H6vtT1H-?!iJ6gH-Z*6XWwJo^mDry|
z8PV|Wlm4zfKzxys@y5$i*23tuez}bDJnEgD+a2pDFI0?La~GN3cNWO7?fEv)Mw~<D
zJqioW5DuSu%X%f?y5qzC-WCQp0y63#^6l>7_Ar{`?rgv*RkmOT_bqrzmg(RlcGwpS
zxlPcrXz(>~eR1(5|3Th@7Pv~zqd4u>Z6J50i`SDa^XmxR61`dT-mO=lotBZhis-Ll
z6x9-8K=syh-VI)*<5-cS&Vx^Rl=HHO0n5(}NB5E)Go=1ya@M@_{@Q7M2{^5mK2xdN
z<<fXE`T*3|B|k*tjv!KzzxP%a(hi($>oQEgX|l_IZ0St~rHQRHP5qF-9$m}N70KWb
z7`%W<|5AX~ce!6{sjUJwY>gE~30yD-PlD=s#htAijV-V4Ub^;C!usX7S_9PdZzHOJ
z;@li3Kp*-xYUM$;%+o%!VhWAkx&>_BY0;$AIjWZ{DdU^zsOLLP!_y!q-Gwi-tP2Ax
z>L6yXUv%niE9<)UCrx(o(_^csvNE?QQAHDuFeR&!F)xcB1SMQDQ)se?`p2Cq*+&|S
zjWl+)*OXX)`Iod#?xB3~O2Z>Anj|s<g1jWAa?iJP+R(mHm7>a&(kYQhG9sa3F_?-+
zM)RH!o}e<4Op?6(tVD{|b4=XrmOJeUayej(kK$Ng`KL*)N6nq@3sJL%5WkV7i-tCN
zg=vPTY>?axx}Weq9FbWlsa@$8=uu%1_@N$${iUyMtU!8+2tS^xmG<zN+o%-q=QLAS
z1twNq+(xDxql*Py<#d07<^?H%v+HF|Gq~uNxdgHV>ZVYyHRHI^bKP{{ol{byOT=3H
z&0kzDLPs=T>UXpF`+fbv^*niKyA)mgJ)L-!L#EkMXfB5~X<)tH4Wqy}Y~IjIQr#PF
zf&fpDH>R6k7qt!<Np#hT{MKIla>KKg{!qHKyy*tXCLX}%pY#XSEVk$Th6eToiDN!!
zU;YC2;f<Mqfxt52`%KOl=T|KExz^S95Dwx{4WBBpIHsICZt>tH67k2qRI6ZIy}9kN
zL7Q{hKP+BIoqMHZai2Fr4ChG}j}4<GLSRXjyE;iWnMyL|OwyE>)qEkhS+ZGr%*2Ly
z;p{=9wsqRzk9PgY^hkWp-xSIXTk|}nJ5ZZT&GoS$?)dY(wU4Ghy1k{tA!}*$!D2JD
zbIE>vfQNxma@fF;joCfvt#{uRGFAD$p(|*T_8G?_jOzN~&Qv<ZYdK8RLxF(0L@%SC
z2wDU66c0pD`<)TBs{{)$u<_>Va^=5eQu{pF;%M9&X;r8K<+v^#s2#Vv<Wxk*6*mws
z>X^EqWm(=Ovtn!eEoackD}$ik*PxFG|11Qr51b}7r*`s_)B0^IhpxQ(YYnbWAkC_j
zZ1(UM;~cjJ?;qq4xL^8AA?;wNB*vx6%kM4PMgg4h+ETtU_1Z_TF8yLjD>vjBFtuc-
zhD7_g>(*|vZ)o2Ga`C|~q#C@gOXD+=tfuGLQVpL|bP3ZHZcMTJzE?8wAo*X|H0R@-
zUyf#6gc%0V?m1-!tP%PcGp=IXHaZC^IWzNO_nrVT!q4CIUnDwvOqSFJnr9P4%tJkI
zj(xML1H#(+#`(!*0PN!iogkj=Cx(qgl^r|&<K#V?PnLH@k;P$KCY5q;K#k3`_LA6?
zq|>am`Zw#lEb>!6jPV%9$pP%s(fP#j&<Z~@_$!%-{`<3S;i~cP3=*N=c@uQ*!0B<%
z9`Pl#A9Z@kulv=CVVi<{mSb`wA`CLdTZ~lK&B@MOlYlaiiem3aFU^0e<$RRdtX_B6
z2HvmMAfic*Ntcx*kx2C9%i13CZHGlv&+!gz3p9Cms$I6K#zoF_RV$v*jOCB>1jYMq
zmAsua9U(RLZ$544?C|x4q`gcT1Ag}O?et#NnEoO(WY#QZ2F!}916Ak1h55dBJ^y9x
z*e%Vn^8@P+Bo{UHx6jh=EG6LTvBOs!)}O-r$Z<m(w<sGczh&lAzcr|7G0j$5Oe#x~
z#CFKTst^Kk-t9!X5$WNI%=)d*sNbK4!S^Q$P9P{y9kU3uJ0q>JC<gNNFT;mX>kJ#4
zzX)&Ne)sx8vpnBQ$#Sy#x2?;6eq>qsI9myoAT4Kw+{0_Wej0fOs?~~_k&BaOY=PW-
z{ewpHp*$kJS=iAWXhWpGpPo4}-B51yn_0o^CXVdmW;Dx!9ds$O9C-xNTVSeJZU$bA
zo^k`dV)<o|M&wT#LOS1&ALsafs}yGkOFk1&xt4Uib~r5|^%SbWd>Vs&?!YN=93!_;
zEq6wjg!r>pCyTA34mFg<Vr}3;32)<0?Ae0c979WJ9MzHuF_^iQ>18#{mVA2jm@5d!
zmW~=OtUD(nGcA0m>L~v*0uZ)U|G5KUfdUR63Y+{prPkUXOFBD{b6>^R-Y~Bqx+;wP
zgR;cZOl$3@CQGfBE=Lf`d3(4Md25ka&(__GhC!QE6r2}0DaeF7j@{7$7({O;rWkew
z3|}no9En`vQCNzbs~es~<xCzhzj(D=K73Dce7wk}^B#*DPhL+|(BpM+A+Q!)IxwZE
zKVrD20D9~ierWcEtn6|e?v={X^@fAq(j;(!-<9~u{p2b*4H-T_31-o;Jy(MpyV#AI
zI9P;H`q*M~Mz0jAfKTXAT*h>AKaZ8YSSlhbLO;nhSYUxB{0wP-JCXV~tw$H;yZ1$x
z9RM{7&-|rjRze4j&<u4#u?`1%KTi1XGQ`U%@^x_x+s8f(d_`$Xi1g3f|Lq>W)M3Uu
z+6#LE9?~pK@byi3Nxdz$+%arX0EDViVqXNY;0~Kv<+kD<tpYS7Zv&lv>?KfxaH5rM
zWu15OHPuIdH5JE&R6a$q-WhsrNf}6i4dG_>AMn=oj}(0EI=vp@;2%d~<EU=eUQnD7
z!f*L$e6b$%2dFje*OnClGywt7v&W#CbV#9<NMq4f^Z*qd$sqJayl*?@zz1K|n1FP8
z0X`cA+$+BI5yNw-H8``VdjSuF2)FVk>vGBJ-yHkNl6$q2n+BXpQNX~)Nxbp))Ubjk
zLN@qX0NL9~;k<{8?Vp3}uM6|RcS^IEtz3OC-b#jN0~5NWf&jYG(dg6RI`am(34NU#
zTv6+H$WDSLk2vFH7Vn?I2InZx&2GH4FeHU-Cq*cy6NBjDmpv7ASYL=QbD@szGzUZ%
zbvpYXM}6^E{Y)43SUe;Js|G3ySECEEvmS1eWG@Go>mb64j&gEEMa<thc>QkNQaJ6X
zGg|wy@R)v$z5`6QUF}*O3uNehIl1djWozb?T@l^|LshBVfXuWxN3@Kx^sKNNJ~}!M
z5Fc~J{UBHg1VFn>$sj@CKajxfd<mdOYsqVSg~sWd#<*+(hhhdIf+gPFLtEW6C!aT;
zFZp2_CBtLW<4WuimB&dLH<}E%`>KdNsIeI69`8Y#a;7M^cOoeV4%*2=?eCCXDwzbm
zkz)*idh?VCJEUzi&4a6#uBDODQ=nY<kQ#sM6}^SjfIDyHsu~tw(}RD;hf+x(Gbo22
z{Vgh%1+71nU7yTV<+NN!93Gz!VBJms)xd>^5_9&4q8RULb7>OL9mLJrT3Ntm1!1^^
z9F3fh)1x}9rB=<t_4@t|`$WZcz3~W4CW0^_3-t=-8)^_%g{8+yXO?`Z`YA^iTzHEL
z!MZF?_A7b@KLZR?+}Ui{ULlZlH@`H$TK=5ermsCA)A~jNlqEh|h5qo%Ju_0&;w;;H
zaLInaT_)O#KZ<n^9pg5FzHIGS60x9FozuF1{aLhkmX=4V_vl5+hmQ+8Cd*KUtmMik
zmCak0a}T`@F=vOCb`k>CA*07#y7*Xnm|x6&vbIrB9nTedxa2myz`chen|!OBu)gnx
z>T;;-)zL~}#oqYQ?%b4PGtegP1+VNBjO@<6nb6Z=kM622i1SmiBp1+pl`?0wm?VSb
zj7hAn@X!ehFwN79*PZoOWvZ?8y)VV?pO8yviL4&m3-cpA6*UicZ)G|IT}GQ49ov}J
zJX$*@C{(ILraqgYftm%4pb?PqSH;8y|A-o`>EB@_xNZlb7+FORKm9wIh@6Fjp)8by
z`k%$d0b;ZHU57h1D$xFK<dPXPNZ)tKM`|ffn303b^&hB#VV|^y8*D95iu&rIlf(^6
zbrB+P<*|htRMnY;me(+pqP>Yk*GiMUdx61n>_EdE{6ce?f~=}ARxk)MlbMB0V?W;C
zKc)N?Kj8GJD6q}aDoi3iIe}W7kK8bnT}U0;CMMHo0AMVLk(%afGO6yAYbu804UXMz
zSbvVwCjGpA^KNfb=ggwcVNO7VEakRCY1GrC?Ltn#A#rnpU5VvMSH^QQkQJdh<9!sz
zWZ>y}@0||6o-dI_Igx(QfVt>H{L{l9v60ng<VKztsyy>7OWEiB*fTYF(c;Vyel~+6
z`fl3vy!hJXHR)mlW?828!F>T7qxzulej5#J!;h%}9%f=+b|IRhd#41+86P1Q|H?OI
zBywbDM_^(Cz`hbdAcl+es9D$zmhW9~sZSYVT&ddv(k&h6;JAJ)E0FuusM=8s0KLj2
znun|U6yNjN(8p)=db*zfIiHtdQ#sVCkY<=pSI3`lVng{hq?`PJn8Lg~M;t8@18YIS
zH^`QP!BvIi+1}wZ^a*bc5(lwvTAL8HT-h7{B+g!RrM3Y#tN6o(_KKT)ihc2$Lfb6M
z0*14x2iIncH!ruBR=lPq-8admUT-Nts-MvzG=*N!Z|4t-qO9)N;V!uM#`D6qBh*<V
zN^<kH2GgpI7Hx#UxsM=R!G}yiDou2yrNAg!dFIOStrQ5{+}xhbXlb@G%3@{q1^z&I
zmT{+Xa?M^yxZ>KIu}{=n#u-`~<b`Z$Yb$49FC~CgtIO38>IVn8*PfWD+_~suSrxYB
z^=ZN-{B$TBfEIQU@?!Y1F1a)*C=nUjGtHte+%vT4V0eS?=>j))2o(c?AD4v{Ead~k
z_Zghgw^<H>sYegpexGtl{NwDZo<9yvoVcoRO+&B93dghV=5PF7xX&xxD$m?_fJ0<f
z&FJm#pxjH<`ReeLPZAG;qCD?zn=Q8*mb_-l=ZwD>iwPmFvcrHs!s;r4lTZ%T&^BrU
z{U^KOw<wHub+T@ZwP|P_{uXEII!CojkAjdRej;xiRjp<fszM7zW=d=t1m|C!^{uP#
z;F!9u{z^xk7%Vp!@>x`MoC^Zi|C5$)9OR%0u?qK$P+LS$nC*parR7Tiy0rd*ba%Pa
zDB3ZKBHa>;#fLYV4d~UPTFF}cBxM%4_3>4xHqF@`_MdU*QY=^3Gk^9I%52#pl=Drt
zFK6ww|7`vDqWMCJTGV)fa}hCTXZ}Sqzsk7*_bx|J!hd%&j0|>A*z)k0S+hdbUyr^K
z_oC8Y{-e=9Wae)jO1?c_D20Sj3}4KZJr|l=ODOohpCeb_ftD{gK_!1<7^c&F-h?av
zo9zGnz~ig)f2P3}`&uK$A9yec<BbKYx^xlxbOXsVNquim{&TGVA$&d{J8Nxr^4;Zd
zD*3Ex!qH6g8dUR7kN@8&hs!RY5k-!O&Ryks7^{oZ)5nDWH)H?4rNEfxg&)EW1l-TM
zeB=0*5XpEMd~v0$>Ayjp9R)CiEDsZl;2S()m-}RexXtl_ZX1rI72*4zY0K3w7~OL>
zkGLAF`lFFHwVoxcVDyD37vc!C?Y-<eT#k|;`<wjlG?U+Qs;MP(uLXxy2$H$z<R2rC
zF{IjCiL!TlBgWhRv%jIo8LjJ>LRk51iU0qjR62x2ZMUnd$gBxvKKp|ZZ4LdqRqEEE
F{|CI7K7#-N

literal 11194
zcmXY12RK{*_r3^%7^TCkP@^bfk7`M6suZ<q?@`n$YDK6~YVS>rnzf6nU0dx$?b@V9
z?cLw)_y5oHWIV|oxu0|Hd*1iFCtTy1A~`8NDF6WE%1UyY0DzAKKf6GQz}K-~o?PG?
z$!jHjHvpjM{P)D`asFfpeo5ypujj7iZ0+u8=4u6adV2ENIXSpln7y{*b9S{!-;tsR
z063s5ht&4U*q-ywFnW>5zqcT{K3~Fil2xpk%Ld8KP0ta8HW$I)qhX?BsB+O~DAb2~
zxWyA7G#DY<k&`oqT0-5QE{98Lz;i_DpKJR<gNZ5F<esp548L~%aPs@;`V(olMk4(W
z$&(D9-E05Hr=!}F&in@?#YXF8-|9)!#)Z|!r{CMmFsF)qwODU&|0SZ2jbra`0{;7j
zzUb@MnG#CMM+8gf8F5!lM>y<>jggklU>Vs6xTA2nizMmVEjS}ZjF2XCF(n1Xqc{&W
zeHLtH4W-*?3{)uc{B?omMJu&FJ;HP6H`CTp6xPy6asI}n#lq(+r@Uw%V>|u=((ekG
z38j$d%sM3+1R5mh93oRa!PZiTW3>3rM(p6kH3czpFa6Ksk@0aYdb;B64EMm6maTC>
zdo1ID$8c(m-?z@GOuDZ(#+?wA47BWkD8LX&4Xx>UJV&fym1B>(NBD|OFv~(m{l*{q
zh0l)&Oc6DIh3NL1$Fj;I7l8W!KoEqgQlx?m+ZY;fkZP>a$!&e4HY<Z2KfpOA-hZpj
z`g!Qq@>{c8$n7K#mc|~1;L%oS&)?UtW)<_-#47Vb%&kz}Ev=1FgeVNY*X>}Nl*!>G
zT1ws0$z+=}lh1d)h$+nxm_AH>Usnzz*kIfx03XgGVLR$gsC93r0!@;*HQ^Y$nPY!B
z-%7m4DM;c8MKQ^bj~F4SFX!)Dop<)GkL`rdS?*pqKYh)ligzj$7JxTGP^cg9db}|3
zj@I9R#7E{<v#2U^Du4<>mR~{qTKkQaRj5jVrh1j>r8JNv2PY^Eg1;p*zEadqC&JbZ
z6CHp-9%W;d4&#`GG}a%nLV3TEq~Q~7kProw>P0#)G*ks#xf3w<{QQnOmxeuf`ms$^
z>^JdfLppq>J{l25<BaaTzv53bF*oNpiId%Z*U^gJd-H0UI9;163%x{~^4gP1n;97|
zEx`~CW|rBKbL0iGL!gCuZf-G7P!U+1UL8i;HtO9!Am%YUn$h&$?sj2}&bekrHR@x>
z*bX^tx6Bbc&vOPk5tpm(EtQ6R&bo+A7QiQ8`{h^rb+wpZ-k6s;ucZi>@il%l<p|)Q
z4Myz^Tu`*Ml~H7HtB9=np(ji0h(zh(IXQ#q1pQ0>dKhqjX7U!$3cuBFX($qKeJKH+
zHROH^^esVtK_}!FZID+TZi=$O$!@pU->lBSukr4|L!=k?5Pls73tpx~RNYivd0p9_
z5YaPve^D8r5*5e}Im%d1*gbiGyDxlZw{^a^xbOHEq4b%Fo?dUP;r)I)fbzg0WM26i
z@FjUNev-H8U#+7YsJ@Nfq|wLb))73u`Ra1IZUFnn!$WOzQWiYvPhngrfnKt!iviiz
z)>buBkAWsL-MJ8+u1GtLK@d{0GLX3`>gZt)S1Yqx1KrjZcPpM;k=l9q;JSX&q>kUG
z&l#kp!N0bFyYZyfiyb=>ZuQCi4tEN}69I!`w)mungjAUYOR)={Kaa1q&d_S<M8=Ts
ziUhW$J!{j0L)p>+QMl<lrCydyc)Gwwe^VjdRD_JDWYm#cj|;_cQ;Hq@*7zNPx?P^k
zkU)oVzWeI+QuM<i9r`g2wtPME2duee^^4-GvTa_T7!O-VJ5Kq_Fq>p)iA43|tLp{o
zgPt)y6$k$;n=91--nZ0)*?}W?84qq$N+Yj&UCMkO4f{A^j-GbcZASy*wk6r>ZLt*&
zO!4N*jr*Plf#-P}cOd6k+TsL7;!!o9wT&Z$NG5b1zUK9mOx;kx@|r|*N$r0Qo^+;p
z?MARQmZqVL{Y{u<vCv2ANpKokhKp5bi!D>gdc){I;wtIobtoccBCySGdF`vNuFmx?
z&UO8TEcGp0k^%BqL8$#Z4INcCM%+!n$I2+QFmMxwt#WF=Df*;F6{8PKyY{Au3SjG=
zX}kQQBWjh#8H~xhK^r<e%CiUS%I0b+tQ|@oODPK-%1sw3pT9`YfkUq?0W4XCOXa%$
zU)B_?Ytz@SyukTnZW1Tw!zjN_%j{bJt8@R5bMZq%1bG{pFlAc;qh>lwl*60F1^~3g
z-9-M&O~)kq$<;sggbovoouf<D((-%0@pV3UW8B}Ssu@w)Q3puo?;8yD_Kv)RY~TcH
z4O#X$!R(RxxlV}OW+B2gnmRf;HC{H^TVA>TZt!`SM!xphJIaHmT6*>gcOe5cW>g<>
zN~TG;I@yq(^MOH30pO=Gw9&=ZA64zTxO)uE$F|v6f{XDX6ve|><i25<ZIkxL8~C!V
zHnIJqX8WE+yg#AoaG66NJwZ`ZlwY=${+#|K0t35>%9nsiHczJKb2f{MKuf*=F<)Vc
zUa404Z-GS}alQ&bi85(ozaLJ!knw6(-ejRo64f&CDX;eOwc~{s$2H#c<`FjkG_x?!
zq^aVo+eh7FsbRCF(k^^DEU(^P|Myy6o0ngDdSH)LU-J&LT}PT4kK?MhuU@vgXgJ_K
za;bVfXPK>CR(cB^%3aH`IDJe9Zz2ELdfoTYsBNpL-tn*2y*pNS_4#IvtK91BP2efX
zJTub}7HCPSOoQ$#k-YKg+)?}YN_5TmYT|y{4$`n|vElmYS<Lz&#-9Ji-jKp%zSV%s
zZ&exJM;vF5$L75h2`-gBK8x}O%i3$S^+q@%+h*oi`u+M^TYiAo5N!ky2`$$ZCebCc
z5)m1$mxgo)f=FPerQl1>ks8{8XqnBiuZ1;Mc<==hem1Z-3qGp~RCpB}X0faQyt`Tb
zF#Pm@YsK1WY@tl;`C%k3$r*oqla+@$>J8(1B^-K}s?XsG*=^G|B1@_@6|&=nL4ohG
z90N>$zt40bMD8`<5(3CZ9)BoPb6~{QM-#gxNe{Nb7Aau{vcWc~lLjb1Ejkga692U)
zc%OU#380avl1awda_=$lMqX6CLdCF5cigbeV|%&%5>vZRa%zLEi#od6Ahs)yeaE#;
zyPT4gO7s}k7O$6(Q$bY8jM8KM{Bt62Ujld&*MDyjpK$qEakI6oTnS!;VaHPwHNQ+c
z4V7DJ@jgm)WH<k00Z}ZA9(HBliA<o26adY2R?mA1F_?wQo$4PD<RcO`hY3ql64NlX
z9`#Rl)(^bv{n%(0-iNewulu8RI159Si(mLnZfOarrHMHkHySg~5|b(l?A-;(H?d}m
zOQW3xn|om9yhwHk$$~CVpB=3K^}0Gg;&r2T?+=JE!b%cHCTfxi?N!0UoD%yTZWm}4
z7wI9g2;uu`Xp~4Ie457f^MjXB4lnsRTFTQlUM)~^I#T=~8|K;mUgbgpT?$g6>;9$x
zxX**B^nE^OER^%YKVd*^JL%QT;|_R>v>gQ<;TO$fh_ay6=UPr4gSnjXGSW76nbWtq
zBWETbEncp88+%hm{V_&8Q^P6v5M@!IH$DI%<GsWHh;5XG-H`??O<gCQ!=jeuL@E0g
z$c1PpV6SYV17axp_Or4N*X_swcr6}2fXF2KhBsEPSv=vt8#+1)^SG~I!W_n;*`vB<
zcZLP|lV<<vGXhbW_c9A2=JVr}d9uFJO}ynGb<pOX<h-hU8_TTl7s#V&KMlI%_LbC}
z1;9TRt(}(3DJZG@)~pkw`qO>>14R(rJGu9bi=nhSg8#(uORX4+hVY*5iw=#Q!z_#|
zI_H?G*Ovyh5b;iMew;6v$NpnT$a!l5h6f`6>ky;umVSZWTZn7kPs=r1plsa<9nFF_
z6Spz5KOs5Hgnz1t>rJyIiP}_EDx!PK$`lfq)cD)SfZY5e`SZh^*?r#%T$A}XfySOS
z(%v~y#`s*eb#Xi1{*$ZLo473~3PUQN6iQ4)LES&*`op6<zD?<1bD;*d3|)wR21UzQ
z9>f9{{)a);x3=-}A^;eBZwdXoShoUFhG?7tCo0AO1a46=WwEz&!cSXOvzES3DQK<K
z=ArZlj|SMQ_ni(YVd`(M+O%)Hb|}z{Pv0o7tDQI4W+(batQox;PI;qm)OE-u!>CPa
zQ@%+kU&cM0S<Yu=yLPY1gLd!gwSHNp7(uB`TzlL8zv7!v;V6XlFhW#3<q6P?#H|fy
zFC2bhdg?>7a#6Q{cbcY+2RTQr>yt0^iRwK+rF78#R9b-cHei6F7hR|p*kZrqpHSPK
zDdtzG7gK)1n!Y{J3vzVwq5pW_fCe%;10BD-NZ-?@lH+2||HMx`62C(~Q>zsaP`Amc
zJZx*+p=4wPz;BiW=8TNR#~WcEGohyPU*YrN$|sHE`p2ggzePphDGl;?6CCC*CapDL
zs!SOZpByf~i}l)VK|J-{Do1W5!X=r5;|t&N!h`}yxtw;uRPdBd&*kiBZWzU)mC{E|
z%<Bvs2~u>GZnd|iKl<vMp>3Qc9<?0E&RIG({gb_vozv5Pvs9<_8|*q)X5tmr!}DhE
zsOzVqq9Vu|ZZ&>0_es^k;)koNfW+Zv$-9bQQb~8a*eC(o!|mGiT)bt4-lJrW(Zl$B
zo6C5^pA{9r-kstnDH~flUabuxd8e#^Tfsi9{c|4jk}*M-WI3o1&Ifoov1t1D9q>&@
z{P@D3s^#I(y<Iz8L|!%n>260CL+&RHTKxoT@(PE{m?EBJs8=%1V7IR+DmCp7hwYps
zj4FO*PJoQa{<&)Ao9YJ+EGds!5_{UV;Gg_;xz!!i3+be}<8MYhjlu4JuP$qZC{fP)
zOi|`hvi`IyHQb2EGeJbk0^0T0decKA7Bxrvo+)9Q=hjT=u^XBG?66is^oYqPfZ&ch
zsA|4x8)Kf*@ZjBiw=|`p7tBFl^?6kbERZHFu2$sweL}U(0&0?EGWv-9Kj0$K*0vGZ
z)_^s@=F<tmr`G+5k3%C|%}*{F3PSh^?}aYh^X_SH4=j8Y_iTEqH-xp%d2sdZs3BiM
zB3BO=Gk*8i@A%=ZFTz=?$cr!-b09q~gerDLOof+NYS^vqft$W4KxE8~nS4HUN&1Qv
zso@hA$lLPgU{mP}-T4R8bIPi=(ak9WO&5sqM5NkSxVq=zyI=PB$}M>kKV34eAP3c^
z)JpqaD}Y9f@6o3^G4sIEmrBL6cO5a-N|WjYiz)?p-6Z9{aU#&XL`YM7yXUF@(XX4y
ztd%a0R?>14@iXyR`&4xwl52{Qu89Qq-q7W+=+lm-corz|X25sc^*^fio{+ly(9fid
z#K^(f;t_LA+*&14B>Zjd?EPv>eyUM+Zw7P<KA*dL*4XoPw=u&mk8&EkFf}{Q#UunQ
zVDsYFEvhC~%Liwb)(&yEe?rb${@MU>>W?XorBpAEaS9tdJN1Ae8*O7df>iAr*TnvJ
zVQ<QCM$OvitcnA~!!`4S#lL(arBD66!j*J>933C)jQdWDU|@c!rj{l~W(Vo-kU^Wm
z?Y|4bqcgnG0f}2TYKj@6({NiRRNMGtWF*W;sm-AapXAQ*(UCHF;e@cvFWOi}A%7$r
zczk)ylOTae_X~^ZaCaeLf;)205Bn(*er8_MwA3}7qMt{!)yB-zbE>+<`brt!co@B`
zASuH$#Lm&7Rs{ZX`btxO&667kZpyIzX)w4&>UPZ*^Kli0YpTE>kn*$fojWOw4zsM2
z07ZMWt&s3uDOdx4p5IB06k?fWbJW+-8oWWn<iiXZEeHJhZ}&D&R6nRf2HD-y(N#G#
zc-5^D0e4N%A6M{xcD8``J3%-!(8kBdhrME9#1+-(pE0svbPq9Tg)c!bV|pN|Bq}T9
zN4ji?x(jK~ZhPR}YE@*H|K?A{{mrid=0w@NDeI|S*|!E&)sn54+7?g96t3Y=drHzZ
zJ4dI(BERztx!X6S`C>n=2I2Jz*9bZ><8Dk+yBgs>D&^9>&5OP;J5m9LRm;Vu*^I7r
z8=Kw3Oi;9t$rtchp4&?92D*)xZ>yT!V+7^-1DV2aB|)w1=z)~MKijTX_W5EP<sXBP
z&bNA>3oMfcX4H$lUC-axT`g?tco*;c11^xs)xgo#Ai9e+D{v)qXIwVy75RQE0eQwi
z(?a+klY0r0QZX<xWqMhdou|W9PHc-xxl*D-Z%di|oEWlfzR}4njY~oN5%2Rv@Y(Ag
zv9{2*z0vfx7#fMEP%01p_04paw~kNdbNOURykm=CVn4@ya7*}gwI2MJEhxZf254$c
z@vMdZ6rf`hQ&>`q>Z*2joCfO3cZK#`+NZJ$y`|$YdW;7wg2-+$IoIh~FqRQ>5+sX%
zs{H4;SZemTP~Z7BIrhv!1*)>DtZGFA0S4DJp^<9~uU?_@w1ZMo?DdQxfePcg2;R*8
zD5G%>Gs!3q6+g=jJ1RLFCxJb+J}F)cIbyQ>3DV3=X{ug3&#qVV(4W>D<A`Tq+2l^_
zCo@%g7Po^Szeby8eVPzOBB``x*v{lEoO`OCQ4gw?X>%W;Y-6?W;yGL*5`55wb%}&z
zh_3_nC!UNrS8P=l%i?X|A(Veyms@{<`o~1I1>mJMMuk6Ds!qnS-h@#k5trePMx}q5
zgguIVV;{$tPq%aNi@d_Z4<FUr4B+!^Z*4UR@plqL5H7nyXWI|okG@<wLRCqbUq|Bf
zviqAN9rsFZNKr+2tVf3irei*v@E?CN97tLge~4%QW{i>0Unwu^jp#J2oGm@~Vv968
z^%s_cyKo$~bn(5!Y`G~bD~s82s**()yPe-ef{(!OphmRtCwwttjDCO@b7qxS@p76W
z@;@q+w~<vIFnZAMgDA~gl;jN!TT&_D5X-jrb?dpdvkBDHOZB*s#u)FNOFbnbG+wg8
zwH?JD_)6&T?B%nNtO^j4R;Dp?a7JG3(Ws!Ht2?a33$0%gjya$$%vWt5zlrf4Yv12z
z#CkaY@V!l#&I2U9&7_Ub?aut%_5LpY)2aXggBwC6Kv@~i)&pXgdogJcAy`S3A-?t<
z0n4VjUj()@YWR`Lca6PXNe+$Bm<j%1{nQDt`5Qn1NQQ$x2D|TmE>V}2(<)mvwPJy9
zEl~P2dhlz7N7-T_1!Q&_#YvYYY5^%LhK$yM`lZiYVM*&D#9uQnNmK#VgjYQ@Vq;1V
z9+xhijrZ(5pJ#kNLp90%)ie#zwbRDB=aOZk3_D2+r$DgvL@2H)H9tkDkLQ2a5GQQY
zz`V1qhJ7-Mvqm6?iB#G^0=GW`dvrs(%$QJSPnnr%O?>sFf0cbag%b%VY;8ROG|7h`
z?2@_O{f1acrY9ff0hu-{hKq~8+=)eYW6<VWYWZkuqtGNGR~pXy-$>GmZ*s_T+J$8G
z=9o(w3C5Tj&Ww-8d&v|9hd83`>!yhiCcI!hPk%IXjnbaij=T!ImEo`KV3S-nJg{n)
zhUlJ4%t^zAZ%+>je9RE<%CEyOsO4(8*(Oz08w_4BN#~xT=`4G4??VcT<MljblUX|a
zU*jnexMof4>CzS1H>S?>l7Of~1t!e>X7vWDdLG&gpHV^HlGSFgKJ(2%rMd}J=l_fp
zl|3j%$F?Tff^sGGoe&Vl<jlD^+n7}MReS!ah&xTK6BALG^8VdeLV~)D2kp%y`bl+E
zPtE5pU}Y5OtxQetG>-7Vq6iNU?Ro3Bwqr`BG?rT|;o~;>;9{2djf{&(DyOkum_0NP
z63k|s8&L!Kf5#pLRo6KgZ9|UFkBseteGc<9JrWsvV@igPK{w)=pG^vTNB0P{e*2>1
z!-acNJHhVHRG2W9o@3|A_|w{!=*osRBwBe@O|C^bjdG|-x#<^p>96j`%>oHH`2+LZ
zG8c(wXH<xJaCsLWx_>NAQVsTrV5*saUasR)!zb?$_I3<q9;=lthE-wg`2&slU4zql
zFU^zkg;y4~EZir9cG+(BK8QOh9MZU*4*!3CK`PnzAHS%#q~4HhZ^p|*ukaw%;;Di*
zSQnzBMm=Tf67So|d~>oqCNGl+7n9{z{s>A6B-JG?YSF5(d3kTo)OvwN=qCgZNWB+<
zmt;>W+U=BjXI*?X8gEMLL5yv|?uGf#ue79_1nLZ=#Ap5uuqU!=b%n8BovJr+cH3GB
znnH)Vkb}bD$c|LoSdPs>QKX+AfdifUH`A5&3I~!o7r_#+gN|>TYPv$yc|X5ef4r6s
zN^+SB4?ErjNvDlbWD+5@_)mG`{m6P#S`Cw6Qz7BbQ;4AWHwg?Iuf)Qqd!pj;E&{2J
zXdp-LcQYfT5g8ci0JtlC|9($4n`2Q#(;IF$g~b*{$J^_ZK^wSrrs}A*`QK$h=2}u{
zq)7y2zN#dQZ87#prUr_st_o}vf*diqKb;S}qvT^46O=>2u=u%pYsGJJ*e-W}xFM88
zs<5-wbR3`rX{`8Kga0F;m;NAlSJ7NI)OH#+xh1fxLy{MAe8W~k-b=y`A|#l76z5s*
z=t&|b>uh;D8}`Vk)0F;XX*ZRX7AyQV&b<&!16rn?uGSl5md5t_I!l2eU0W*n*2)PG
zP<_1#FDqo%;7)UQ#mL*9waF~aG^egq+^~Nk*Kkwji}qcD^-!}Z&O|Y-L_{-XhuYYO
zEVj(hk00I#euqf0C9J1%1W=G<ExGGs@JKKHbKe5pMo2v$jR)!uQXP*5PqPOH;PZG-
zvsFOBMLkgM6qRQiuWe-D#@dmV->Cv^7DGMTz?Lk#GTKO^&pq3FO^A_)FWQ^sdhciS
zmH5*Fv7F)-YGWn~H|2+wR@68s)8P=LM<UMP1m%*T1Rv-8+P|89UNTu^y*HWvv<!$a
z`XNaz^n<2<Oi-4R5P4sM+pOCVJAEgTAYLvdSe-yzv4o>sf=&SQscV<OHfmkpxkNiJ
zTX|x_Jk3e)(ESB>;vK`UzJy`<Ye$Y*Y<?*k^P;QfV3l@>lrIdt2Nz0g%z-uGx7Dgg
zc-LQ<S^G!Gkbk5FDM!L-tQ)J`K<Xwg6%$i2d--kz8j>gf{&Zai{Xz1VaF2@<487>x
z*k5a{#c5yx9V}l>p~TA(L%wtYnKS-za=*oh+J_V{a{hYX`rXQ+5tSMov6LPe`xoe(
z^GB)8;j7j(H|0m^vAh8!Pi?{=)Yso#GCF&R9;)?$-Abbmr_`}WiZ9H&L2WAZu~Gnf
zB=%Kj^iD{rQBM<DTNR19B<j>0V5a6)b$DV%d)!!Yq>G3%dSp5B3*d_@E)n?$c2?+E
zZxHKx5dxXtyH1n|qePS!l8>+6rHUyyhyOGrk||RUR5FUNdv#c1=6wT|nU5hJ*PQdr
zEnv|RXKqu9gW>zNxn%rh*N(&Fr)+Ze$-YJ{i=ThGy!~`(U<3^!^Cns61)i@7U#ur1
z<8O7p@#Io94Jybjs=i0zX3g_1W$}gk#bDEiST3R<iNo{*bBIY&|9CC0{PWrn<QI-e
z7PJA2W(=n5)GUySHA<Q_W}F&yN1_9xNfg#fKu8JEPX;6QY&t&Qt&RjASfF$cZmSv^
zf;QxrmJVe7K6noj62NtNLD4x}{V=|%&{37Kv_Q%KWPIk)%wx^GyX6uB<J__Eyye)=
zT`oF=(i=hdj7iIl%`F^*o+@gEr)@}WthjHPdFn>YhF!__lGn<mqaPZVG!I>rYZuXd
zk+*TTT}sU-(Z23nFswp3?PHiOT=6Q<z~Y_RDI*Ym0rT{jbB%jkm$WF3=>eLRx%qKm
z6?j3CcdI%+|4gHYYv4oKPnEl3$i1r#J`^)mq|}VZTqft`{d8T~os%a5uqv#5T0QS~
z^L0`pW0g2$8xAbDNHCqb`UV9ogxyDs=Jk?9lHqS@3-9DKi^{57id)AuJK{ksRzcM6
zk4sHobp<mSZVbyM8GsC$>nuk-@HE2dp(qpeY+jYW41&j`qsr}OMrLMSinI;T5OWzy
z{WEFs+3nQAa(7~s^ZdY)Y?E5HriJL@lr^D(AmzC~Kkq<X-a!3LQe9!fbp{o`B69os
ze>B4>zW?4on(@t})MN)(Rw)(%<kplPc;87YR4bv@{!0^_3M0B==(r3tU^^dA$4G%K
zD};;#d-c@ft(epkjJBq2$n^^$VOjMq1kMIoshk#Avz|2YIlh19!Iyu4R$Qu15Yyz5
znNx}X5|-j-gmahKnP<S~TbNq-t6MCkmYzqe^jJDLkgR`o{|)|;?U16t>6_U6QLV<j
zI}W0O6i<AGrYpg`DHc{v?_MW709Jfyv`Z&NsZP0zrHJN#|2!tfYja@)ofDDrOrz?G
z<bm<p;)AP=3%C?WLbz!gH1gr)n<)YTvfL4<;l>IcF*n|I#vTLGk;nf@3bh5D=OlTF
zR8rfS3i0k0Vn{%Pq=bjX%@q%5>(c!JT0Iocn;B)2_4p>1{&~8pt9H}wT=MJqB)c;3
z^L)pdzxTCbF;D-2o6KpWh9OCqvm!%=<lzIH41N++LDT}|`ysRnd1TpN6mlHc((~O~
zK|ZYXG)_8`Z{xFq7<zMAGZ1#k3N7(e0T2B&*z8o5Eqls<4Lu<rW)PoCyL=a(dZRlq
z%R<>$HsRsi+}v%3Q~DL-AiUs%^eS*+EOd3Rl1*38ps8k4(2-_F6Rbiz&$9RFuQQbS
zo#FCb2GWSwL(bP~G*mo>s4lGCo{LeE2TDWPfTmb(^c!&Xj5|8;0L)PZZhjlmd-r4)
zh~RIGpai9U(oJgI{QMxo6a3`ZtjXt47W>Wt1sLAnW(j2=Ct7$O&e}+xR>)2zKUxE8
zk}!jD5$J~9%oky>j+B0883K)b*e#Jd^SjrhEFBt?QD5P8i2tY1F(gZ=K+~J$o${EN
z5=3Nx1L|kikcM0U`$oJ12qo>25BUZaT~!7lD@ELt`=DJI=vvD`1DV_n{6pN{`?KtI
z^c%=nB><2;{5K1LOFF(iIGC`kmy9p2FPyNUYDRmlA}plHjjTln+N@BYWkzXTf63=k
zs{xT_oKYY4Iw~#N?jKW8n@QuyH7GwuOPqPo6ICcM2-n+x&g<<W>`0}kacRZ<xbYB~
zcnixHkOaTJ7$10Q?#Cw~0;BDS&*%EXwpln0IXWUQv(PY(%5_Ezp{M@%sl`u7e67U~
zmJ=$4PR+SI=zDln%kjKeP8zc8)b|wMm_U@-Wp2Mcu8~~Zoe9SSS9zT4=fH~r)tQv2
zve+cJu#%G2MoEYKo+L2erq7?%?PER`hEG?T$$m)XS4oqNTz&ND5qo%AEr&&lP2t|k
z9MQNR*m+~iNaqIoqS&fkf*6H4$_xJkP!#cXmSvM~a%@84^V6MZCP1LD>RuP*TMmcY
zexvX&uY(|ulDfVz1)p^W)zt*6jHQL`UIg5BvYsoWiTvD9e^h5ZW%|$f?D}JmTHwn5
zarJ&vdu4A#-JqL8SvBg}Mwr?%nb6fz;S2;%hDy!^PrSx1&EiW}gyBw!9oKs^w(R#m
za@fHVp_m6RZ`4L_kpk-Z)X&7tURVCbW?3<%sQA0qNcA+S-1tjLK&GW1xZ!3-`#4$U
zP1FnY20cEem;FbxauXMLC*Fic2<oT0=u$x_EvxGb)#7&*J@LH6%_3hscu(<QGOb6W
z>@_3Gp8gl)_YWffwP}p*!0LXpFeZfr>@%&2W1Q*qV13B2fKX--tf$?p-JJFM)Z?<A
z$Gg`!p%*VEL)%Zyf|8lS>DEQO)YMev1Tv%Zqi0gDJL|Ut;)<0VIG*1uXDL+AZ-4~u
zjiQwu48Jx{xG?g6udDAnDErbFbp`NTCIGQ!zBE$^&GqF`v`}E7{cp356Ue<g6QRUK
zbKK*#AO@NW{vblkHV~e5-+vF`4;rsM1Zxd#Tz2pMG`91*CJwA1Wn*gN<>Frsw53B{
zeRnJ4q4&Zo{U!dxtis~aAgMIr=|}>$RA?g9p>8@P0o(Vevupso4(oVuWCo3fd)@6a
zaT;+=Oi>P?8A+q=)I$eLBM_%oMoCAvbK`0`wCeMO0Acf#?C)*VuP-^VRd+!;lOUW$
z@y~q2)+IMa9Wf%WXxkp<V`(V`vgsgoq4ZTc^93VpMv+6CYH?2INZY|RLG(ES)wszg
zH;{LKHerjN%WO#^$*SPuBA$hH2?4vPuDCahhM#`KO^XGc)j|aFp}K&$QVCzVciGF(
zK-bg-BgAF^@Ee0-Cuy4@@p_o!<jREm0x82EzlRu-*5*#EOF4I#H`mqm7Ax2Y2>PP~
z5Z83@kxM7YU%pe(xH7l6GxSA;%4k^}6OR7OOzD!b?}F6G93!V;tF<jO#6G4O#1<r~
zx2362ZHEWn(fSdL(_;W$VK+R|pH5P;I^<C{{TtsnpedJeuip(VzM^voc5RgUY&zx!
z{AQD|T-;D$w|AGG=P~Rl!B=`yoA%mOXwbUK@d6rjdYvy}d2?2!fLdu{rD^Af#-ESj
ze=}7gFYuQ25q}AZbi<2D+n6tM27)C_r?c80Qtd^@d=>UlfF91NQ2A4xTmC5(k8+!u
z8W-o{zx~j9dwyo-p`C2Nxb%Pgr*%UT!x@WF=Ons)FNi#*P`Lzb@>duc$GR_-Yc`O!
z`2kykiFQL>Cs)!w#%EBSLoFaxM1Vk;cu@?!r-1l&o=OpYt$5dejB_uV2k9<#NMvh>
zq>@^c#GC-l>i1nktCYu7IWD>icSxo?pwI}det+?S?6)Fnn*VHoN<QW8DGz!zkw3ce
zFN(FZwV&?dR|?<^_M_UQx%DS4@L*O(s#zaIlp{Wvn=byYa6LPUui&XO#ei-YgX!kk
zqxWW(LeWa&nw6l=rd9Gixp9NfKE$ro;B5!OH`44k<>DFWbyw}`qPMDW7&AET$-YyW
z^0{VvZ=T?$YD)fgk(DSoB4$3S`~xye4?$YT7mM(cBdPq@|9H?xZif=t+mI#Zixk75
zSz=^IUkJPeaR#oT8w%24^TehIoMxIldsv0pB3*lL5Jx)OoMt*4oLj<^DfwAtAJwJ$
zu3DH^-5ewscxi_o^z@Z><82YJWjIh;UBS+&5OyZKK&h_CKq;G%YbFz}9=EfzleM?_
zRP~>Pyn9fD$JR2Ji5d_+KC$tC`K_?P?C{8C$=P7lJofRu=h2v1H2l*k(p@}^?}?At
z-(&RYp30}__F-+RiMwngo`m;2!xTd}GT7!^({uijO*~x=)!aI8GU7<^mYu<4u0pby
zOUVz>IGk{;K-F&OfQ;{bT3iiZ;hzm;jmAO`*lM;kQm(n@CfB5V{s6R3CFXk!oM<4C
zExM4s<W&DhRg%u2FMEY%w=>uQNds+Q-29xDa2P)gmy!cXi;15lo6ZILwKJwpxEBcf
z@h50+@1K!2*p#9(f)%Tb3XCAUM^+Z;vff;`!U1iAf4EL~=m;yT2-Zp$F18zb7WhMP
z|Keo6==1nU{Xf2S>hFG&PMn^^AMS`REpY4j=xWbE*!R-ifegJNNtAUvQmU(FfO+ct
z&P$Jf48)%ySh48L>ec-R=<Ck1x^Qr|1G<zRx7J0ZlOxYD;^4}ixDlFJv({`@CZGF`
z=*K=;*P$60)GDs`Z0bUG0r%$FhU9qao#LE+f-M)F_dmW1iweV9K2pn%&xzX^Xhm2U
zP{2nMNg<4jO_<Rq>~_4Cogw6PDpZ-y+`kDs#71Z1yTt-4dnrrEc+5`A%OdPN{+Tf{
zo}>ybCCW{SVn}G`UNY?ZSo?8uATsaCzC~vm#xn2zw9;Z}{2JA=N^S28skG*Q!vnjD
zJwlOlBn@<ZCY_>V5wHgWgG+oZ%;mvZSK|oFs0=cY9tnK`13d;}J!0szc{JjfLzc&H
z!Fkman<knY_u?B-PMxwKt=@(>rcYDEt{j}dS_f~{5;Ke^laE@`jI}wDPSEQ(X1T8r
zr27jK6O*MsJky8yApUA|%vGKR)LwGf9*V_qe7(A`8h>Co8G{z)4ORF&zPN(51MWCG
zAW{n*F<?+DsO8o#kcS2#F5=Jy)S~*OkfH;Lo>|vr2t14eEP<lJSgq2e_$Tp)?dj?H
zFa1HC!NgR<M{L&79tt!3<auiF6^v%7lK)lvo1<SyzswL(CT=L2*jHLI*A0__MV)?w
zx8MaA(bxJjwS7j`W|;k6C;s&;5pFQ6TP|wv=jghxHf9l@KeKe@=j`k}=~6cQ;alkZ
zG!2q!4zrO4{%Lc~1{C=v;yLN)l6S#az%e)+LNZxU7yfVf47fa*l1|a({#NLe7FwBm
zOl;wadPA^hFnGyk4qJ={vLK_oshguwQBfv`$H!}nBnkDrU`%`6e(xeq#baeUF3)>M
z#@<Md**8QT(W0*EFXcmAyr>{!Uxp+8fOK56@g7ccKQe49QRT2E{PB)AW+p*1QyGJZ
zjI!YYJMzkcY431Jl-AZl$_iDz>W;Hh+5HoX%~;O9`fa{W4Ij>-KUZ7oqpVo{2YRTV
zllFEjrk4Z&<@<g)Ln2e27y=ae>t4BggN#(;M1@}8=*CShN+zYBh>3`mH@eyEQuTb-
zN;R2WXRw&XRg1UjoP42sHF5KMCm<uoD_n0zYWT&s5~p|BFJ14UixII)E3`{X`XDE*
zCXx<Lno19E{hqxO{0>ks@Ae4^4HdI8N(MJ5v@-zvRX-;^%2=>`Cg~o$|Mn`hDk{%{
z;I)m6Xjtjkgyx!-(bCh?_cq=AZ%noN&tE);)~=B^tJ^ogxm!S4{+V3G6I1m60L&LM
A;Q#;t

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 7be565d51260d..0b0a2b13fb52a 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -381,7 +381,7 @@
     "nifi": {"requests", "packaging", "requests-gssapi"},
     "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"} | sqlglot_lib,
     "powerbi-report-server": powerbi_report_server,
-    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8"},
+    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.1"},
     "unity-catalog": databricks | sqllineage_lib,
 }
 
diff --git a/metadata-ingestion/tests/integration/vertica/ddl.sql b/metadata-ingestion/tests/integration/vertica/ddl.sql
index 59a71a1a1f7b5..ceebcd8e9ce2a 100644
--- a/metadata-ingestion/tests/integration/vertica/ddl.sql
+++ b/metadata-ingestion/tests/integration/vertica/ddl.sql
@@ -1,5 +1,4 @@
-\set AUTOCOMMIT on
-ALTER USER dbadmin IDENTIFIED BY 'abc123';
+
 
 -- Create a Top-k projection
 CREATE TABLE readings (meter_id INT, reading_date TIMESTAMP, reading_value FLOAT);
@@ -35,12 +34,16 @@ SELECT tokenize(phrase) OVER () FROM phrases;
 
 -- Create a temp table
 
-CREATE TEMPORARY TABLE sampletemp (a int, b int) ON COMMIT PRESERVE ROWS;
-INSERT INTO sampletemp VALUES(1,2);
+-- CREATE TEMPORARY TABLE sampletemp (a int, b int) ON COMMIT PRESERVE ROWS;
+-- INSERT INTO sampletemp VALUES(1,2);
 
 -- Create partition key
-ALTER TABLE store.store_orders_fact PARTITION BY date_ordered::DATE GROUP BY DATE_TRUNC('month', (date_ordered)::DATE);
-SELECT PARTITION_TABLE('store.store_orders_fact');
-CREATE PROJECTION ytd_orders AS SELECT * FROM store.store_orders_fact ORDER BY date_ordered
-    ON PARTITION RANGE BETWEEN date_trunc('year',now())::date AND NULL;
+-- ALTER TABLE store.store_orders_fact PARTITION BY date_ordered::DATE GROUP BY DATE_TRUNC('month', (date_ordered)::DATE);
+-- SELECT PARTITION_TABLE('store.store_orders_fact');
+-- CREATE PROJECTION ytd_orders AS SELECT * FROM store.store_orders_fact ORDER BY date_ordered
+--     ON PARTITION RANGE BETWEEN date_trunc('year',now())::date AND NULL;
+
+
+
+
 SELECT start_refresh();
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/vertica/docker-compose.yml b/metadata-ingestion/tests/integration/vertica/docker-compose.yml
index 84af5c32a60e3..1ba7990c826b2 100644
--- a/metadata-ingestion/tests/integration/vertica/docker-compose.yml
+++ b/metadata-ingestion/tests/integration/vertica/docker-compose.yml
@@ -6,7 +6,7 @@ services:
       APP_DB_USER: "dbadmin"
       APP_DB_PASSWORD: "abc123"
     container_name: vertica-ce
-    image: vertica/vertica-ce:12.0.2-0
+    image: vertica/vertica-ce:23.4.0-0
     ports:
       - "5433:5433"
       - "5444:5444"
diff --git a/metadata-ingestion/tests/integration/vertica/test_vertica.py b/metadata-ingestion/tests/integration/vertica/test_vertica.py
index fe306d1d0b2b8..94ad33ba21ce4 100644
--- a/metadata-ingestion/tests/integration/vertica/test_vertica.py
+++ b/metadata-ingestion/tests/integration/vertica/test_vertica.py
@@ -1,6 +1,5 @@
 import subprocess
-import time
-from typing import List, Optional
+from typing import List
 
 import pytest
 from freezegun import freeze_time
@@ -17,13 +16,12 @@ def test_resources_dir(pytestconfig):
     return pytestconfig.rootpath / "tests/integration/vertica"
 
 
-def is_vertica_responsive(
-    container_name: str, port: int, hostname: Optional[str]
-) -> bool:
-    if hostname:
-        cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' "
-    ret = subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL)
-
+def is_vertica_responsive(container_name: str) -> bool:
+    cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' "
+    ret = subprocess.run(
+        cmd,
+        shell=True,
+    )
     return ret.returncode == 0
 
 
@@ -37,28 +35,22 @@ def vertica_runner(docker_compose_runner, test_resources_dir):
             "vertica-ce",
             5433,
             timeout=120,
-            checker=lambda: is_vertica_responsive(
-                "vertica-ce", 5433, hostname="vertica-ce"
-            ),
+            checker=lambda: is_vertica_responsive("vertica-ce"),
         )
 
         commands = """
                     docker cp tests/integration/vertica/ddl.sql vertica-ce:/home/dbadmin/ &&
-                    docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql
+                    docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql"
                 """
 
         ret = subprocess.run(commands, shell=True, stdout=subprocess.DEVNULL)
-        # waiting for vertica to create default table and system table and ml models
-        time.sleep(60)
 
-        assert ret.returncode >= 1
+        assert ret.returncode == 0
 
         yield docker_services
 
 
-# Test needs more work to be done , currently it is working fine.
 @freeze_time(FROZEN_TIME)
-@pytest.mark.skip("Failing in CI, cmd failing with exit code 1")
 @pytest.mark.integration
 def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica"
@@ -72,7 +64,7 @@ def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
     ignore_paths: List[str] = [
         r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['create_time'\]",
         r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['table_size'\]",
-        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['projection_size'\]",
+        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['Projection_size'\]",
         r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['ROS_Count'\]",
         r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['cluster_size'\]",
         r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['udx_language'\]",
diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json
index 44a5e07d7b996..ef535158165da 100644
--- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json
+++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json
@@ -11,7 +11,7 @@
                 "env": "PROD",
                 "database": "vmart",
                 "cluster_type": "Enterprise",
-                "cluster_size": "122 GB",
+                "cluster_size": "101 GB",
                 "subcluster": " ",
                 "communal_storage_path": ""
             },
@@ -20,7 +20,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -35,7 +36,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -50,7 +52,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -67,7 +70,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -82,7 +86,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -97,16 +102,17 @@
                 "env": "PROD",
                 "database": "vmart",
                 "schema": "public",
-                "projection_count": "9",
-                "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_prune, xgb_save_model, yule_walker, ",
-                "udx_language": "ComplexTypesLib -- Functions for Complex Types |  DelimitedExportLib -- Delimited data export package |  JsonExportLib -- Json data export package |  MachineLearningLib -- Machine learning package |  OrcExportLib -- Orc export package |  ParquetExportLib -- Parquet export package |  ApproximateLib -- Approximate package |  FlexTableLib -- Flexible Tables Data Load and Query |  KafkaLib -- Kafka streaming load and export |  PlaceLib -- Geospatial package |  VoltageSecureLib -- Voltage SecureData Connector |  "
+                "projection_count": "12",
+                "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ",
+                "udx_language": "ComplexTypesLib -- Functions for Complex Types |  DelimitedExportLib -- Delimited data export package |  JsonExportLib -- Json data export package |  MachineLearningLib -- Machine learning package |  OrcExportLib -- Orc export package |  ParquetExportLib -- Parquet export package |  ApproximateLib -- Approximate package |  FlexTableLib -- Flexible Tables Data Load and Query |  KafkaLib -- Kafka streaming load and export |  PlaceLib -- Geospatial package |  VoltageSecureLib -- Voltage SecureData Connector |  TransformFunctions -- User-defined Python library |  "
             },
             "name": "public"
         }
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -121,7 +127,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -136,7 +143,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -153,7 +161,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -168,7 +177,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -188,7 +198,184 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "create_time": "2023-10-13 11:23:05.308022+00:00",
+                            "table_size": "0 KB"
+                        },
+                        "name": "clicks",
+                        "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.clicks",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "user_id",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "page_id",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "click_time",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "TIMESTAMP_WITH_PRECISION()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -212,7 +399,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -227,7 +415,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -243,7 +432,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.358215+00:00"
+                            "create_time": "2023-10-13 11:22:37.846965+00:00",
+                            "table_size": "2119 KB"
                         },
                         "name": "customer_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -551,7 +741,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -568,7 +759,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -592,7 +784,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -616,7 +809,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -631,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -647,7 +842,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.368954+00:00"
+                            "create_time": "2023-10-13 11:22:37.857152+00:00",
+                            "table_size": "138 KB"
                         },
                         "name": "date_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -955,7 +1151,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -972,7 +1169,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -996,7 +1194,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1020,7 +1219,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1035,7 +1235,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1051,7 +1252,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.375896+00:00"
+                            "create_time": "2023-10-13 11:22:37.863745+00:00",
+                            "table_size": "327 KB"
                         },
                         "name": "employee_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -1320,7 +1522,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1337,7 +1540,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1361,7 +1565,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1385,7 +1590,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1400,7 +1606,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1416,7 +1623,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.385843+00:00"
+                            "create_time": "2023-10-13 11:22:37.873181+00:00",
+                            "table_size": "2564 KB"
                         },
                         "name": "inventory_fact",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -1529,7 +1737,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1546,7 +1755,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1570,12 +1780,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -1594,12 +1805,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -1609,13 +1821,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -1625,16 +1838,17 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.362016+00:00"
+                            "create_time": "2023-10-13 11:23:05.408507+00:00",
+                            "table_size": "0 KB"
                         },
-                        "name": "product_dimension",
+                        "name": "phrases",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.product_dimension",
+                        "schemaName": "public.phrases",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -1653,33 +1867,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "product_key",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": true
-                            },
-                            {
-                                "fieldPath": "product_version",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "product_description",
+                                "fieldPath": "phrase",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -1690,76 +1878,252 @@
                                 "nativeDataType": "VARCHAR(length=128)",
                                 "recursive": false,
                                 "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "sku_number",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=32)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "category_description",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=32)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "department_description",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=32)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "package_type_description",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=32)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "package_size",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=32)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "fat_content",
-                                "nullable": true,
-                                "description": "",
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "create_time": "2023-10-13 11:22:37.850505+00:00",
+                            "table_size": "19 KB"
+                        },
+                        "name": "product_dimension",
+                        "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.product_dimension",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "product_key",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "product_version",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "product_description",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=128)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "sku_number",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "category_description",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "department_description",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "package_type_description",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "package_size",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "fat_content",
+                                "nullable": true,
+                                "description": "",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -1933,7 +2297,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2315,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1974,7 +2340,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1998,7 +2365,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2013,7 +2381,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2029,7 +2398,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.365453+00:00"
+                            "create_time": "2023-10-13 11:22:37.853878+00:00",
+                            "table_size": "3 KB"
                         },
                         "name": "promotion_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -2220,7 +2590,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2237,7 +2608,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2261,12 +2633,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -2285,12 +2658,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -2300,13 +2674,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -2316,16 +2691,17 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.379273+00:00"
+                            "create_time": "2023-10-13 11:23:05.296044+00:00",
+                            "table_size": "0 KB"
                         },
-                        "name": "shipping_dimension",
+                        "name": "readings",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.shipping_dimension",
+                        "schemaName": "public.readings",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -2344,7 +2720,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "shipping_key",
+                                "fieldPath": "meter_id",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -2354,39 +2730,215 @@
                                 },
                                 "nativeDataType": "INTEGER()",
                                 "recursive": false,
-                                "isPartOfKey": true
+                                "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ship_type",
+                                "fieldPath": "reading_date",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=30)",
+                                "nativeDataType": "TIMESTAMP_WITH_PRECISION()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ship_mode",
+                                "fieldPath": "reading_value",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=10)",
+                                "nativeDataType": "FLOAT()",
                                 "recursive": false,
                                 "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "ship_carrier",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "create_time": "2023-10-13 11:22:37.867119+00:00",
+                            "table_size": "1 KB"
+                        },
+                        "name": "shipping_dimension",
+                        "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.shipping_dimension",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "shipping_key",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": true
+                            },
+                            {
+                                "fieldPath": "ship_type",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=30)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ship_mode",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=10)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ship_carrier",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
@@ -2403,7 +2955,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2420,7 +2973,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2444,7 +2998,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2468,7 +3023,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2483,7 +3039,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2499,7 +3056,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.372409+00:00"
+                            "create_time": "2023-10-13 11:22:37.860541+00:00",
+                            "table_size": "1 KB"
                         },
                         "name": "vendor_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -2638,7 +3196,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2655,7 +3214,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2679,7 +3239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2703,7 +3264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2718,7 +3280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2734,7 +3297,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:38:20.045598+00:00"
+                            "create_time": "2023-10-13 11:23:04.970568+00:00",
+                            "table_size": "0 KB"
                         },
                         "name": "vmart_load_success",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -2782,7 +3346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2799,7 +3364,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2823,7 +3389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2847,7 +3414,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2862,7 +3430,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2878,7 +3447,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.382549+00:00"
+                            "create_time": "2023-10-13 11:22:37.870169+00:00",
+                            "table_size": "2 KB"
                         },
                         "name": "warehouse_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -2991,7 +3561,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3008,7 +3579,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3032,12 +3604,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -3056,12 +3629,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -3071,13 +3645,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -3087,23 +3662,19 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "ROS_Count": "1",
-                            "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(date_dimension.date_key)",
-                            "projection_size": "138 KB",
-                            "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
-                            "Projection_Cached": "False"
+                            "create_time": "2023-10-13 11:23:05.319029+00:00",
+                            "table_size": "0 KB",
+                            "view_definition": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state",
+                            "is_view": "True"
                         },
-                        "name": "date_dimension_super",
-                        "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
+                        "name": "sampleview",
+                        "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.date_dimension_super",
+                        "schemaName": "public.sampleview",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -3122,7 +3693,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "date_key",
+                                "fieldPath": "SUM",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3135,33 +3706,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "date",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.DateType": {}
-                                    }
-                                },
-                                "nativeDataType": "DATE()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "full_date_description",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=18)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "day_of_week",
+                                "fieldPath": "customer_state",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3169,228 +3714,7 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=9)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "day_number_in_calendar_month",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "day_number_in_calendar_year",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "day_number_in_fiscal_month",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "day_number_in_fiscal_year",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "last_day_in_week_indicator",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "last_day_in_month_indicator",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_week_number_in_year",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_month_name",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=9)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_month_number_in_year",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_year_month",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=7)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_quarter",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_year_quarter",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=7)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_half_year",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "calendar_year",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "holiday_indicator",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=10)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "weekday_indicator",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=7)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "selling_season",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=2)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -3402,29 +3726,49 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "Projections"
+                "View"
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -3435,7 +3779,15 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)",
+                    "type": "TRANSFORMED"
+                },
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -3443,12 +3795,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -3467,12 +3820,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -3491,12 +3845,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -3506,13 +3861,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -3524,21 +3880,21 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)",
-                            "projection_size": "19 KB",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(date_dimension.date_key)",
+                            "Projection_size": "138 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "product_dimension_super",
+                        "name": "date_dimension_super",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.product_dimension_super",
+                        "schemaName": "public.date_dimension_super",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -3557,7 +3913,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "product_key",
+                                "fieldPath": "date_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3570,20 +3926,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "product_version",
+                                "fieldPath": "date",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "DATE()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "product_description",
+                                "fieldPath": "full_date_description",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3591,12 +3947,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=128)",
+                                "nativeDataType": "VARCHAR(length=18)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "sku_number",
+                                "fieldPath": "day_of_week",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3604,64 +3960,64 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "VARCHAR(length=9)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "category_description",
+                                "fieldPath": "day_number_in_calendar_month",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "department_description",
+                                "fieldPath": "day_number_in_calendar_year",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "package_type_description",
+                                "fieldPath": "day_number_in_fiscal_month",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "package_size",
+                                "fieldPath": "day_number_in_fiscal_year",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "fat_content",
+                                "fieldPath": "last_day_in_week_indicator",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3674,20 +4030,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "diet_type",
+                                "fieldPath": "last_day_in_month_indicator",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "weight",
+                                "fieldPath": "calendar_week_number_in_year",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3700,7 +4056,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "weight_units_of_measure",
+                                "fieldPath": "calendar_month_name",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3708,12 +4064,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "VARCHAR(length=9)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "shelf_width",
+                                "fieldPath": "calendar_month_number_in_year",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3726,20 +4082,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "shelf_height",
+                                "fieldPath": "calendar_year_month",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "CHAR(length=7)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "shelf_depth",
+                                "fieldPath": "calendar_quarter",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3752,20 +4108,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "product_price",
+                                "fieldPath": "calendar_year_quarter",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "CHAR(length=7)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "product_cost",
+                                "fieldPath": "calendar_half_year",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3778,7 +4134,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "lowest_competitor_price",
+                                "fieldPath": "calendar_year",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -3791,41 +4147,41 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "highest_competitor_price",
+                                "fieldPath": "holiday_indicator",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "VARCHAR(length=10)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "average_competitor_price",
+                                "fieldPath": "weekday_indicator",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "CHAR(length=7)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "discontinued_flag",
+                                "fieldPath": "selling_season",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "VARCHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -3837,12 +4193,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -3854,12 +4211,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -3870,7 +4228,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -3878,12 +4236,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -3902,12 +4261,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -3926,12 +4286,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -3941,13 +4302,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -3959,21 +4321,21 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(promotion_dimension.promotion_key)",
-                            "projection_size": "3 KB",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)",
+                            "Projection_size": "19 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "promotion_dimension_super",
+                        "name": "product_dimension_super",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.promotion_dimension_super",
+                        "schemaName": "public.product_dimension_super",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -3992,7 +4354,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "promotion_key",
+                                "fieldPath": "product_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4005,7 +4367,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "promotion_name",
+                                "fieldPath": "product_version",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "product_description",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4018,7 +4393,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "price_reduction_type",
+                                "fieldPath": "sku_number",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4026,12 +4401,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "promotion_media_type",
+                                "fieldPath": "category_description",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4039,12 +4414,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ad_type",
+                                "fieldPath": "department_description",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4052,12 +4427,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "display_type",
+                                "fieldPath": "package_type_description",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4065,12 +4440,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "coupon_type",
+                                "fieldPath": "package_size",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4078,12 +4453,25 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "ad_media_name",
+                                "fieldPath": "fat_content",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "diet_type",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4091,12 +4479,25 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "display_provider",
+                                "fieldPath": "weight",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "weight_units_of_measure",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4104,12 +4505,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=128)",
+                                "nativeDataType": "CHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "promotion_cost",
+                                "fieldPath": "shelf_width",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4122,28 +4523,106 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "promotion_begin_date",
+                                "fieldPath": "shelf_height",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "DATE()",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "promotion_end_date",
+                                "fieldPath": "shelf_depth",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "DATE()",
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "product_price",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "product_cost",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "lowest_competitor_price",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "highest_competitor_price",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "average_competitor_price",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "discontinued_flag",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -4155,12 +4634,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -4172,12 +4652,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -4188,7 +4669,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -4196,12 +4677,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -4220,12 +4702,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -4244,12 +4727,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -4259,13 +4743,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -4277,21 +4762,21 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(vendor_dimension.vendor_key)",
-                            "projection_size": "1 KB",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(promotion_dimension.promotion_key)",
+                            "Projection_size": "3 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "vendor_dimension_super",
+                        "name": "promotion_dimension_super",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.vendor_dimension_super",
+                        "schemaName": "public.promotion_dimension_super",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -4310,7 +4795,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "vendor_key",
+                                "fieldPath": "promotion_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4323,7 +4808,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "vendor_name",
+                                "fieldPath": "promotion_name",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4331,12 +4816,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=64)",
+                                "nativeDataType": "VARCHAR(length=128)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "vendor_address",
+                                "fieldPath": "price_reduction_type",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4344,12 +4829,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=64)",
+                                "nativeDataType": "VARCHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "vendor_city",
+                                "fieldPath": "promotion_media_type",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4357,12 +4842,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=64)",
+                                "nativeDataType": "VARCHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "vendor_state",
+                                "fieldPath": "ad_type",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4370,12 +4855,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=2)",
+                                "nativeDataType": "VARCHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "vendor_region",
+                                "fieldPath": "display_type",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4388,7 +4873,46 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "deal_size",
+                                "fieldPath": "coupon_type",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ad_media_name",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "display_provider",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=128)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "promotion_cost",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4401,7 +4925,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "last_deal_update",
+                                "fieldPath": "promotion_begin_date",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "promotion_end_date",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4421,12 +4958,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -4438,12 +4976,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -4454,7 +4993,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -4462,12 +5001,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -4486,12 +5026,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -4510,12 +5051,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -4525,13 +5067,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -4543,21 +5086,21 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(customer_dimension.customer_key)",
-                            "projection_size": "2119 KB",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(vendor_dimension.vendor_key)",
+                            "Projection_size": "1 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "customer_dimension_super",
+                        "name": "vendor_dimension_super",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.customer_dimension_super",
+                        "schemaName": "public.vendor_dimension_super",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -4576,7 +5119,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "customer_key",
+                                "fieldPath": "vendor_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4589,46 +5132,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_type",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=16)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "customer_name",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=256)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "customer_gender",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=8)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "title",
+                                "fieldPath": "vendor_name",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4636,25 +5140,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=8)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "household_id",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "VARCHAR(length=64)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_address",
+                                "fieldPath": "vendor_address",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4662,12 +5153,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=256)",
+                                "nativeDataType": "VARCHAR(length=64)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_city",
+                                "fieldPath": "vendor_city",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4680,7 +5171,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_state",
+                                "fieldPath": "vendor_state",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4693,20 +5184,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_region",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=64)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "marital_status",
+                                "fieldPath": "vendor_region",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4719,72 +5197,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_age",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "number_of_children",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "annual_income",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "occupation",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=64)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "largest_bill_amount",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "store_membership_card",
+                                "fieldPath": "deal_size",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4797,46 +5210,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "customer_since",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.DateType": {}
-                                    }
-                                },
-                                "nativeDataType": "DATE()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "deal_stage",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=32)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "deal_size",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "last_deal_update",
+                                "fieldPath": "last_deal_update",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -4856,12 +5230,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -4873,12 +5248,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -4889,7 +5265,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -4897,12 +5273,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -4921,12 +5298,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -4945,12 +5323,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -4960,13 +5339,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -4978,21 +5358,21 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(employee_dimension.employee_key)",
-                            "projection_size": "327 KB",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(customer_dimension.customer_key)",
+                            "Projection_size": "2119 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "employee_dimension_super",
+                        "name": "customer_dimension_super",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.employee_dimension_super",
+                        "schemaName": "public.customer_dimension_super",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -5011,7 +5391,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "employee_key",
+                                "fieldPath": "customer_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5024,7 +5404,33 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_gender",
+                                "fieldPath": "customer_type",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=16)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_name",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=256)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_gender",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5037,7 +5443,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "courtesy_title",
+                                "fieldPath": "title",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5050,7 +5456,33 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_first_name",
+                                "fieldPath": "household_id",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_address",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=256)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_city",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5063,7 +5495,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_middle_initial",
+                                "fieldPath": "customer_state",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5071,12 +5503,12 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=8)",
+                                "nativeDataType": "CHAR(length=2)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_last_name",
+                                "fieldPath": "customer_region",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5089,7 +5521,20 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_age",
+                                "fieldPath": "marital_status",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_age",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5102,33 +5547,33 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "hire_date",
+                                "fieldPath": "number_of_children",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "DATE()",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_street_address",
+                                "fieldPath": "annual_income",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=256)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_city",
+                                "fieldPath": "occupation",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5141,20 +5586,46 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_state",
+                                "fieldPath": "largest_bill_amount",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=2)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "employee_region",
+                                "fieldPath": "store_membership_card",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "customer_since",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "deal_stage",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5162,25 +5633,1087 @@
                                         "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=32)",
+                                "nativeDataType": "VARCHAR(length=32)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "job_title",
+                                "fieldPath": "deal_size",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "last_deal_update",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Projections"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "ROS_Count": "1",
+                            "Projection_Type": "is_super_projection",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(employee_dimension.employee_key)",
+                            "Projection_size": "327 KB",
+                            "Partition_Key": "Not Available",
+                            "Number_Of_Partitions": "0",
+                            "Projection_Cached": "False"
+                        },
+                        "name": "employee_dimension_super",
+                        "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.employee_dimension_super",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "employee_key",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_gender",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=8)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "courtesy_title",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=8)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_first_name",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=64)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_middle_initial",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=8)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_last_name",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=64)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_age",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "hire_date",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                    }
+                                },
+                                "nativeDataType": "DATE()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_street_address",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=256)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_city",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=64)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_state",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=2)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "employee_region",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "job_title",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=64)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "reports_to",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "salaried_flag",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "annual_salary",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "hourly_rate",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "FLOAT()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "vacation_days",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Projections"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "ROS_Count": "1",
+                            "Projection_Type": "is_super_projection",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(warehouse_dimension.warehouse_key)",
+                            "Projection_size": "2 KB",
+                            "Partition_Key": "Not Available",
+                            "Number_Of_Partitions": "0",
+                            "Projection_Cached": "False"
+                        },
+                        "name": "warehouse_dimension_super",
+                        "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.warehouse_dimension_super",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "warehouse_key",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warehouse_name",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=20)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warehouse_address",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=256)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warehouse_city",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=60)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warehouse_state",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=2)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "warehouse_region",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=32)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Projections"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "ROS_Count": "1",
+                            "Projection_Type": "is_super_projection",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(shipping_dimension.shipping_key)",
+                            "Projection_size": "1 KB",
+                            "Partition_Key": "Not Available",
+                            "Number_Of_Partitions": "0",
+                            "Projection_Cached": "False"
+                        },
+                        "name": "shipping_dimension_super",
+                        "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.shipping_dimension_super",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "shipping_key",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INTEGER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ship_type",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=30)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ship_mode",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=10)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "ship_carrier",
+                                "nullable": true,
+                                "description": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "CHAR(length=20)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Projections"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:343f520ad0fb3259b298736800bb1385",
+                    "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385"
+                },
+                {
+                    "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4",
+                    "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:dbadmin",
+                    "type": "DATAOWNER"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "ROS_Count": "1",
+                            "Projection_Type": "is_super_projection",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)",
+                            "Projection_size": "2564 KB",
+                            "Partition_Key": "Not Available",
+                            "Number_Of_Partitions": "0",
+                            "Projection_Cached": "False"
+                        },
+                        "name": "inventory_fact_super",
+                        "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "public.inventory_fact_super",
+                        "platform": "urn:li:dataPlatform:vertica",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "date_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=64)",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "reports_to",
+                                "fieldPath": "product_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5193,7 +6726,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "salaried_flag",
+                                "fieldPath": "product_version",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5206,7 +6739,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "annual_salary",
+                                "fieldPath": "warehouse_key",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5219,7 +6752,7 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "hourly_rate",
+                                "fieldPath": "qty_in_stock",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5227,20 +6760,20 @@
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "FLOAT()",
+                                "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "vacation_days",
+                                "fieldPath": "inventory_date",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                        "com.linkedin.pegasus2avro.schema.DateType": {}
                                     }
                                 },
-                                "nativeDataType": "INTEGER()",
+                                "nativeDataType": "DATE()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -5252,12 +6785,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -5269,12 +6803,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -5285,7 +6820,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -5293,12 +6828,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -5317,12 +6853,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -5341,12 +6878,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -5356,13 +6894,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -5373,22 +6912,22 @@
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
                             "ROS_Count": "1",
-                            "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(warehouse_dimension.warehouse_key)",
-                            "projection_size": "2 KB",
+                            "Projection_Type": "is_aggregate_projection, has_expressions",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(readings.meter_id)",
+                            "Projection_size": "0 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "warehouse_dimension_super",
+                        "name": "readings_topk",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.warehouse_dimension_super",
+                        "schemaName": "public.readings_topk",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -5407,7 +6946,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "warehouse_key",
+                                "fieldPath": "meter_id",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5420,67 +6959,28 @@
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "warehouse_name",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=20)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "warehouse_address",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=256)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "warehouse_city",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "VARCHAR(length=60)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "warehouse_state",
+                                "fieldPath": "recent_date",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
                                     }
                                 },
-                                "nativeDataType": "CHAR(length=2)",
+                                "nativeDataType": "TIMESTAMP_WITH_PRECISION()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             },
                             {
-                                "fieldPath": "warehouse_region",
+                                "fieldPath": "recent_value",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                     }
                                 },
-                                "nativeDataType": "VARCHAR(length=32)",
+                                "nativeDataType": "FLOAT()",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -5492,12 +6992,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -5509,12 +7010,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -5525,7 +7027,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -5533,12 +7035,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -5557,12 +7060,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -5581,12 +7085,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -5596,13 +7101,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -5613,22 +7119,22 @@
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
                             "ROS_Count": "1",
-                            "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(shipping_dimension.shipping_key)",
-                            "projection_size": "1 KB",
+                            "Projection_Type": "is_aggregate_projection, has_expressions",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(clicks.page_id, (clicks.click_time)::date)",
+                            "Projection_size": "0 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "shipping_dimension_super",
+                        "name": "clicks_agg",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.shipping_dimension_super",
+                        "schemaName": "public.clicks_agg",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -5647,7 +7153,7 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "shipping_key",
+                                "fieldPath": "page_id",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
@@ -5658,45 +7164,6 @@
                                 "nativeDataType": "INTEGER()",
                                 "recursive": false,
                                 "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "ship_type",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=30)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "ship_mode",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=10)",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "ship_carrier",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "CHAR(length=20)",
-                                "recursive": false,
-                                "isPartOfKey": false
                             }
                         ]
                     }
@@ -5706,12 +7173,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -5723,12 +7191,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -5739,7 +7208,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -5747,12 +7216,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -5771,12 +7241,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -5795,12 +7266,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
@@ -5810,13 +7282,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.common.Status": {
@@ -5828,21 +7301,21 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
-                            "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)",
-                            "projection_size": "2566 KB",
+                            "Is_Segmented": "True",
+                            "Segmentation_key": "hash(phrases.phrase)",
+                            "Projection_size": "0 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
-                        "name": "inventory_fact_super",
+                        "name": "phrases_super",
                         "description": "Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs",
                         "tags": []
                     }
                 },
                 {
                     "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "public.inventory_fact_super",
+                        "schemaName": "public.phrases_super",
                         "platform": "urn:li:dataPlatform:vertica",
                         "version": 0,
                         "created": {
@@ -5861,80 +7334,15 @@
                         },
                         "fields": [
                             {
-                                "fieldPath": "date_key",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "product_key",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "product_version",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "warehouse_key",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "qty_in_stock",
-                                "nullable": true,
-                                "description": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "INTEGER()",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "inventory_date",
+                                "fieldPath": "phrase",
                                 "nullable": true,
                                 "description": "",
                                 "type": {
                                     "type": {
-                                        "com.linkedin.pegasus2avro.schema.DateType": {}
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                     }
                                 },
-                                "nativeDataType": "DATE()",
+                                "nativeDataType": "VARCHAR(length=128)",
                                 "recursive": false,
                                 "isPartOfKey": false
                             }
@@ -5946,12 +7354,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -5963,12 +7372,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -5979,7 +7389,7 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)",
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)",
                     "type": "TRANSFORMED"
                 }
             ]
@@ -5987,12 +7397,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
@@ -6011,7 +7422,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6035,7 +7447,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6050,7 +7463,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6065,7 +7479,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6082,7 +7497,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6097,7 +7513,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6117,7 +7534,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6141,7 +7559,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6156,7 +7575,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6172,7 +7592,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.393181+00:00"
+                            "create_time": "2023-10-13 11:22:37.879951+00:00",
+                            "table_size": "2 KB"
                         },
                         "name": "store_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -6441,7 +7862,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6458,7 +7880,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6482,7 +7905,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6506,7 +7930,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6521,7 +7946,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6537,7 +7963,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.404717+00:00"
+                            "create_time": "2023-10-13 11:22:37.890717+00:00",
+                            "table_size": "8646 KB"
                         },
                         "name": "store_orders_fact",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -6819,7 +8246,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6836,7 +8264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6860,7 +8289,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6884,7 +8314,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6899,7 +8330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6915,7 +8347,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.396731+00:00"
+                            "create_time": "2023-10-13 11:22:37.883186+00:00",
+                            "table_size": "225060 KB"
                         },
                         "name": "store_sales_fact",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -7171,7 +8604,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7188,7 +8622,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7212,7 +8647,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7236,7 +8672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7251,7 +8688,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7269,11 +8707,11 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
+                            "Is_Segmented": "True",
                             "Segmentation_key": "hash(store_dimension.store_key)",
-                            "projection_size": "2 KB",
+                            "Projection_size": "2 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
                         "name": "store_dimension_super",
@@ -7543,7 +8981,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7560,7 +8999,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7584,7 +9024,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7608,7 +9049,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7632,7 +9074,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7647,7 +9090,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7665,11 +9109,11 @@
                         "customProperties": {
                             "ROS_Count": "2",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
+                            "Is_Segmented": "True",
                             "Segmentation_key": "hash(store_sales_fact.date_key, store_sales_fact.product_key, store_sales_fact.product_version, store_sales_fact.store_key, store_sales_fact.promotion_key, store_sales_fact.customer_key, store_sales_fact.employee_key, store_sales_fact.pos_transaction_number)",
-                            "projection_size": "225089 KB",
+                            "Projection_size": "225060 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
                         "name": "store_sales_fact_super",
@@ -7926,7 +9370,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7943,7 +9388,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7967,7 +9413,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -7991,7 +9438,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8015,7 +9463,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8030,7 +9479,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8048,11 +9498,11 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
+                            "Is_Segmented": "True",
                             "Segmentation_key": "hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)",
-                            "projection_size": "8648 KB",
+                            "Projection_size": "8646 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
                         "name": "store_orders_fact_super",
@@ -8335,7 +9785,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8352,7 +9803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8376,7 +9828,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8400,7 +9853,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8424,7 +9878,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8439,7 +9894,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8454,7 +9910,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8471,7 +9928,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8486,7 +9944,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8506,7 +9965,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8530,7 +9990,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8545,7 +10006,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8561,7 +10023,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.415595+00:00"
+                            "create_time": "2023-10-13 11:22:37.900841+00:00",
+                            "table_size": "6 KB"
                         },
                         "name": "call_center_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -8752,7 +10215,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8769,7 +10233,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8793,7 +10258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8817,7 +10283,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8832,7 +10299,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8848,7 +10316,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.412266+00:00"
+                            "create_time": "2023-10-13 11:22:37.897788+00:00",
+                            "table_size": "9 KB"
                         },
                         "name": "online_page_dimension",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -8961,7 +10430,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -8978,7 +10448,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9002,7 +10473,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9026,7 +10498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9041,7 +10514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9057,7 +10531,8 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "create_time": "2023-07-26 06:37:53.419260+00:00"
+                            "create_time": "2023-10-13 11:22:37.903963+00:00",
+                            "table_size": "182356 KB"
                         },
                         "name": "online_sales_fact",
                         "description": "References the properties of a native table in Vertica.             Vertica physically stores table data in projections, which are collections of table columns.             Projections store data in a format that optimizes query execution.             In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ",
@@ -9352,7 +10827,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9369,7 +10845,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9393,7 +10870,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9417,7 +10895,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9432,7 +10911,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9450,11 +10930,11 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
+                            "Is_Segmented": "True",
                             "Segmentation_key": "hash(online_page_dimension.online_page_key)",
-                            "projection_size": "9 KB",
+                            "Projection_size": "9 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
                         "name": "online_page_dimension_super",
@@ -9568,7 +11048,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9585,7 +11066,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9609,7 +11091,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9633,7 +11116,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9657,7 +11141,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9672,7 +11157,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9690,11 +11176,11 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
+                            "Is_Segmented": "True",
                             "Segmentation_key": "hash(call_center_dimension.call_center_key)",
-                            "projection_size": "6 KB",
+                            "Projection_size": "6 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
                         "name": "call_center_dimension_super",
@@ -9886,7 +11372,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9903,7 +11390,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9927,7 +11415,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9951,7 +11440,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9975,7 +11465,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -9990,7 +11481,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -10008,11 +11500,11 @@
                         "customProperties": {
                             "ROS_Count": "1",
                             "Projection_Type": "is_super_projection",
-                            "is_segmented": "True",
+                            "Is_Segmented": "True",
                             "Segmentation_key": "hash(online_sales_fact.sale_date_key, online_sales_fact.ship_date_key, online_sales_fact.product_key, online_sales_fact.product_version, online_sales_fact.customer_key, online_sales_fact.call_center_key, online_sales_fact.online_page_key, online_sales_fact.shipping_key)",
-                            "projection_size": "182385 KB",
+                            "Projection_size": "182356 KB",
                             "Partition_Key": "Not Available",
-                            "Partition_Size": "0",
+                            "Number_Of_Partitions": "0",
                             "Projection_Cached": "False"
                         },
                         "name": "online_sales_fact_super",
@@ -10308,7 +11800,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -10325,7 +11818,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -10349,7 +11843,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -10373,7 +11868,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "vertica-2020_04_14-07_00_00"
+        "runId": "vertica-2020_04_14-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml
index ebd800ee09ff5..a182e54bd53c7 100644
--- a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml
+++ b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml
@@ -5,6 +5,13 @@ source:
     database: Vmart
     username: dbadmin
     password: abc123
+    include_tables: true
+    include_views: true
+    include_projections: true
+    include_models: true
+    include_view_lineage: true
+    include_projection_lineage: true
+
 
 sink:
   type: file

From d2eb42373fa1d36011c020500668da8cf863e165 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 18 Oct 2023 11:34:45 -0400
Subject: [PATCH 063/259] fix(ingest/sqlalchemy): Fix URL parsing when
 sqlalchemy_uri provided (#9032)

---
 .../ingestion/source/sql/sql_config.py        | 47 +++++++++++--------
 .../source/sql/two_tier_sql_source.py         | 31 ++++++++----
 .../tests/unit/test_athena_source.py          | 12 ++---
 .../tests/unit/test_clickhouse_source.py      |  4 +-
 .../tests/unit/test_snowflake_source.py       | 30 +++++++-----
 5 files changed, 76 insertions(+), 48 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 08cc74aec3977..57aae32b361cf 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -1,10 +1,10 @@
 import logging
 from abc import abstractmethod
 from typing import Any, Dict, Optional
-from urllib.parse import quote_plus
 
 import pydantic
 from pydantic import Field
+from sqlalchemy.engine import URL
 
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
 from datahub.configuration.source_common import (
@@ -125,7 +125,11 @@ class SQLAlchemyConnectionConfig(ConfigModel):
     # Duplicate of SQLCommonConfig.options
     options: dict = pydantic.Field(
         default_factory=dict,
-        description="Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.",
+        description=(
+            "Any options specified here will be passed to "
+            "[SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs."
+            " To set connection arguments in the URL, specify them under `connect_args`."
+        ),
     )
 
     _database_alias_deprecation = pydantic_field_deprecated(
@@ -161,21 +165,26 @@ def make_sqlalchemy_uri(
     db: Optional[str],
     uri_opts: Optional[Dict[str, Any]] = None,
 ) -> str:
-    url = f"{scheme}://"
-    if username is not None:
-        url += f"{quote_plus(username)}"
-        if password is not None:
-            url += f":{quote_plus(password)}"
-        url += "@"
-    if at is not None:
-        url += f"{at}"
-    if db is not None:
-        url += f"/{db}"
-    if uri_opts is not None:
-        if db is None:
-            url += "/"
-        params = "&".join(
-            f"{key}={quote_plus(value)}" for (key, value) in uri_opts.items() if value
+    host: Optional[str] = None
+    port: Optional[int] = None
+    if at:
+        try:
+            host, port_str = at.rsplit(":", 1)
+            port = int(port_str)
+        except ValueError:
+            host = at
+            port = None
+    if uri_opts:
+        uri_opts = {k: v for k, v in uri_opts.items() if v is not None}
+
+    return str(
+        URL.create(
+            drivername=scheme,
+            username=username,
+            password=password,
+            host=host,
+            port=port,
+            database=db,
+            query=uri_opts or {},
         )
-        url = f"{url}?{params}"
-    return url
+    )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
index d9062cef06eae..7a49551dc1235 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
@@ -1,8 +1,10 @@
 import typing
+import urllib.parse
 from typing import Any, Dict, Iterable, Optional
 
 from pydantic.fields import Field
 from sqlalchemy import create_engine, inspect
+from sqlalchemy.engine import URL
 from sqlalchemy.engine.reflection import Inspector
 
 from datahub.configuration.common import AllowDenyPattern
@@ -41,14 +43,27 @@ def get_sql_alchemy_url(
         uri_opts: typing.Optional[typing.Dict[str, typing.Any]] = None,
         current_db: typing.Optional[str] = None,
     ) -> str:
-        return self.sqlalchemy_uri or make_sqlalchemy_uri(
-            self.scheme,
-            self.username,
-            self.password.get_secret_value() if self.password else None,
-            self.host_port,
-            current_db if current_db else self.database,
-            uri_opts=uri_opts,
-        )
+        if self.sqlalchemy_uri:
+            parsed_url = urllib.parse.urlsplit(self.sqlalchemy_uri)
+            url = URL.create(
+                drivername=parsed_url.scheme,
+                username=parsed_url.username,
+                password=parsed_url.password,
+                host=parsed_url.hostname,
+                port=parsed_url.port,
+                database=current_db or parsed_url.path.lstrip("/"),
+                query=urllib.parse.parse_qs(parsed_url.query),
+            ).update_query_dict(uri_opts or {})
+            return str(url)
+        else:
+            return make_sqlalchemy_uri(
+                self.scheme,
+                self.username,
+                self.password.get_secret_value() if self.password else None,
+                self.host_port,
+                current_db or self.database,
+                uri_opts=uri_opts,
+            )
 
 
 class TwoTierSQLAlchemySource(SQLAlchemySource):
diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py
index 2558f6a46715e..7a947e8f86bfe 100644
--- a/metadata-ingestion/tests/unit/test_athena_source.py
+++ b/metadata-ingestion/tests/unit/test_athena_source.py
@@ -10,7 +10,6 @@
 FROZEN_TIME = "2020-04-14 07:00:00"
 
 
-@pytest.mark.integration
 def test_athena_config_query_location_old_plus_new_value_not_allowed():
     from datahub.ingestion.source.sql.athena import AthenaConfig
 
@@ -25,7 +24,6 @@ def test_athena_config_query_location_old_plus_new_value_not_allowed():
         )
 
 
-@pytest.mark.integration
 def test_athena_config_staging_dir_is_set_as_query_result():
     from datahub.ingestion.source.sql.athena import AthenaConfig
 
@@ -48,7 +46,6 @@ def test_athena_config_staging_dir_is_set_as_query_result():
     assert config.json() == expected_config.json()
 
 
-@pytest.mark.integration
 def test_athena_uri():
     from datahub.ingestion.source.sql.athena import AthenaConfig
 
@@ -59,9 +56,12 @@ def test_athena_uri():
             "work_group": "test-workgroup",
         }
     )
-    assert (
-        config.get_sql_alchemy_url()
-        == "awsathena+rest://@athena.us-west-1.amazonaws.com:443/?s3_staging_dir=s3%3A%2F%2Fquery-result-location%2F&work_group=test-workgroup&catalog_name=awsdatacatalog&duration_seconds=3600"
+    assert config.get_sql_alchemy_url() == (
+        "awsathena+rest://@athena.us-west-1.amazonaws.com:443"
+        "?catalog_name=awsdatacatalog"
+        "&duration_seconds=3600"
+        "&s3_staging_dir=s3%3A%2F%2Fquery-result-location%2F"
+        "&work_group=test-workgroup"
     )
 
 
diff --git a/metadata-ingestion/tests/unit/test_clickhouse_source.py b/metadata-ingestion/tests/unit/test_clickhouse_source.py
index de7e7d66f2129..1b2ffb70c8d19 100644
--- a/metadata-ingestion/tests/unit/test_clickhouse_source.py
+++ b/metadata-ingestion/tests/unit/test_clickhouse_source.py
@@ -26,9 +26,7 @@ def test_clickhouse_uri_native():
             "scheme": "clickhouse+native",
         }
     )
-    assert (
-        config.get_sql_alchemy_url() == "clickhouse+native://user:password@host:1111/"
-    )
+    assert config.get_sql_alchemy_url() == "clickhouse+native://user:password@host:1111"
 
 
 def test_clickhouse_uri_native_secure():
diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py
index 1c26ca2487e5c..888a7c0441554 100644
--- a/metadata-ingestion/tests/unit/test_snowflake_source.py
+++ b/metadata-ingestion/tests/unit/test_snowflake_source.py
@@ -179,10 +179,12 @@ def test_snowflake_uri_default_authentication():
         }
     )
 
-    assert (
-        config.get_sql_alchemy_url()
-        == "snowflake://user:password@acctname/?authenticator=SNOWFLAKE&warehouse=COMPUTE_WH&role"
-        "=sysadmin&application=acryl_datahub"
+    assert config.get_sql_alchemy_url() == (
+        "snowflake://user:password@acctname"
+        "?application=acryl_datahub"
+        "&authenticator=SNOWFLAKE"
+        "&role=sysadmin"
+        "&warehouse=COMPUTE_WH"
     )
 
 
@@ -198,10 +200,12 @@ def test_snowflake_uri_external_browser_authentication():
         }
     )
 
-    assert (
-        config.get_sql_alchemy_url()
-        == "snowflake://user@acctname/?authenticator=EXTERNALBROWSER&warehouse=COMPUTE_WH&role"
-        "=sysadmin&application=acryl_datahub"
+    assert config.get_sql_alchemy_url() == (
+        "snowflake://user@acctname"
+        "?application=acryl_datahub"
+        "&authenticator=EXTERNALBROWSER"
+        "&role=sysadmin"
+        "&warehouse=COMPUTE_WH"
     )
 
 
@@ -219,10 +223,12 @@ def test_snowflake_uri_key_pair_authentication():
         }
     )
 
-    assert (
-        config.get_sql_alchemy_url()
-        == "snowflake://user@acctname/?authenticator=SNOWFLAKE_JWT&warehouse=COMPUTE_WH&role"
-        "=sysadmin&application=acryl_datahub"
+    assert config.get_sql_alchemy_url() == (
+        "snowflake://user@acctname"
+        "?application=acryl_datahub"
+        "&authenticator=SNOWFLAKE_JWT"
+        "&role=sysadmin"
+        "&warehouse=COMPUTE_WH"
     )
 
 

From 1eaf9c8c5ff2676a5c4ac456d7b4a6d351697e73 Mon Sep 17 00:00:00 2001
From: Tim <50115603+bossenti@users.noreply.github.com>
Date: Wed, 18 Oct 2023 18:39:59 +0200
Subject: [PATCH 064/259] feature(ingest/athena): introduce support for complex
 and nested schemas in Athena (#8137)

Co-authored-by: dnks23 <dominik.s23@live.de>
Co-authored-by: Tamas Nemeth <treff7es@gmail.com>
Co-authored-by: Tim <tim@MBP-von-Tim.fritz.box>
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 metadata-ingestion/setup.py                   |   5 +-
 .../datahub/ingestion/source/sql/athena.py    | 200 +++++++++++++++++-
 .../ingestion/source/sql/sql_common.py        |   4 +
 .../datahub/ingestion/source/sql/sql_types.py |  12 +-
 .../utilities/sqlalchemy_type_converter.py    | 200 ++++++++++++++++++
 .../tests/unit/test_athena_source.py          |  86 +++++++-
 .../test_sqlalchemy_type_converter.py         |  93 ++++++++
 7 files changed, 589 insertions(+), 11 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
 create mode 100644 metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 0b0a2b13fb52a..c46409ecbf52f 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -280,8 +280,9 @@
     # Misc plugins.
     "sql-parser": sqlglot_lib,
     # Source plugins
-    # PyAthena is pinned with exact version because we use private method in PyAthena
-    "athena": sql_common | {"PyAthena[SQLAlchemy]==2.4.1"},
+    # sqlalchemy-bigquery is included here since it provides an implementation of
+    # a SQLalchemy-conform STRUCT type definition
+    "athena": sql_common | {"PyAthena[SQLAlchemy]>=2.6.0,<3.0.0", "sqlalchemy-bigquery>=1.4.1"},
     "azure-ad": set(),
     "bigquery": sql_common
     | bigquery_common
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
index 9cb613bde1e9f..dad61e5173166 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
@@ -1,12 +1,17 @@
 import json
 import logging
+import re
 import typing
-from typing import Any, Dict, Iterable, List, Optional, Tuple, cast
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, cast
 
 import pydantic
 from pyathena.common import BaseCursor
 from pyathena.model import AthenaTableMetadata
+from pyathena.sqlalchemy_athena import AthenaRestDialect
+from sqlalchemy import create_engine, inspect, types
 from sqlalchemy.engine.reflection import Inspector
+from sqlalchemy.types import TypeEngine
+from sqlalchemy_bigquery import STRUCT
 
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.emitter.mcp_builder import ContainerKey, DatabaseKey
@@ -21,13 +26,164 @@
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.aws.s3_util import make_s3_urn
 from datahub.ingestion.source.common.subtypes import DatasetContainerSubTypes
-from datahub.ingestion.source.sql.sql_common import SQLAlchemySource
+from datahub.ingestion.source.sql.sql_common import (
+    SQLAlchemySource,
+    register_custom_type,
+)
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, make_sqlalchemy_uri
+from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     gen_database_container,
     gen_database_key,
 )
+from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
+from datahub.metadata.schema_classes import RecordTypeClass
+from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
+from datahub.utilities.sqlalchemy_type_converter import (
+    get_schema_fields_for_sqlalchemy_column,
+)
+
+logger = logging.getLogger(__name__)
+
+register_custom_type(STRUCT, RecordTypeClass)
+
+
+class CustomAthenaRestDialect(AthenaRestDialect):
+    """Custom definition of the Athena dialect.
+
+    Custom implementation that allows to extend/modify the behavior of the SQLalchemy
+    dialect that is used by PyAthena (which is the library that is used by DataHub
+    to extract metadata from Athena).
+    This dialect can then be used by the inspector (see get_inspectors()).
+
+    """
+
+    # regex to identify complex types in DDL strings which are embedded in `<>`.
+    _complex_type_pattern = re.compile(r"(<.+>)")
+
+    @typing.no_type_check
+    def _get_column_type(
+        self, type_: Union[str, Dict[str, Any]]
+    ) -> TypeEngine:  # noqa: C901
+        """Derives the data type of the Athena column.
+
+        This method is overwritten to extend the behavior of PyAthena.
+        Pyathena is not capable of detecting complex data types, e.g.,
+        arrays, maps, or, structs (as of version 2.25.2).
+        The custom implementation extends the functionality by the above-mentioned data types.
+        """
+
+        # Originally, this method only handles `type_` as a string
+        # With the workaround used below to parse DDL strings for structs,
+        # `type` might also be a dictionary
+        if isinstance(type_, str):
+            match = self._pattern_column_type.match(type_)
+            if match:
+                type_name = match.group(1).lower()
+                type_meta_information = match.group(2)
+            else:
+                type_name = type_.lower()
+                type_meta_information = None
+        elif isinstance(type_, dict):
+            # this occurs only when a type parsed as part of a STRUCT is passed
+            # in such case type_ is a dictionary whose type can be retrieved from the attribute
+            type_name = type_.get("type", None)
+            type_meta_information = None
+        else:
+            raise RuntimeError(f"Unsupported type definition: {type_}")
+
+        args = []
+
+        if type_name in ["array"]:
+            detected_col_type = types.ARRAY
+
+            # here we need to account again for two options how `type_` is passed to this method
+            # first, the simple array definition as a DDL string (something like array<string>)
+            # this is always the case when the array is not part of a complex data type (mainly STRUCT)
+            # second, the array definition can also be passed in form of dictionary
+            # this is the case when the array is part of a complex data type
+            if isinstance(type_, str):
+                # retrieve the raw name of the data type as a string
+                array_type_raw = self._complex_type_pattern.findall(type_)[0][
+                    1:-1
+                ]  # array type without enclosing <>
+                # convert the string name of the data type into a SQLalchemy type (expected return)
+                array_type = self._get_column_type(array_type_raw)
+            elif isinstance(type_, dict):
+                # retrieve the data type of the array items and
+                # transform it into a SQLalchemy type
+                array_type = self._get_column_type(type_["items"])
+            else:
+                raise RuntimeError(f"Unsupported array definition: {type_}")
+
+            args = [array_type]
+
+        elif type_name in ["struct", "record"]:
+            # STRUCT is not part of the SQLalchemy types selection
+            # but is provided by another official SQLalchemy library and
+            # compatible with the other SQLalchemy types
+            detected_col_type = STRUCT
+
+            if isinstance(type_, dict):
+                # in case a struct as part of another struct is passed
+                # it is provided in form of a dictionary and
+                # can simply be used for the further processing
+                struct_type = type_
+            else:
+                # this is the case when the type definition of the struct is passed as a DDL string
+                # therefore, it is required to parse the DDL string
+                # here a method provided in another Datahub source is used so that the parsing
+                # doesn't need to be implemented twice
+                # `get_avro_schema_for_hive_column` accepts a DDL description as column type and
+                # returns the parsed data types in form of a dictionary
+                schema = get_avro_schema_for_hive_column(
+                    hive_column_name=type_name, hive_column_type=type_
+                )
+
+                # the actual type description needs to be extracted
+                struct_type = schema["fields"][0]["type"]
+
+            # A STRUCT consist of multiple attributes which are expected to be passed as
+            # a list of tuples consisting of name data type pairs. e.g., `('age', Integer())`
+            # See the reference:
+            # https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/sqlalchemy_bigquery/_struct.py#L53
+            #
+            # To extract all of them, we simply iterate over all detected fields and
+            # convert them to SQLalchemy types
+            struct_args = []
+            for field in struct_type["fields"]:
+                struct_args.append(
+                    (
+                        field["name"],
+                        self._get_column_type(field["type"]["type"])
+                        if field["type"]["type"] not in ["record", "array"]
+                        else self._get_column_type(field["type"]),
+                    )
+                )
+
+            args = struct_args
+
+        elif type_name in ["map"]:
+            # Instead of SQLalchemy's TupleType the custom MapType is used here
+            # which is just a simple wrapper around TupleType
+            detected_col_type = MapType
+
+            # the type definition for maps looks like the following: key_type:val_type (e.g., string:string)
+            key_type_raw, value_type_raw = type_meta_information.split(",")
+
+            # convert both type names to actual SQLalchemy types
+            args = [
+                self._get_column_type(key_type_raw),
+                self._get_column_type(value_type_raw),
+            ]
+        # by using get_avro_schema_for_hive_column() for parsing STRUCTs the data type `long`
+        # can also be returned, so we need to extend the handling here as well
+        elif type_name in ["bigint", "long"]:
+            detected_col_type = types.BIGINT
+        else:
+            return super()._get_column_type(type_name)
+        return detected_col_type(*args)
 
 
 class AthenaConfig(SQLCommonConfig):
@@ -129,6 +285,18 @@ def create(cls, config_dict, ctx):
         config = AthenaConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
+    # overwrite this method to allow to specify the usage of a custom dialect
+    def get_inspectors(self) -> Iterable[Inspector]:
+        url = self.config.get_sql_alchemy_url()
+        logger.debug(f"sql_alchemy_url={url}")
+        engine = create_engine(url, **self.config.options)
+
+        # set custom dialect to be used by the inspector
+        engine.dialect = CustomAthenaRestDialect()
+        with engine.connect() as conn:
+            inspector = inspect(conn)
+            yield inspector
+
     def get_table_properties(
         self, inspector: Inspector, schema: str, table: str
     ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]:
@@ -136,9 +304,7 @@ def get_table_properties(
             self.cursor = cast(BaseCursor, inspector.engine.raw_connection().cursor())
             assert self.cursor
 
-        # Unfortunately properties can be only get through private methods as those are not exposed
-        # https://github.com/laughingman7743/PyAthena/blob/9e42752b0cc7145a87c3a743bb2634fe125adfa7/pyathena/model.py#L201
-        metadata: AthenaTableMetadata = self.cursor._get_table_metadata(
+        metadata: AthenaTableMetadata = self.cursor.get_table_metadata(
             table_name=table, schema_name=schema
         )
         description = metadata.comment
@@ -241,6 +407,30 @@ def get_schema_names(self, inspector: Inspector) -> List[str]:
             return [schema for schema in schemas if schema == athena_config.database]
         return schemas
 
+    # Overwrite to modify the creation of schema fields
+    def get_schema_fields_for_column(
+        self,
+        dataset_name: str,
+        column: Dict,
+        pk_constraints: Optional[dict] = None,
+        tags: Optional[List[str]] = None,
+    ) -> List[SchemaField]:
+        fields = get_schema_fields_for_sqlalchemy_column(
+            column_name=column["name"],
+            column_type=column["type"],
+            description=column.get("comment", None),
+            nullable=column.get("nullable", True),
+            is_part_of_key=True
+            if (
+                pk_constraints is not None
+                and isinstance(pk_constraints, dict)
+                and column["name"] in pk_constraints.get("constrained_columns", [])
+            )
+            else False,
+        )
+
+        return fields
+
     def close(self):
         if self.cursor:
             self.cursor.close()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index 056be6c2e50ac..6524eea8222d4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -37,6 +37,7 @@
     DatasetSubTypes,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
+from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     downgrade_schema_from_v2,
@@ -80,6 +81,7 @@
     DatasetLineageTypeClass,
     DatasetPropertiesClass,
     GlobalTagsClass,
+    MapTypeClass,
     SubTypesClass,
     TagAssociationClass,
     UpstreamClass,
@@ -154,6 +156,8 @@ class SqlWorkUnit(MetadataWorkUnit):
     types.DATETIME: TimeTypeClass,
     types.TIMESTAMP: TimeTypeClass,
     types.JSON: RecordTypeClass,
+    # additional type definitions that are used by the Athena source
+    MapType: MapTypeClass,  # type: ignore
     # Because the postgresql dialect is used internally by many other dialects,
     # we add some postgres types here. This is ok to do because the postgresql
     # dialect is built-in to sqlalchemy.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
index 3b4a7e1dc0287..51626891e9fef 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
@@ -1,13 +1,15 @@
 import re
 from typing import Any, Dict, ValuesView
 
+from sqlalchemy import types
+
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     ArrayType,
     BooleanType,
     BytesType,
     DateType,
     EnumType,
-    MapType,
+    MapType as MapTypeAvro,
     NullType,
     NumberType,
     RecordType,
@@ -363,10 +365,16 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "time": TimeType,
     "timestamp": TimeType,
     "row": RecordType,
-    "map": MapType,
+    "map": MapTypeAvro,
     "array": ArrayType,
 }
 
+
+class MapType(types.TupleType):
+    # Wrapper class around SQLalchemy's TupleType to increase compatibility with DataHub
+    pass
+
+
 # https://docs.aws.amazon.com/athena/latest/ug/data-types.html
 # https://github.com/dbt-athena/dbt-athena/tree/main
 ATHENA_SQL_TYPES_MAP: Dict[str, Any] = {
diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
new file mode 100644
index 0000000000000..a431f262a85fd
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
@@ -0,0 +1,200 @@
+import json
+import logging
+import uuid
+from typing import Any, Dict, List, Optional, Type, Union
+
+from sqlalchemy import types
+from sqlalchemy_bigquery import STRUCT
+
+from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields
+from datahub.ingestion.source.sql.sql_types import MapType
+from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
+from datahub.metadata.schema_classes import NullTypeClass, SchemaFieldDataTypeClass
+
+logger = logging.getLogger(__name__)
+
+
+class SqlAlchemyColumnToAvroConverter:
+    """Helper class that collects some methods to convert SQLalchemy columns to Avro schema."""
+
+    # tuple of complex data types that require a special handling
+    _COMPLEX_TYPES = (STRUCT, types.ARRAY, MapType)
+
+    # mapping of primitive SQLalchemy data types to AVRO schema data types
+    PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE: Dict[Type[types.TypeEngine], str] = {
+        types.String: "string",
+        types.BINARY: "string",
+        types.BOOLEAN: "boolean",
+        types.FLOAT: "float",
+        types.INTEGER: "int",
+        types.BIGINT: "long",
+        types.VARCHAR: "string",
+        types.CHAR: "string",
+    }
+
+    @classmethod
+    def get_avro_type(
+        cls, column_type: Union[types.TypeEngine, STRUCT, MapType], nullable: bool
+    ) -> Dict[str, Any]:
+        """Determines the concrete AVRO schema type for a SQLalchemy-typed column"""
+
+        if type(column_type) in cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE.keys():
+            return {
+                "type": cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE[type(column_type)],
+                "native_data_type": str(column_type),
+                "_nullable": nullable,
+            }
+        if isinstance(column_type, types.DECIMAL):
+            return {
+                "type": "bytes",
+                "logicalType": "decimal",
+                "precision": int(column_type.precision),
+                "scale": int(column_type.scale),
+                "native_data_type": str(column_type),
+                "_nullable": nullable,
+            }
+        if isinstance(column_type, types.DATE):
+            return {
+                "type": "int",
+                "logicalType": "date",
+                "native_data_type": str(column_type),
+                "_nullable": nullable,
+            }
+        if isinstance(column_type, types.TIMESTAMP):
+            return {
+                "type": "long",
+                "logicalType": "timestamp-millis",
+                "native_data_type": str(column_type),
+                "_nullable": nullable,
+            }
+        if isinstance(column_type, types.ARRAY):
+            array_type = column_type.item_type
+            return {
+                "type": "array",
+                "items": cls.get_avro_type(column_type=array_type, nullable=nullable),
+                "native_data_type": f"array<{str(column_type.item_type)}>",
+            }
+        if isinstance(column_type, MapType):
+            key_type = column_type.types[0]
+            value_type = column_type.types[1]
+            return {
+                "type": "map",
+                "values": cls.get_avro_type(column_type=value_type, nullable=nullable),
+                "native_data_type": str(column_type),
+                "key_type": cls.get_avro_type(column_type=key_type, nullable=nullable),
+                "key_native_data_type": str(key_type),
+            }
+        if isinstance(column_type, STRUCT):
+            fields = []
+            for field_def in column_type._STRUCT_fields:
+                field_name, field_type = field_def
+                fields.append(
+                    {
+                        "name": field_name,
+                        "type": cls.get_avro_type(
+                            column_type=field_type, nullable=nullable
+                        ),
+                    }
+                )
+            struct_name = f"__struct_{str(uuid.uuid4()).replace('-', '')}"
+
+            return {
+                "type": "record",
+                "name": struct_name,
+                "fields": fields,
+                "native_data_type": str(column_type),
+                "_nullable": nullable,
+            }
+
+        return {
+            "type": "null",
+            "native_data_type": str(column_type),
+            "_nullable": nullable,
+        }
+
+    @classmethod
+    def get_avro_for_sqlalchemy_column(
+        cls,
+        column_name: str,
+        column_type: types.TypeEngine,
+        nullable: bool,
+    ) -> Union[object, Dict[str, object]]:
+        """Returns the AVRO schema representation of a SQLalchemy column."""
+        if isinstance(column_type, cls._COMPLEX_TYPES):
+            return {
+                "type": "record",
+                "name": "__struct_",
+                "fields": [
+                    {
+                        "name": column_name,
+                        "type": cls.get_avro_type(
+                            column_type=column_type, nullable=nullable
+                        ),
+                    }
+                ],
+            }
+        return cls.get_avro_type(column_type=column_type, nullable=nullable)
+
+
+def get_schema_fields_for_sqlalchemy_column(
+    column_name: str,
+    column_type: types.TypeEngine,
+    description: Optional[str] = None,
+    nullable: Optional[bool] = True,
+    is_part_of_key: Optional[bool] = False,
+) -> List[SchemaField]:
+    """Creates SchemaFields from a given SQLalchemy column.
+
+    This function is analogous to `get_schema_fields_for_hive_column` from datahub.utilities.hive_schema_to_avro.
+    The main purpose of implementing it this way, is to make it ready/compatible for second field path generation,
+    which allows to explore nested structures within the UI.
+    """
+
+    if nullable is None:
+        nullable = True
+
+    try:
+        # as a first step, the column is converted to AVRO JSON which can then be used by an existing function
+        avro_schema_json = (
+            SqlAlchemyColumnToAvroConverter.get_avro_for_sqlalchemy_column(
+                column_name=column_name,
+                column_type=column_type,
+                nullable=nullable,
+            )
+        )
+        # retrieve schema field definitions from the above generated AVRO JSON structure
+        schema_fields = avro_schema_to_mce_fields(
+            avro_schema=json.dumps(avro_schema_json),
+            default_nullable=nullable,
+            swallow_exceptions=False,
+        )
+    except Exception as e:
+        logger.warning(
+            f"Unable to parse column {column_name} and type {column_type} the error was: {e}"
+        )
+
+        # fallback description in case any exception occurred
+        schema_fields = [
+            SchemaField(
+                fieldPath=column_name,
+                type=SchemaFieldDataTypeClass(type=NullTypeClass()),
+                nativeDataType=str(column_type),
+            )
+        ]
+
+    # for all non-nested data types an additional modification of the `fieldPath` property is required
+    if type(column_type) in (
+        *SqlAlchemyColumnToAvroConverter.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE.keys(),
+        types.TIMESTAMP,
+        types.DATE,
+        types.DECIMAL,
+    ):
+        schema_fields[0].fieldPath += f".{column_name}"
+
+    if description:
+        schema_fields[0].description = description
+    schema_fields[0].isPartOfKey = (
+        is_part_of_key if is_part_of_key is not None else False
+    )
+
+    return schema_fields
diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py
index 7a947e8f86bfe..6d3ed20eafde2 100644
--- a/metadata-ingestion/tests/unit/test_athena_source.py
+++ b/metadata-ingestion/tests/unit/test_athena_source.py
@@ -3,9 +3,13 @@
 
 import pytest
 from freezegun import freeze_time
+from sqlalchemy import types
+from sqlalchemy_bigquery import STRUCT
 
 from datahub.ingestion.api.common import PipelineContext
-from src.datahub.ingestion.source.aws.s3_util import make_s3_urn
+from datahub.ingestion.source.aws.s3_util import make_s3_urn
+from datahub.ingestion.source.sql.athena import CustomAthenaRestDialect
+from datahub.ingestion.source.sql.sql_types import MapType
 
 FROZEN_TIME = "2020-04-14 07:00:00"
 
@@ -104,7 +108,7 @@ def test_athena_get_table_properties():
     mock_cursor = mock.MagicMock()
     mock_inspector = mock.MagicMock()
     mock_inspector.engine.raw_connection().cursor.return_value = mock_cursor
-    mock_cursor._get_table_metadata.return_value = AthenaTableMetadata(
+    mock_cursor.get_table_metadata.return_value = AthenaTableMetadata(
         response=table_metadata
     )
 
@@ -126,3 +130,81 @@ def test_athena_get_table_properties():
     }
 
     assert location == make_s3_urn("s3://testLocation", "PROD")
+
+
+def test_get_column_type_simple_types():
+    assert isinstance(
+        CustomAthenaRestDialect()._get_column_type(type_="int"), types.Integer
+    )
+    assert isinstance(
+        CustomAthenaRestDialect()._get_column_type(type_="string"), types.String
+    )
+    assert isinstance(
+        CustomAthenaRestDialect()._get_column_type(type_="boolean"), types.BOOLEAN
+    )
+    assert isinstance(
+        CustomAthenaRestDialect()._get_column_type(type_="long"), types.BIGINT
+    )
+    assert isinstance(
+        CustomAthenaRestDialect()._get_column_type(type_="double"), types.FLOAT
+    )
+
+
+def test_get_column_type_array():
+    result = CustomAthenaRestDialect()._get_column_type(type_="array<string>")
+
+    assert isinstance(result, types.ARRAY)
+    assert isinstance(result.item_type, types.String)
+
+
+def test_get_column_type_map():
+    result = CustomAthenaRestDialect()._get_column_type(type_="map<string,int>")
+
+    assert isinstance(result, MapType)
+    assert isinstance(result.types[0], types.String)
+    assert isinstance(result.types[1], types.Integer)
+
+
+def test_column_type_struct():
+
+    result = CustomAthenaRestDialect()._get_column_type(type_="struct<test:string>")
+
+    assert isinstance(result, STRUCT)
+    assert isinstance(result._STRUCT_fields[0], tuple)
+    assert result._STRUCT_fields[0][0] == "test"
+    assert isinstance(result._STRUCT_fields[0][1], types.String)
+
+
+def test_column_type_complex_combination():
+
+    result = CustomAthenaRestDialect()._get_column_type(
+        type_="struct<id:string,name:string,choices:array<struct<id:string,label:string>>>"
+    )
+
+    assert isinstance(result, STRUCT)
+
+    assert isinstance(result._STRUCT_fields[0], tuple)
+    assert result._STRUCT_fields[0][0] == "id"
+    assert isinstance(result._STRUCT_fields[0][1], types.String)
+
+    assert isinstance(result._STRUCT_fields[1], tuple)
+    assert result._STRUCT_fields[1][0] == "name"
+    assert isinstance(result._STRUCT_fields[1][1], types.String)
+
+    assert isinstance(result._STRUCT_fields[2], tuple)
+    assert result._STRUCT_fields[2][0] == "choices"
+    assert isinstance(result._STRUCT_fields[2][1], types.ARRAY)
+
+    assert isinstance(result._STRUCT_fields[2][1].item_type, STRUCT)
+
+    assert isinstance(result._STRUCT_fields[2][1].item_type._STRUCT_fields[0], tuple)
+    assert result._STRUCT_fields[2][1].item_type._STRUCT_fields[0][0] == "id"
+    assert isinstance(
+        result._STRUCT_fields[2][1].item_type._STRUCT_fields[0][1], types.String
+    )
+
+    assert isinstance(result._STRUCT_fields[2][1].item_type._STRUCT_fields[1], tuple)
+    assert result._STRUCT_fields[2][1].item_type._STRUCT_fields[1][0] == "label"
+    assert isinstance(
+        result._STRUCT_fields[2][1].item_type._STRUCT_fields[1][1], types.String
+    )
diff --git a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
new file mode 100644
index 0000000000000..959da0987a825
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
@@ -0,0 +1,93 @@
+from typing import no_type_check
+
+from sqlalchemy import types
+from sqlalchemy_bigquery import STRUCT
+
+from datahub.ingestion.source.sql.sql_types import MapType
+from datahub.metadata.schema_classes import (
+    ArrayTypeClass,
+    MapTypeClass,
+    NullTypeClass,
+    NumberTypeClass,
+    RecordTypeClass,
+)
+from datahub.utilities.sqlalchemy_type_converter import (
+    get_schema_fields_for_sqlalchemy_column,
+)
+
+
+def test_get_avro_schema_for_sqlalchemy_column():
+    schema_fields = get_schema_fields_for_sqlalchemy_column(
+        column_name="test", column_type=types.INTEGER()
+    )
+    assert len(schema_fields) == 1
+    assert schema_fields[0].fieldPath == "[version=2.0].[type=int].test"
+    assert schema_fields[0].type.type == NumberTypeClass()
+    assert schema_fields[0].nativeDataType == "INTEGER"
+    assert schema_fields[0].nullable is True
+
+    schema_fields = get_schema_fields_for_sqlalchemy_column(
+        column_name="test", column_type=types.String(), nullable=False
+    )
+    assert len(schema_fields) == 1
+    assert schema_fields[0].fieldPath == "[version=2.0].[type=string].test"
+    assert schema_fields[0].type.type == NumberTypeClass()
+    assert schema_fields[0].nativeDataType == "VARCHAR"
+    assert schema_fields[0].nullable is False
+
+
+def test_get_avro_schema_for_sqlalchemy_array_column():
+    schema_fields = get_schema_fields_for_sqlalchemy_column(
+        column_name="test", column_type=types.ARRAY(types.FLOAT())
+    )
+    assert len(schema_fields) == 1
+    assert (
+        schema_fields[0].fieldPath
+        == "[version=2.0].[type=struct].[type=array].[type=float].test"
+    )
+    assert schema_fields[0].type.type == ArrayTypeClass(nestedType=["float"])
+    assert schema_fields[0].nativeDataType == "array<FLOAT>"
+
+
+def test_get_avro_schema_for_sqlalchemy_map_column():
+    schema_fields = get_schema_fields_for_sqlalchemy_column(
+        column_name="test", column_type=MapType(types.String(), types.BOOLEAN())
+    )
+    assert len(schema_fields) == 1
+    assert (
+        schema_fields[0].fieldPath
+        == "[version=2.0].[type=struct].[type=map].[type=boolean].test"
+    )
+    assert schema_fields[0].type.type == MapTypeClass(
+        keyType="string", valueType="boolean"
+    )
+    assert schema_fields[0].nativeDataType == "MapType(String(), BOOLEAN())"
+
+
+def test_get_avro_schema_for_sqlalchemy_struct_column() -> None:
+
+    schema_fields = get_schema_fields_for_sqlalchemy_column(
+        column_name="test", column_type=STRUCT(("test", types.INTEGER()))
+    )
+    assert len(schema_fields) == 2
+    assert (
+        schema_fields[0].fieldPath == "[version=2.0].[type=struct].[type=struct].test"
+    )
+    assert schema_fields[0].type.type == RecordTypeClass()
+    assert schema_fields[0].nativeDataType == "STRUCT<test INT64>"
+
+    assert (
+        schema_fields[1].fieldPath
+        == "[version=2.0].[type=struct].[type=struct].test.[type=int].test"
+    )
+    assert schema_fields[1].type.type == NumberTypeClass()
+    assert schema_fields[1].nativeDataType == "INTEGER"
+
+
+@no_type_check
+def test_get_avro_schema_for_sqlalchemy_unknown_column():
+    schema_fields = get_schema_fields_for_sqlalchemy_column("invalid", "test")
+    assert len(schema_fields) == 1
+    assert schema_fields[0].type.type == NullTypeClass()
+    assert schema_fields[0].fieldPath == "[version=2.0].[type=null]"
+    assert schema_fields[0].nativeDataType == "test"

From bd5c4e0d70681b4640d25e62326e05be1c9deb65 Mon Sep 17 00:00:00 2001
From: Saketh-Mahesh <81051119+Saketh-Mahesh@users.noreply.github.com>
Date: Wed, 18 Oct 2023 11:48:39 -0500
Subject: [PATCH 065/259] docs: adding documentation for deployment of DataHub
 on Azure (#8612)

Co-authored-by: Saketh Mahesh <sakethmahesh@Sakeths-MacBook-Air.local>
---
 docs-website/sidebars.js |   1 +
 docs/deploy/azure.md     | 234 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 235 insertions(+)
 create mode 100644 docs/deploy/azure.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 4fa73c995157a..b2b3df4dfb33c 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -158,6 +158,7 @@ module.exports = {
         // The purpose of this section is to provide the minimum steps required to deploy DataHub to the vendor of your choosing
         "docs/deploy/aws",
         "docs/deploy/gcp",
+        "docs/deploy/azure",
         "docker/README",
         "docs/deploy/kubernetes",
         "docs/deploy/environment-vars",
diff --git a/docs/deploy/azure.md b/docs/deploy/azure.md
new file mode 100644
index 0000000000000..b940b82827e94
--- /dev/null
+++ b/docs/deploy/azure.md
@@ -0,0 +1,234 @@
+---
+title: "Deploying to Azure"
+---
+
+# Azure setup guide
+
+The following is a set of instructions to quickstart DataHub on Azure Kubernetes Service (AKS). Note, the guide
+assumes that you do not have a Kubernetes cluster set up. 
+
+## Prerequisites
+
+This guide requires the following tools:
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/) to manage Kubernetes resources
+- [helm](https://helm.sh/docs/intro/install/) to deploy the resources based on helm charts. Note, we only support Helm
+    3.
+- [AZ CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) to manage Azure resources
+
+To use the above tools, you need to set up Azure credentials by following
+this [guide](https://learn.microsoft.com/en-us/cli/azure/authenticate-azure-cli).
+
+## Start up a Kubernetes cluster on AKS
+
+You can follow this [guide](https://learn.microsoft.com/en-us/azure/aks/learn/quick-kubernetes-deploy-cli) to create a new
+cluster using az cli. 
+
+Note: you can skip the application deployment step since we are deploying DataHub instead. If you are deploying DataHub to an existing cluster, please
+skip the corresponding sections.
+
+- Verify you have the Microsoft.OperationsManagement and Microsoft.OperationalInsights providers registered on your subscription. These Azure resource providers are required to support Container insights. Check the registration status using the following commands:
+
+```
+az provider show -n Microsoft.OperationsManagement -o table
+az provider show -n Microsoft.OperationalInsights -o table
+```
+
+If they're not registered, register them using the following commands:
+
+```
+az provider register --namespace Microsoft.OperationsManagement
+az provider register --namespace Microsoft.OperationalInsights
+```
+
+- Create a resource group. Change name, location to your choosing.
+
+```
+az group create --name myResourceGroup --location eastus
+```
+
+The following output indicates that the command execution was successful:
+
+```
+{
+  "id": "/subscriptions/<guid>/resourceGroups/myResourceGroup",
+  "location": "eastus",
+  "managedBy": null,
+  "name": "myResourceGroup",
+  "properties": {
+    "provisioningState": "Succeeded"
+  },
+  "tags": null
+}
+```
+- Create an AKS Cluster. For this project, it is best to increase node count to at least 3. Change cluster name, node count, and addons to your choosing.
+
+```
+az aks create -g myResourceGroup -n myAKSCluster --enable-managed-identity --node-count 3 --enable-addons monitoring --generate-ssh-keys
+```
+
+After a few minutes, the command completes and returns JSON-formatted information about the cluster.
+
+- Connect to the cluster
+
+Configure kubectl to connect to your Kubernetes cluster using the az aks get-credentials command.
+
+```
+az aks get-credentials --resource-group myResourceGroup --name myAKSCluster
+```
+
+Verify the connection to your cluster using the `kubectl get` command. This command returns a list of the cluster nodes.
+
+```
+kubectl get nodes
+```
+
+You should get results like below. Make sure node status is Ready.
+
+```
+NAME                                          STATUS   ROLES    AGE   VERSION
+aks-nodepool1-37660971-vmss000000              Ready    agent   24h   v1.25.6
+aks-nodepool1-37660971-vmss000001              Ready    agent   24h   v1.25.6
+aks-nodepool1-37660971-vmss000002              Ready    agent   24h   v1.25.6
+```
+
+## Setup DataHub using Helm
+
+Once the Kubernetes cluster has been set up, you can deploy DataHub and its prerequisites using helm. Please follow the
+steps in this [guide](kubernetes.md). 
+
+
+Notes:
+Since we are using PostgreSQL as the storage layer, change postgresql enabled to true and mysql to false in the values.yaml file of prerequisites.
+Additionally, create a postgresql secret. Make sure to include 3 passwords for the postgresql secret: postgres-password, replication-password, and password.
+
+## Expose endpoints using a load balancer
+
+Now that all the pods are up and running, you need to expose the datahub-frontend end point by setting
+up [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). To do this, you need to first set up an
+ingress controller. 
+
+
+There are many [ingress controllers](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/)  to choose
+from, but here, we will follow this [guide](https://learn.microsoft.com/en-us/azure/application-gateway/tutorial-ingress-controller-add-on-existing) to set up the Azure
+Application Gateway Ingress Controller. 
+
+- Deploy a New Application Gateway.
+
+First, you need to create a WAF policy
+
+```
+az network application-gateway waf-policy create -g myResourceGroup -n myWAFPolicy
+```
+
+- Before the application gateway can be deployed, you'll also need to create a public IP resource, a new virtual network with address space 10.0.0.0/16, and a subnet with address space 10.0.0.0/24. 
+Then, you can deploy your application gateway in the subnet using the publicIP.
+
+Caution: When you use an AKS cluster and application gateway in separate virtual networks, the address spaces of the two virtual networks must not overlap. The default address space that an AKS cluster deploys in is 10.224.0.0/12.
+
+
+```
+az network public-ip create -n myPublicIp -g myResourceGroup --allocation-method Static --sku Standard
+az network vnet create -n myVnet -g myResourceGroup --address-prefix 10.0.0.0/16 --subnet-name mySubnet --subnet-prefix 10.0.0.0/24 
+az network application-gateway create -n myApplicationGateway -l eastus -g myResourceGroup --sku WAF_v2 --public-ip-address myPublicIp --vnet-name myVnet --subnet mySubnet --priority 100 --waf-policy /subscriptions/{subscription_id}/resourceGroups/myResourceGroup/providers/Microsoft.Network/ApplicationGatewayWebApplicationFirewallPolicies/myWAFPolicy
+```
+Change myPublicIp, myResourceGroup, myVnet, mySubnet, and myApplicationGateway to names of your choosing.
+
+
+- Enable the AGIC Add-On in Existing AKS Cluster Through Azure CLI
+
+```
+appgwId=$(az network application-gateway show -n myApplicationGateway -g myResourceGroup -o tsv --query "id") 
+az aks enable-addons -n myCluster -g myResourceGroup -a ingress-appgw --appgw-id $appgwId
+```
+
+- Peer the Two Virtual Networks Together
+
+Since you deployed the AKS cluster in its own virtual network and the Application gateway in another virtual network, you'll need to peer the two virtual networks together in order for traffic to flow from the Application gateway to the pods in the cluster.
+
+```
+nodeResourceGroup=$(az aks show -n myCluster -g myResourceGroup -o tsv --query "nodeResourceGroup")
+aksVnetName=$(az network vnet list -g $nodeResourceGroup -o tsv --query "[0].name")
+
+aksVnetId=$(az network vnet show -n $aksVnetName -g $nodeResourceGroup -o tsv --query "id")
+az network vnet peering create -n AppGWtoAKSVnetPeering -g myResourceGroup --vnet-name myVnet --remote-vnet $aksVnetId --allow-vnet-access
+
+appGWVnetId=$(az network vnet show -n myVnet -g myResourceGroup -o tsv --query "id")
+az network vnet peering create -n AKStoAppGWVnetPeering -g $nodeResourceGroup --vnet-name $aksVnetName --remote-vnet $appGWVnetId --allow-vnet-access
+```
+
+- Deploy the Ingress on the Frontend Pod
+
+In order to use the ingress controller to expose frontend pod, we need to update the datahub-frontend section of the values.yaml file that was used to deploy DataHub. Here is a sample configuration:
+
+```
+datahub-frontend:
+  enabled: true
+  image:
+    repository: linkedin/datahub-frontend-react
+    # tag: "v0.10.0 # defaults to .global.datahub.version
+
+  # Set up ingress to expose react front-end
+  ingress:
+    enabled: true
+    annotations:
+      kubernetes.io/ingress.class: azure/application-gateway
+      appgw.ingress.kubernetes.io/backend-protocol: "http" 
+    
+    hosts:
+    - paths:
+      - /*
+  defaultUserCredentials: {}
+```
+
+You can then apply the updates:
+
+```
+helm upgrade --install datahub datahub/datahub --values values.yaml
+```
+
+You can now verify that the ingress was created correctly
+
+```
+kubectl get ingress
+```
+
+You should see a result like this:
+
+![frontend-image](https://github.com/Saketh-Mahesh/azure-docs-images/blob/main/frontend-status.png?raw=true)
+
+## Use PostgresSQL for the storage layer
+Configure a PostgreSQL database in the same virtual network as the Kubernetes cluster or implement virtual network peering to connect both networks. Once the database is provisioned, you should be able to see the following page under the Connect tab on the left side. 
+
+
+Note: PostgreSQL Database MUST be deployed in same location as AKS/resource group (eastus, centralus, etc.)
+Take a note of the connection details:
+
+![postgres-info](https://github.com/Saketh-Mahesh/azure-docs-images/blob/main/postgres-info.png?raw=true)
+
+
+
+
+
+- Update the postgresql settings under global in the values.yaml as follows.
+
+```
+global:
+  sql:
+    datasource:
+      host: "${POSTGRES_HOST}.postgres.database.azure.com:5432"
+      hostForpostgresqlClient: "${POSTGRES_HOST}.postgres.database.azure.com"
+      port: "5432"
+      url: "jdbc:postgresql://${POSTGRES_HOST}.postgres.database.azure.com:5432/datahub?user=${POSTGRES_ADMIN_LOGIN}&password=${POSTGRES_ADMIN_PASSWORD}&sslmode=require"
+      driver: "org.postgresql.Driver"
+      username: "${POSTGRES_ADMIN_LOGIN}"
+      password:
+        value: "${POSTGRES_ADMIN_PASSWORD}"
+```
+Run this command helm command to update datahub configuration
+
+```
+helm upgrade --install datahub datahub/datahub --values values.yaml
+```
+
+And there you go! You have now installed DataHub on an Azure Kubernetes Cluster with an ingress controller set up to expose the frontend. Additionally you have utilized PostgreSQL as the storage layer of DataHub.
\ No newline at end of file

From b3ac42b1e4f43ae2ddcd9e884dc182d3a963f99a Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 18 Oct 2023 13:42:03 -0400
Subject: [PATCH 066/259] feat(frontend/ingestion): Support flagged / warning /
 connection failure statuses; add recipe (#8920)

---
 .../ExecutionRequestDetailsModal.tsx          | 45 +++++++++++++++++--
 .../src/app/ingest/source/utils.ts            | 40 +++++++++++------
 .../src/graphql/ingestion.graphql             |  4 ++
 3 files changed, 73 insertions(+), 16 deletions(-)

diff --git a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
index 849efabdcde97..00fdc89964f88 100644
--- a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
@@ -2,6 +2,7 @@ import { DownloadOutlined } from '@ant-design/icons';
 import { Button, message, Modal, Typography } from 'antd';
 import React, { useEffect, useState } from 'react';
 import styled from 'styled-components';
+import YAML from 'yamljs';
 import { useGetIngestionExecutionRequestQuery } from '../../../../graphql/ingestion.generated';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 import { downloadFile } from '../../../search/utils/csvUtils';
@@ -65,6 +66,13 @@ const IngestedAssetsSection = styled.div`
     padding-right: 30px;
 `;
 
+const RecipeSection = styled.div`
+    border-top: 1px solid ${ANTD_GRAY[4]};
+    padding-top: 16px;
+    padding-left: 30px;
+    padding-right: 30px;
+`;
+
 const LogsSection = styled.div`
     padding-top: 16px;
     padding-left: 30px;
@@ -91,6 +99,8 @@ type Props = {
 
 export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     const [showExpandedLogs, setShowExpandedLogs] = useState(false);
+    const [showExpandedRecipe, setShowExpandedRecipe] = useState(false);
+
     const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
     const output = data?.executionRequest?.result?.report || 'No output found.';
 
@@ -120,7 +130,18 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     const resultSummaryText =
         (result && <Typography.Text type="secondary">{getExecutionRequestSummaryText(result)}</Typography.Text>) ||
         undefined;
-    const isOutputExpandable = output.length > 100;
+
+    const recipeJson = data?.executionRequest?.input.arguments?.find((arg) => arg.key === 'recipe')?.value;
+    let recipeYaml: string;
+    try {
+        recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
+    } catch (e) {
+        recipeYaml = '';
+    }
+    const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n').slice(0, 1).join('\n');
+
+    const areLogsExpandable = output.length > 100;
+    const isRecipeExpandable = recipeYaml?.includes('\n');
 
     return (
         <Modal
@@ -161,14 +182,32 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
                         </Button>
                     </SectionSubHeader>
                     <Typography.Paragraph ellipsis>
-                        <pre>{`${logs}${!showExpandedLogs && isOutputExpandable ? '...' : ''}`}</pre>
-                        {isOutputExpandable && (
+                        <pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
+                        {areLogsExpandable && (
                             <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
                                 {showExpandedLogs ? 'Hide' : 'Show More'}
                             </ShowMoreButton>
                         )}
                     </Typography.Paragraph>
                 </LogsSection>
+                {recipe && (
+                    <RecipeSection>
+                        <SectionHeader level={5}>Recipe</SectionHeader>
+                        <SectionSubHeader>
+                            <SubHeaderParagraph type="secondary">
+                                The recipe used for this ingestion run.
+                            </SubHeaderParagraph>
+                        </SectionSubHeader>
+                        <Typography.Paragraph ellipsis>
+                            <pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '\n...' : ''}`}</pre>
+                        </Typography.Paragraph>
+                        {isRecipeExpandable && (
+                            <ShowMoreButton type="link" onClick={() => setShowExpandedRecipe((v) => !v)}>
+                                {showExpandedRecipe ? 'Hide' : 'Show More'}
+                            </ShowMoreButton>
+                        )}
+                    </RecipeSection>
+                )}
             </Section>
         </Modal>
     );
diff --git a/datahub-web-react/src/app/ingest/source/utils.ts b/datahub-web-react/src/app/ingest/source/utils.ts
index c372388e958b7..f789ed8434721 100644
--- a/datahub-web-react/src/app/ingest/source/utils.ts
+++ b/datahub-web-react/src/app/ingest/source/utils.ts
@@ -1,17 +1,19 @@
-import YAML from 'yamljs';
 import {
     CheckCircleOutlined,
     ClockCircleOutlined,
     CloseCircleOutlined,
+    ExclamationCircleOutlined,
     LoadingOutlined,
+    StopOutlined,
     WarningOutlined,
 } from '@ant-design/icons';
-import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
+import YAML from 'yamljs';
+import { ListIngestionSourcesDocument, ListIngestionSourcesQuery } from '../../../graphql/ingestion.generated';
 import { EntityType, FacetMetadata } from '../../../types.generated';
-import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
 import EntityRegistry from '../../entity/EntityRegistry';
+import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
+import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
 import { SourceConfig } from './builder/types';
-import { ListIngestionSourcesDocument, ListIngestionSourcesQuery } from '../../../graphql/ingestion.generated';
 
 export const getSourceConfigs = (ingestionSources: SourceConfig[], sourceType: string) => {
     const sourceConfigs = ingestionSources.find((source) => source.name === sourceType);
@@ -40,7 +42,9 @@ export function getPlaceholderRecipe(ingestionSources: SourceConfig[], type?: st
 
 export const RUNNING = 'RUNNING';
 export const SUCCESS = 'SUCCESS';
+export const WARNING = 'WARNING';
 export const FAILURE = 'FAILURE';
+export const CONNECTION_FAILURE = 'CONNECTION_FAILURE';
 export const CANCELLED = 'CANCELLED';
 export const UP_FOR_RETRY = 'UP_FOR_RETRY';
 export const ROLLING_BACK = 'ROLLING_BACK';
@@ -56,8 +60,10 @@ export const getExecutionRequestStatusIcon = (status: string) => {
     return (
         (status === RUNNING && LoadingOutlined) ||
         (status === SUCCESS && CheckCircleOutlined) ||
+        (status === WARNING && ExclamationCircleOutlined) ||
         (status === FAILURE && CloseCircleOutlined) ||
-        (status === CANCELLED && CloseCircleOutlined) ||
+        (status === CONNECTION_FAILURE && CloseCircleOutlined) ||
+        (status === CANCELLED && StopOutlined) ||
         (status === UP_FOR_RETRY && ClockCircleOutlined) ||
         (status === ROLLED_BACK && WarningOutlined) ||
         (status === ROLLING_BACK && LoadingOutlined) ||
@@ -70,7 +76,9 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
     return (
         (status === RUNNING && 'Running') ||
         (status === SUCCESS && 'Succeeded') ||
+        (status === WARNING && 'Completed') ||
         (status === FAILURE && 'Failed') ||
+        (status === CONNECTION_FAILURE && 'Connection Failed') ||
         (status === CANCELLED && 'Cancelled') ||
         (status === UP_FOR_RETRY && 'Up for Retry') ||
         (status === ROLLED_BACK && 'Rolled Back') ||
@@ -83,21 +91,25 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
 export const getExecutionRequestSummaryText = (status: string) => {
     switch (status) {
         case RUNNING:
-            return 'Ingestion is running';
+            return 'Ingestion is running...';
         case SUCCESS:
-            return 'Ingestion successfully completed';
+            return 'Ingestion succeeded with no errors or suspected missing data.';
+        case WARNING:
+            return 'Ingestion completed with minor or intermittent errors.';
         case FAILURE:
-            return 'Ingestion completed with errors';
+            return 'Ingestion failed to complete, or completed with serious errors.';
+        case CONNECTION_FAILURE:
+            return 'Ingestion failed due to network, authentication, or permission issues.';
         case CANCELLED:
-            return 'Ingestion was cancelled';
+            return 'Ingestion was cancelled.';
         case ROLLED_BACK:
-            return 'Ingestion was rolled back';
+            return 'Ingestion was rolled back.';
         case ROLLING_BACK:
-            return 'Ingestion is in the process of rolling back';
+            return 'Ingestion is in the process of rolling back.';
         case ROLLBACK_FAILED:
-            return 'Ingestion rollback failed';
+            return 'Ingestion rollback failed.';
         default:
-            return 'Ingestion status not recognized';
+            return 'Ingestion status not recognized.';
     }
 };
 
@@ -105,7 +117,9 @@ export const getExecutionRequestStatusDisplayColor = (status: string) => {
     return (
         (status === RUNNING && REDESIGN_COLORS.BLUE) ||
         (status === SUCCESS && 'green') ||
+        (status === WARNING && 'orangered') ||
         (status === FAILURE && 'red') ||
+        (status === CONNECTION_FAILURE && 'crimson') ||
         (status === UP_FOR_RETRY && 'orange') ||
         (status === CANCELLED && ANTD_GRAY[9]) ||
         (status === ROLLED_BACK && 'orange') ||
diff --git a/datahub-web-react/src/graphql/ingestion.graphql b/datahub-web-react/src/graphql/ingestion.graphql
index 80f66642fe11f..c127e9ec03f9a 100644
--- a/datahub-web-react/src/graphql/ingestion.graphql
+++ b/datahub-web-react/src/graphql/ingestion.graphql
@@ -90,6 +90,10 @@ query getIngestionExecutionRequest($urn: String!) {
             source {
                 type
             }
+            arguments {
+                key
+                value
+            }
         }
         result {
             status

From 1b737243b266843136918ec92f6d20573b999272 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Wed, 18 Oct 2023 13:45:46 -0500
Subject: [PATCH 067/259] feat(avro): upgrade avro to 1.11 (#9031)

---
 build.gradle                                             | 7 +++----
 buildSrc/build.gradle                                    | 9 ++++++++-
 docker/datahub-frontend/start.sh                         | 1 +
 metadata-dao-impl/kafka-producer/build.gradle            | 4 ++--
 metadata-events/{mxe-avro-1.7 => mxe-avro}/.gitignore    | 0
 metadata-events/{mxe-avro-1.7 => mxe-avro}/build.gradle  | 6 +++---
 metadata-events/mxe-registration/build.gradle            | 2 +-
 metadata-events/mxe-schemas/build.gradle                 | 2 +-
 .../{mxe-utils-avro-1.7 => mxe-utils-avro}/.gitignore    | 0
 .../{mxe-utils-avro-1.7 => mxe-utils-avro}/build.gradle  | 2 +-
 .../src/main/java/com/linkedin/metadata/EventUtils.java  | 0
 .../test/java/com/linkedin/metadata/EventUtilsTests.java | 0
 .../src/test/resources/test-avro2pegasus-mae.json        | 0
 .../src/test/resources/test-avro2pegasus-mce.json        | 0
 .../src/test/resources/test-pegasus2avro-fmce.json       | 0
 .../src/test/resources/test-pegasus2avro-mae.json        | 0
 .../src/test/resources/test-pegasus2avro-mce.json        | 0
 metadata-integration/java/datahub-client/build.gradle    | 2 +-
 .../main/java/datahub/client/kafka/AvroSerializer.java   | 4 +++-
 metadata-io/build.gradle                                 | 4 ++--
 metadata-jobs/mae-consumer/build.gradle                  | 4 ++--
 metadata-jobs/mce-consumer/build.gradle                  | 4 ++--
 metadata-jobs/pe-consumer/build.gradle                   | 4 ++--
 metadata-service/restli-servlet-impl/build.gradle        | 2 +-
 metadata-service/services/build.gradle                   | 4 ++--
 metadata-utils/build.gradle                              | 6 +++---
 settings.gradle                                          | 4 ++--
 27 files changed, 40 insertions(+), 31 deletions(-)
 rename metadata-events/{mxe-avro-1.7 => mxe-avro}/.gitignore (100%)
 rename metadata-events/{mxe-avro-1.7 => mxe-avro}/build.gradle (81%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/.gitignore (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/build.gradle (95%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/main/java/com/linkedin/metadata/EventUtils.java (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/test/java/com/linkedin/metadata/EventUtilsTests.java (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/test/resources/test-avro2pegasus-mae.json (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/test/resources/test-avro2pegasus-mce.json (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/test/resources/test-pegasus2avro-fmce.json (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/test/resources/test-pegasus2avro-mae.json (100%)
 rename metadata-events/{mxe-utils-avro-1.7 => mxe-utils-avro}/src/test/resources/test-pegasus2avro-mce.json (100%)

diff --git a/build.gradle b/build.gradle
index 025c588da2b52..cf55a59cfe694 100644
--- a/build.gradle
+++ b/build.gradle
@@ -27,7 +27,7 @@ buildscript {
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
-    classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.8.1'
+    classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.2.0'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:3.0.0"
@@ -67,8 +67,8 @@ project.ext.externalDependency = [
     'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2',
     'antlr4': 'org.antlr:antlr4:4.7.2',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
-    'avro_1_7': 'org.apache.avro:avro:1.7.7',
-    'avroCompiler_1_7': 'org.apache.avro:avro-compiler:1.7.7',
+    'avro': 'org.apache.avro:avro:1.11.3',
+    'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
     'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
@@ -127,7 +127,6 @@ project.ext.externalDependency = [
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
     'jna': 'net.java.dev.jna:jna:5.12.1',
     'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
-    'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20230227',
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 65b3780431db9..1f9d30d520171 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -5,7 +5,14 @@ buildscript {
 }
 
 dependencies {
-  implementation('io.acryl:json-schema-avro:0.1.5') {
+  /**
+   * Forked version of abandoned repository: https://github.com/fge/json-schema-avro
+   * Maintainer last active 2014, we maintain an active fork of this repository to utilize mapping Avro schemas to Json Schemas,
+   * repository is as close to official library for this as you can get. Original maintainer is one of the authors of Json Schema spec.
+   * Other companies are also separately maintaining forks (like: https://github.com/java-json-tools/json-schema-avro).
+   * We have built several customizations on top of it for various bug fixes, especially around union scheams
+   */
+  implementation('io.acryl:json-schema-avro:0.2.2') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
diff --git a/docker/datahub-frontend/start.sh b/docker/datahub-frontend/start.sh
index 9dc1514144bb1..430982aa2456b 100755
--- a/docker/datahub-frontend/start.sh
+++ b/docker/datahub-frontend/start.sh
@@ -50,6 +50,7 @@ export JAVA_OPTS="-Xms512m \
    -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf \
    -Dlogback.configurationFile=datahub-frontend/conf/logback.xml \
    -Dlogback.debug=false \
+   -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 \
    ${PROMETHEUS_AGENT:-} ${OTEL_AGENT:-} \
    ${TRUSTSTORE_FILE:-} ${TRUSTSTORE_TYPE:-} ${TRUSTSTORE_PASSWORD:-} \
    ${HTTP_PROXY:-} ${HTTPS_PROXY:-} ${NO_PROXY:-} \
diff --git a/metadata-dao-impl/kafka-producer/build.gradle b/metadata-dao-impl/kafka-producer/build.gradle
index 393b10b0e9d24..bc3415b2ccc8c 100644
--- a/metadata-dao-impl/kafka-producer/build.gradle
+++ b/metadata-dao-impl/kafka-producer/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'java'
 
 dependencies {
-  implementation project(':metadata-events:mxe-avro-1.7')
+  implementation project(':metadata-events:mxe-avro')
   implementation project(':metadata-events:mxe-registration')
-  implementation project(':metadata-events:mxe-utils-avro-1.7')
+  implementation project(':metadata-events:mxe-utils-avro')
   implementation project(':entity-registry')
   implementation project(':metadata-io')
 
diff --git a/metadata-events/mxe-avro-1.7/.gitignore b/metadata-events/mxe-avro/.gitignore
similarity index 100%
rename from metadata-events/mxe-avro-1.7/.gitignore
rename to metadata-events/mxe-avro/.gitignore
diff --git a/metadata-events/mxe-avro-1.7/build.gradle b/metadata-events/mxe-avro/build.gradle
similarity index 81%
rename from metadata-events/mxe-avro-1.7/build.gradle
rename to metadata-events/mxe-avro/build.gradle
index 8c0a26d22dc7d..9d11eeb160ff0 100644
--- a/metadata-events/mxe-avro-1.7/build.gradle
+++ b/metadata-events/mxe-avro/build.gradle
@@ -6,8 +6,8 @@ apply plugin: 'io.acryl.gradle.plugin.avro'
 apply plugin: 'java-library'
 
 dependencies {
-  api externalDependency.avro_1_7
-  implementation(externalDependency.avroCompiler_1_7) {
+  api externalDependency.avro
+  implementation(externalDependency.avroCompiler) {
     exclude group: 'org.apache.velocity', module: 'velocity'
   }
   constraints {
@@ -21,7 +21,7 @@ dependencies {
 
 def genDir = file("src/generated/java")
 
-task avroCodeGen(type: com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask, dependsOn: configurations.avsc) {
+task avroCodeGen(type: com.github.davidmc24.gradle.plugin.avro.GenerateAvroJavaTask, dependsOn: configurations.avsc) {
   source("$rootDir/metadata-events/mxe-schemas/src/renamed/avro")
   outputDir = genDir
   dependsOn(':metadata-events:mxe-schemas:renameNamespace')
diff --git a/metadata-events/mxe-registration/build.gradle b/metadata-events/mxe-registration/build.gradle
index 60e0da59616d9..032870d93329f 100644
--- a/metadata-events/mxe-registration/build.gradle
+++ b/metadata-events/mxe-registration/build.gradle
@@ -5,7 +5,7 @@ configurations {
 }
 
 dependencies {
-  implementation project(':metadata-events:mxe-avro-1.7')
+  implementation project(':metadata-events:mxe-avro')
   implementation project(':metadata-models')
   implementation spec.product.pegasus.dataAvro1_6
 
diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle
index fe46601fb68b7..8dc8b71bd1cd8 100644
--- a/metadata-events/mxe-schemas/build.gradle
+++ b/metadata-events/mxe-schemas/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'java'
+apply plugin: 'java-library'
 apply plugin: 'pegasus'
 
 dependencies {
diff --git a/metadata-events/mxe-utils-avro-1.7/.gitignore b/metadata-events/mxe-utils-avro/.gitignore
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/.gitignore
rename to metadata-events/mxe-utils-avro/.gitignore
diff --git a/metadata-events/mxe-utils-avro-1.7/build.gradle b/metadata-events/mxe-utils-avro/build.gradle
similarity index 95%
rename from metadata-events/mxe-utils-avro-1.7/build.gradle
rename to metadata-events/mxe-utils-avro/build.gradle
index 3b137965d6c19..a7bf287ab224d 100644
--- a/metadata-events/mxe-utils-avro-1.7/build.gradle
+++ b/metadata-events/mxe-utils-avro/build.gradle
@@ -1,7 +1,7 @@
 apply plugin: 'java-library'
 
 dependencies {
-  api project(':metadata-events:mxe-avro-1.7')
+  api project(':metadata-events:mxe-avro')
   api project(':metadata-models')
   api spec.product.pegasus.dataAvro1_6
 
diff --git a/metadata-events/mxe-utils-avro-1.7/src/main/java/com/linkedin/metadata/EventUtils.java b/metadata-events/mxe-utils-avro/src/main/java/com/linkedin/metadata/EventUtils.java
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/main/java/com/linkedin/metadata/EventUtils.java
rename to metadata-events/mxe-utils-avro/src/main/java/com/linkedin/metadata/EventUtils.java
diff --git a/metadata-events/mxe-utils-avro-1.7/src/test/java/com/linkedin/metadata/EventUtilsTests.java b/metadata-events/mxe-utils-avro/src/test/java/com/linkedin/metadata/EventUtilsTests.java
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/test/java/com/linkedin/metadata/EventUtilsTests.java
rename to metadata-events/mxe-utils-avro/src/test/java/com/linkedin/metadata/EventUtilsTests.java
diff --git a/metadata-events/mxe-utils-avro-1.7/src/test/resources/test-avro2pegasus-mae.json b/metadata-events/mxe-utils-avro/src/test/resources/test-avro2pegasus-mae.json
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/test/resources/test-avro2pegasus-mae.json
rename to metadata-events/mxe-utils-avro/src/test/resources/test-avro2pegasus-mae.json
diff --git a/metadata-events/mxe-utils-avro-1.7/src/test/resources/test-avro2pegasus-mce.json b/metadata-events/mxe-utils-avro/src/test/resources/test-avro2pegasus-mce.json
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/test/resources/test-avro2pegasus-mce.json
rename to metadata-events/mxe-utils-avro/src/test/resources/test-avro2pegasus-mce.json
diff --git a/metadata-events/mxe-utils-avro-1.7/src/test/resources/test-pegasus2avro-fmce.json b/metadata-events/mxe-utils-avro/src/test/resources/test-pegasus2avro-fmce.json
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/test/resources/test-pegasus2avro-fmce.json
rename to metadata-events/mxe-utils-avro/src/test/resources/test-pegasus2avro-fmce.json
diff --git a/metadata-events/mxe-utils-avro-1.7/src/test/resources/test-pegasus2avro-mae.json b/metadata-events/mxe-utils-avro/src/test/resources/test-pegasus2avro-mae.json
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/test/resources/test-pegasus2avro-mae.json
rename to metadata-events/mxe-utils-avro/src/test/resources/test-pegasus2avro-mae.json
diff --git a/metadata-events/mxe-utils-avro-1.7/src/test/resources/test-pegasus2avro-mce.json b/metadata-events/mxe-utils-avro/src/test/resources/test-pegasus2avro-mce.json
similarity index 100%
rename from metadata-events/mxe-utils-avro-1.7/src/test/resources/test-pegasus2avro-mce.json
rename to metadata-events/mxe-utils-avro/src/test/resources/test-pegasus2avro-mce.json
diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle
index 95de3cdb3c526..e6210f1f073f6 100644
--- a/metadata-integration/java/datahub-client/build.gradle
+++ b/metadata-integration/java/datahub-client/build.gradle
@@ -30,7 +30,7 @@ dependencies {
   implementation(externalDependency.kafkaAvroSerializer) {
     exclude group: "org.apache.avro"
   }
-  implementation externalDependency.avro_1_7
+  implementation externalDependency.avro
   constraints {
     implementation('commons-collections:commons-collections:3.2.2') {
       because 'Vulnerability Issue'
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
index ee0d459aaa7d3..6212e57470be4 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
@@ -16,12 +16,14 @@ class AvroSerializer {
 
   private final Schema _recordSchema;
   private final Schema _genericAspectSchema;
+  private final Schema _changeTypeEnumSchema;
   private final EventFormatter _eventFormatter;
 
   public AvroSerializer() throws IOException {
     _recordSchema = new Schema.Parser()
         .parse(this.getClass().getClassLoader().getResourceAsStream("MetadataChangeProposal.avsc"));
     _genericAspectSchema = this._recordSchema.getField("aspect").schema().getTypes().get(1);
+    _changeTypeEnumSchema = this._recordSchema.getField("changeType").schema();
     _eventFormatter = new EventFormatter(EventFormatter.Format.PEGASUS_JSON);
   }
 
@@ -43,7 +45,7 @@ public GenericRecord serialize(MetadataChangeProposal mcp) throws IOException {
     genericRecord.put("aspect", genericAspect);
     genericRecord.put("aspectName", mcp.getAspectName());
     genericRecord.put("entityType", mcp.getEntityType());
-    genericRecord.put("changeType", mcp.getChangeType());
+    genericRecord.put("changeType", new GenericData.EnumSymbol(_changeTypeEnumSchema, mcp.getChangeType()));
     return genericRecord;
   }
 }
\ No newline at end of file
diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle
index ad54cf6524398..740fed61f13d5 100644
--- a/metadata-io/build.gradle
+++ b/metadata-io/build.gradle
@@ -8,9 +8,9 @@ configurations {
 dependencies {
   implementation project(':entity-registry')
   api project(':metadata-utils')
-  api project(':metadata-events:mxe-avro-1.7')
+  api project(':metadata-events:mxe-avro')
   api project(':metadata-events:mxe-registration')
-  api project(':metadata-events:mxe-utils-avro-1.7')
+  api project(':metadata-events:mxe-utils-avro')
   api project(':metadata-models')
   api project(':metadata-service:restli-client')
   api project(':metadata-service:configuration')
diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle
index d36fd0de40d03..fcb8b62e4ac9d 100644
--- a/metadata-jobs/mae-consumer/build.gradle
+++ b/metadata-jobs/mae-consumer/build.gradle
@@ -21,9 +21,9 @@ dependencies {
     implementation project(':ingestion-scheduler')
     implementation project(':metadata-utils')
     implementation project(":entity-registry")
-    implementation project(':metadata-events:mxe-avro-1.7')
+    implementation project(':metadata-events:mxe-avro')
     implementation project(':metadata-events:mxe-registration')
-    implementation project(':metadata-events:mxe-utils-avro-1.7')
+    implementation project(':metadata-events:mxe-utils-avro')
     implementation project(':datahub-graphql-core')
 
     implementation externalDependency.elasticSearchRest
diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle
index 0bca55e0e5f92..97eec9fcff051 100644
--- a/metadata-jobs/mce-consumer/build.gradle
+++ b/metadata-jobs/mce-consumer/build.gradle
@@ -17,9 +17,9 @@ dependencies {
     }
     implementation project(':metadata-utils')
     implementation project(':metadata-events:mxe-schemas')
-    implementation project(':metadata-events:mxe-avro-1.7')
+    implementation project(':metadata-events:mxe-avro')
     implementation project(':metadata-events:mxe-registration')
-    implementation project(':metadata-events:mxe-utils-avro-1.7')
+    implementation project(':metadata-events:mxe-utils-avro')
     implementation project(':metadata-io')
     implementation project(':metadata-service:restli-client')
     implementation spec.product.pegasus.restliClient
diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle
index 1899a4de15635..81e8b8c9971f0 100644
--- a/metadata-jobs/pe-consumer/build.gradle
+++ b/metadata-jobs/pe-consumer/build.gradle
@@ -10,9 +10,9 @@ configurations {
 dependencies {
   avro project(path: ':metadata-models', configuration: 'avroSchema')
   implementation project(':li-utils')
-  implementation project(':metadata-events:mxe-avro-1.7')
+  implementation project(':metadata-events:mxe-avro')
   implementation project(':metadata-events:mxe-registration')
-  implementation project(':metadata-events:mxe-utils-avro-1.7')
+  implementation project(':metadata-events:mxe-utils-avro')
   implementation(project(':metadata-service:factories')) {
     exclude group: 'org.neo4j.test'
   }
diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle
index cb307863748c3..de6fb6690e693 100644
--- a/metadata-service/restli-servlet-impl/build.gradle
+++ b/metadata-service/restli-servlet-impl/build.gradle
@@ -48,7 +48,7 @@ dependencies {
   implementation externalDependency.dropwizardMetricsCore
   implementation externalDependency.dropwizardMetricsJmx
 
-  compileOnly externalDependency.lombok
+  implementation externalDependency.lombok
   implementation externalDependency.neo4jJavaDriver
   implementation externalDependency.opentelemetryAnnotations
 
diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle
index 22c62af324c12..b6af3d330d185 100644
--- a/metadata-service/services/build.gradle
+++ b/metadata-service/services/build.gradle
@@ -9,9 +9,9 @@ dependencies {
     implementation externalDependency.jsonPatch
     implementation project(':entity-registry')
     implementation project(':metadata-utils')
-    implementation project(':metadata-events:mxe-avro-1.7')
+    implementation project(':metadata-events:mxe-avro')
     implementation project(':metadata-events:mxe-registration')
-    implementation project(':metadata-events:mxe-utils-avro-1.7')
+    implementation project(':metadata-events:mxe-utils-avro')
     implementation project(':metadata-models')
     implementation project(':metadata-service:restli-client')
     implementation project(':metadata-service:configuration')
diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle
index 1c1c368611488..7bc6aa2d43442 100644
--- a/metadata-utils/build.gradle
+++ b/metadata-utils/build.gradle
@@ -1,7 +1,7 @@
 apply plugin: 'java-library'
 
 dependencies {
-  api externalDependency.avro_1_7
+  api externalDependency.avro
   implementation externalDependency.commonsLang
   api externalDependency.dropwizardMetricsCore
   implementation externalDependency.dropwizardMetricsJmx
@@ -16,8 +16,8 @@ dependencies {
 
   api project(':li-utils')
   api project(':entity-registry')
-  api project(':metadata-events:mxe-avro-1.7')
-  api project(':metadata-events:mxe-utils-avro-1.7')
+  api project(':metadata-events:mxe-avro')
+  api project(':metadata-events:mxe-utils-avro')
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
diff --git a/settings.gradle b/settings.gradle
index d6777b07b3fb3..52de461383b5e 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -20,10 +20,10 @@ include 'metadata-service:openapi-analytics-servlet'
 include 'metadata-service:plugin'
 include 'metadata-service:plugin:src:test:sample-test-plugins'
 include 'metadata-dao-impl:kafka-producer'
-include 'metadata-events:mxe-avro-1.7'
+include 'metadata-events:mxe-avro'
 include 'metadata-events:mxe-registration'
 include 'metadata-events:mxe-schemas'
-include 'metadata-events:mxe-utils-avro-1.7'
+include 'metadata-events:mxe-utils-avro'
 include 'metadata-ingestion'
 include 'metadata-jobs:mae-consumer'
 include 'metadata-jobs:mce-consumer'

From aae1347efce9edf1b5c4512ba3c72569e165947d Mon Sep 17 00:00:00 2001
From: Indy Prentice <iprentic@users.noreply.github.com>
Date: Wed, 18 Oct 2023 16:26:24 -0300
Subject: [PATCH 068/259] fix(search): Detect field type for use in defining
 the sort order (#8992)

Co-authored-by: Indy Prentice <indy@Indys-MacBook-Pro.local>
---
 .../indexbuilder/MappingsBuilder.java         | 48 +++++-------
 .../query/request/SearchRequestHandler.java   |  8 +-
 .../metadata/search/utils/ESUtils.java        | 74 ++++++++++++++++++-
 .../fixtures/SampleDataFixtureTestBase.java   | 64 ++++++++++++++--
 4 files changed, 154 insertions(+), 40 deletions(-)

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
index 004b2e0a2adc4..1edc77bbd214c 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
@@ -5,6 +5,7 @@
 import com.linkedin.metadata.models.SearchScoreFieldSpec;
 import com.linkedin.metadata.models.SearchableFieldSpec;
 import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType;
+import com.linkedin.metadata.search.utils.ESUtils;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,15 +32,6 @@ public static Map<String, String> getPartialNgramConfigWithOverrides(Map<String,
 
   public static final Map<String, String> KEYWORD_TYPE_MAP = ImmutableMap.of(TYPE, KEYWORD);
 
-  // Field Types
-  public static final String BOOLEAN = "boolean";
-  public static final String DATE = "date";
-  public static final String DOUBLE = "double";
-  public static final String LONG = "long";
-  public static final String OBJECT = "object";
-  public static final String TEXT = "text";
-  public static final String TOKEN_COUNT = "token_count";
-
   // Subfields
   public static final String DELIMITED = "delimited";
   public static final String LENGTH = "length";
@@ -74,7 +66,7 @@ public static Map<String, Object> getMappings(@Nonnull final EntitySpec entitySp
   private static Map<String, Object> getMappingsForUrn() {
     Map<String, Object> subFields = new HashMap<>();
     subFields.put(DELIMITED, ImmutableMap.of(
-            TYPE, TEXT,
+            TYPE, ESUtils.TEXT_FIELD_TYPE,
             ANALYZER, URN_ANALYZER,
             SEARCH_ANALYZER, URN_SEARCH_ANALYZER,
             SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER)
@@ -85,13 +77,13 @@ private static Map<String, Object> getMappingsForUrn() {
             )
     ));
     return ImmutableMap.<String, Object>builder()
-            .put(TYPE, KEYWORD)
+            .put(TYPE, ESUtils.KEYWORD_FIELD_TYPE)
             .put(FIELDS, subFields)
             .build();
   }
 
   private static Map<String, Object> getMappingsForRunId() {
-    return ImmutableMap.<String, Object>builder().put(TYPE, KEYWORD).build();
+    return ImmutableMap.<String, Object>builder().put(TYPE, ESUtils.KEYWORD_FIELD_TYPE).build();
   }
 
   private static Map<String, Object> getMappingsForField(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
@@ -104,23 +96,23 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
     } else if (fieldType == FieldType.TEXT || fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) {
       mappingForField.putAll(getMappingsForSearchText(fieldType));
     } else if (fieldType == FieldType.BROWSE_PATH) {
-      mappingForField.put(TYPE, TEXT);
+      mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
       mappingForField.put(FIELDS,
           ImmutableMap.of(LENGTH, ImmutableMap.of(
-              TYPE, TOKEN_COUNT,
+              TYPE, ESUtils.TOKEN_COUNT_FIELD_TYPE,
               ANALYZER, SLASH_PATTERN_ANALYZER)));
       mappingForField.put(ANALYZER, BROWSE_PATH_HIERARCHY_ANALYZER);
       mappingForField.put(FIELDDATA, true);
     } else if (fieldType == FieldType.BROWSE_PATH_V2) {
-      mappingForField.put(TYPE, TEXT);
+      mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
       mappingForField.put(FIELDS,
           ImmutableMap.of(LENGTH, ImmutableMap.of(
-              TYPE, TOKEN_COUNT,
+              TYPE, ESUtils.TOKEN_COUNT_FIELD_TYPE,
               ANALYZER, UNIT_SEPARATOR_PATTERN_ANALYZER)));
       mappingForField.put(ANALYZER, BROWSE_PATH_V2_HIERARCHY_ANALYZER);
       mappingForField.put(FIELDDATA, true);
     }  else if (fieldType == FieldType.URN || fieldType == FieldType.URN_PARTIAL) {
-      mappingForField.put(TYPE, TEXT);
+      mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
       mappingForField.put(ANALYZER, URN_ANALYZER);
       mappingForField.put(SEARCH_ANALYZER, URN_SEARCH_ANALYZER);
       mappingForField.put(SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER);
@@ -135,13 +127,13 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
       subFields.put(KEYWORD, KEYWORD_TYPE_MAP);
       mappingForField.put(FIELDS, subFields);
     } else if (fieldType == FieldType.BOOLEAN) {
-      mappingForField.put(TYPE, BOOLEAN);
+      mappingForField.put(TYPE, ESUtils.BOOLEAN_FIELD_TYPE);
     } else if (fieldType == FieldType.COUNT) {
-      mappingForField.put(TYPE, LONG);
+      mappingForField.put(TYPE, ESUtils.LONG_FIELD_TYPE);
     } else if (fieldType == FieldType.DATETIME) {
-      mappingForField.put(TYPE, DATE);
+      mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
     } else if (fieldType == FieldType.OBJECT) {
-      mappingForField.put(TYPE, OBJECT);
+      mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
     } else {
       log.info("FieldType {} has no mappings implemented", fieldType);
     }
@@ -149,10 +141,10 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
 
     searchableFieldSpec.getSearchableAnnotation()
         .getHasValuesFieldName()
-        .ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, BOOLEAN)));
+        .ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.BOOLEAN_FIELD_TYPE)));
     searchableFieldSpec.getSearchableAnnotation()
         .getNumValuesFieldName()
-        .ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, LONG)));
+        .ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.LONG_FIELD_TYPE)));
     mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec));
 
     return mappings;
@@ -160,7 +152,7 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
 
   private static Map<String, Object> getMappingsForKeyword() {
     Map<String, Object> mappingForField = new HashMap<>();
-    mappingForField.put(TYPE, KEYWORD);
+    mappingForField.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE);
     mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER);
     // Add keyword subfield without lowercase filter
     mappingForField.put(FIELDS, ImmutableMap.of(KEYWORD, KEYWORD_TYPE_MAP));
@@ -169,7 +161,7 @@ private static Map<String, Object> getMappingsForKeyword() {
 
   private static Map<String, Object> getMappingsForSearchText(FieldType fieldType) {
     Map<String, Object> mappingForField = new HashMap<>();
-    mappingForField.put(TYPE, KEYWORD);
+    mappingForField.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE);
     mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER);
     Map<String, Object> subFields = new HashMap<>();
     if (fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) {
@@ -186,14 +178,14 @@ private static Map<String, Object> getMappingsForSearchText(FieldType fieldType)
           String fieldName = entry.getKey();
           String analyzerName = entry.getValue();
           subFields.put(fieldName, ImmutableMap.of(
-                  TYPE, TEXT,
+                  TYPE, ESUtils.TEXT_FIELD_TYPE,
                   ANALYZER, analyzerName
           ));
         }
       }
     }
     subFields.put(DELIMITED, ImmutableMap.of(
-            TYPE, TEXT,
+            TYPE, ESUtils.TEXT_FIELD_TYPE,
             ANALYZER, TEXT_ANALYZER,
             SEARCH_ANALYZER, TEXT_SEARCH_ANALYZER,
             SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER));
@@ -206,7 +198,7 @@ private static Map<String, Object> getMappingsForSearchText(FieldType fieldType)
   private static Map<String, Object> getMappingsForSearchScoreField(
       @Nonnull final SearchScoreFieldSpec searchScoreFieldSpec) {
     return ImmutableMap.of(searchScoreFieldSpec.getSearchScoreAnnotation().getFieldName(),
-        ImmutableMap.of(TYPE, DOUBLE));
+        ImmutableMap.of(TYPE, ESUtils.DOUBLE_FIELD_TYPE));
   }
 
   private static Map<String, Object> getMappingsForFieldNameAliases(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
index 5fcc10b7af5cf..c06907e800d5e 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
@@ -202,7 +202,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
     if (!finalSearchFlags.isSkipHighlighting()) {
       searchSourceBuilder.highlighter(_highlights);
     }
-    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
+    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
 
     if (finalSearchFlags.isGetSuggestions()) {
       ESUtils.buildNameSuggestions(searchSourceBuilder, input);
@@ -243,7 +243,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
     searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery));
     _aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
     searchSourceBuilder.highlighter(getHighlights());
-    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
+    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
     searchRequest.source(searchSourceBuilder);
     log.debug("Search request is: " + searchRequest);
     searchRequest.indicesOptions(null);
@@ -270,7 +270,7 @@ public SearchRequest getFilterRequest(@Nullable Filter filters, @Nullable SortCr
     final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
     searchSourceBuilder.query(filterQuery);
     searchSourceBuilder.from(from).size(size);
-    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
+    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
     searchRequest.source(searchSourceBuilder);
 
     return searchRequest;
@@ -301,7 +301,7 @@ public SearchRequest getFilterRequest(@Nullable Filter filters, @Nullable SortCr
     searchSourceBuilder.size(size);
 
     ESUtils.setSearchAfter(searchSourceBuilder, sort, pitId, keepAlive);
-    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
+    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
     searchRequest.source(searchSourceBuilder);
 
     return searchRequest;
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
index 9a7d9a1b4c420..53765acb8e29e 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
@@ -2,6 +2,9 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.models.SearchableFieldSpec;
+import com.linkedin.metadata.models.annotation.SearchableAnnotation;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.Criterion;
@@ -49,7 +52,28 @@ public class ESUtils {
   public static final int MAX_RESULT_SIZE = 10000;
   public static final String OPAQUE_ID_HEADER = "X-Opaque-Id";
   public static final String HEADER_VALUE_DELIMITER = "|";
-  public static final String KEYWORD_TYPE = "keyword";
+
+  // Field types
+  public static final String KEYWORD_FIELD_TYPE = "keyword";
+  public static final String BOOLEAN_FIELD_TYPE = "boolean";
+  public static final String DATE_FIELD_TYPE = "date";
+  public static final String DOUBLE_FIELD_TYPE = "double";
+  public static final String LONG_FIELD_TYPE = "long";
+  public static final String OBJECT_FIELD_TYPE = "object";
+  public static final String TEXT_FIELD_TYPE = "text";
+  public static final String TOKEN_COUNT_FIELD_TYPE = "token_count";
+  // End of field types
+
+  public static final Set<SearchableAnnotation.FieldType> FIELD_TYPES_STORED_AS_KEYWORD = Set.of(
+      SearchableAnnotation.FieldType.KEYWORD,
+      SearchableAnnotation.FieldType.TEXT,
+      SearchableAnnotation.FieldType.TEXT_PARTIAL,
+      SearchableAnnotation.FieldType.WORD_GRAM);
+  public static final Set<SearchableAnnotation.FieldType> FIELD_TYPES_STORED_AS_TEXT = Set.of(
+      SearchableAnnotation.FieldType.BROWSE_PATH,
+      SearchableAnnotation.FieldType.BROWSE_PATH_V2,
+      SearchableAnnotation.FieldType.URN,
+      SearchableAnnotation.FieldType.URN_PARTIAL);
   public static final String ENTITY_NAME_FIELD = "_entityName";
   public static final String NAME_SUGGESTION = "nameSuggestion";
 
@@ -174,6 +198,25 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
     return getQueryBuilderFromCriterionForSingleField(criterion, isTimeseries);
   }
 
+  public static String getElasticTypeForFieldType(SearchableAnnotation.FieldType fieldType) {
+    if (FIELD_TYPES_STORED_AS_KEYWORD.contains(fieldType)) {
+      return KEYWORD_FIELD_TYPE;
+    } else if (FIELD_TYPES_STORED_AS_TEXT.contains(fieldType)) {
+      return TEXT_FIELD_TYPE;
+    } else if (fieldType == SearchableAnnotation.FieldType.BOOLEAN) {
+      return BOOLEAN_FIELD_TYPE;
+    } else if (fieldType == SearchableAnnotation.FieldType.COUNT) {
+      return LONG_FIELD_TYPE;
+    } else if (fieldType == SearchableAnnotation.FieldType.DATETIME) {
+      return DATE_FIELD_TYPE;
+    } else if (fieldType == SearchableAnnotation.FieldType.OBJECT) {
+      return OBJECT_FIELD_TYPE;
+    } else {
+      log.warn("FieldType {} has no mappings implemented", fieldType);
+      return null;
+    }
+  }
+
   /**
    * Populates source field of search query with the sort order as per the criterion provided.
    *
@@ -189,14 +232,39 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
    * @param sortCriterion {@link SortCriterion} to be applied to the search results
    */
   public static void buildSortOrder(@Nonnull SearchSourceBuilder searchSourceBuilder,
-      @Nullable SortCriterion sortCriterion) {
+      @Nullable SortCriterion sortCriterion, List<EntitySpec> entitySpecs) {
     if (sortCriterion == null) {
       searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
     } else {
+      Optional<SearchableAnnotation.FieldType> fieldTypeForDefault = Optional.empty();
+      for (EntitySpec entitySpec : entitySpecs) {
+        List<SearchableFieldSpec> fieldSpecs = entitySpec.getSearchableFieldSpecs();
+        for (SearchableFieldSpec fieldSpec : fieldSpecs) {
+          SearchableAnnotation annotation = fieldSpec.getSearchableAnnotation();
+          if (annotation.getFieldName().equals(sortCriterion.getField())
+              || annotation.getFieldNameAliases().contains(sortCriterion.getField())) {
+            fieldTypeForDefault = Optional.of(fieldSpec.getSearchableAnnotation().getFieldType());
+            break;
+          }
+        }
+        if (fieldTypeForDefault.isPresent()) {
+          break;
+        }
+      }
+      if (fieldTypeForDefault.isEmpty()) {
+        log.warn("Sort criterion field " + sortCriterion.getField() + " was not found in any entity spec to be searched");
+      }
       final SortOrder esSortOrder =
           (sortCriterion.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING) ? SortOrder.ASC
               : SortOrder.DESC;
-      searchSourceBuilder.sort(new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder).unmappedType(KEYWORD_TYPE));
+      FieldSortBuilder sortBuilder = new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder);
+      if (fieldTypeForDefault.isPresent()) {
+        String esFieldtype = getElasticTypeForFieldType(fieldTypeForDefault.get());
+        if (esFieldtype != null) {
+          sortBuilder.unmappedType(esFieldtype);
+        }
+      }
+      searchSourceBuilder.sort(sortBuilder);
     }
     if (sortCriterion == null || !sortCriterion.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)) {
       searchSourceBuilder.sort(new FieldSortBuilder(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD).order(SortOrder.ASC));
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
index 1660504810296..69dd5c80bef1d 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java
@@ -22,12 +22,15 @@
 import com.linkedin.metadata.query.filter.Criterion;
 import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.query.filter.SortOrder;
 import com.linkedin.metadata.search.AggregationMetadata;
 import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchResult;
 import com.linkedin.metadata.search.SearchService;
 import com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig;
+import com.linkedin.metadata.search.utils.ESUtils;
 import com.linkedin.r2.RemoteInvocationException;
 import org.junit.Assert;
 import org.opensearch.client.RequestOptions;
@@ -36,6 +39,9 @@
 import org.opensearch.client.indices.AnalyzeResponse;
 import org.opensearch.client.indices.GetMappingsRequest;
 import org.opensearch.client.indices.GetMappingsResponse;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.sort.FieldSortBuilder;
+import org.opensearch.search.sort.SortBuilder;
 import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
 import org.testng.annotations.Test;
 
@@ -54,11 +60,7 @@
 import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME;
 import static com.linkedin.metadata.search.elasticsearch.query.request.SearchQueryBuilder.STRUCTURED_QUERY_PREFIX;
 import static com.linkedin.metadata.utils.SearchUtil.AGGREGATION_SEPARATOR_CHAR;
-import static io.datahubproject.test.search.SearchTestUtils.autocomplete;
-import static io.datahubproject.test.search.SearchTestUtils.scroll;
-import static io.datahubproject.test.search.SearchTestUtils.search;
-import static io.datahubproject.test.search.SearchTestUtils.searchAcrossEntities;
-import static io.datahubproject.test.search.SearchTestUtils.searchStructured;
+import static io.datahubproject.test.search.SearchTestUtils.*;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
@@ -174,6 +176,48 @@ public void testSearchFieldConfig() throws IOException {
         }
     }
 
+    @Test
+    public void testGetSortOrder() {
+        String dateFieldName = "lastOperationTime";
+        List<String> entityNamesToTestSearch = List.of("dataset", "chart", "corpgroup");
+        List<EntitySpec> entitySpecs = entityNamesToTestSearch.stream().map(
+                name -> getEntityRegistry().getEntitySpec(name))
+            .collect(Collectors.toList());
+        SearchSourceBuilder builder = new SearchSourceBuilder();
+        SortCriterion sortCriterion = new SortCriterion().setOrder(SortOrder.DESCENDING).setField(dateFieldName);
+        ESUtils.buildSortOrder(builder, sortCriterion, entitySpecs);
+        List<SortBuilder<?>> sorts = builder.sorts();
+        assertEquals(sorts.size(), 2); // sort by last modified and then by urn
+        for (SortBuilder sort : sorts) {
+            assertTrue(sort instanceof FieldSortBuilder);
+            FieldSortBuilder fieldSortBuilder = (FieldSortBuilder) sort;
+            if (fieldSortBuilder.getFieldName().equals(dateFieldName)) {
+                assertEquals(fieldSortBuilder.order(), org.opensearch.search.sort.SortOrder.DESC);
+                assertEquals(fieldSortBuilder.unmappedType(), "date");
+            } else {
+                assertEquals(fieldSortBuilder.getFieldName(), "urn");
+            }
+        }
+
+        // Test alias field
+        String entityNameField = "_entityName";
+        SearchSourceBuilder nameBuilder = new SearchSourceBuilder();
+        SortCriterion nameCriterion = new SortCriterion().setOrder(SortOrder.ASCENDING).setField(entityNameField);
+        ESUtils.buildSortOrder(nameBuilder, nameCriterion, entitySpecs);
+        sorts = nameBuilder.sorts();
+        assertEquals(sorts.size(), 2);
+        for (SortBuilder sort : sorts) {
+            assertTrue(sort instanceof FieldSortBuilder);
+            FieldSortBuilder fieldSortBuilder = (FieldSortBuilder) sort;
+            if (fieldSortBuilder.getFieldName().equals(entityNameField)) {
+                assertEquals(fieldSortBuilder.order(), org.opensearch.search.sort.SortOrder.ASC);
+                assertEquals(fieldSortBuilder.unmappedType(), "keyword");
+            } else {
+                assertEquals(fieldSortBuilder.getFieldName(), "urn");
+            }
+        }
+    }
+
     @Test
     public void testDatasetHasTags() throws IOException {
         GetMappingsRequest req = new GetMappingsRequest()
@@ -1454,6 +1498,16 @@ public void testColumnExactMatch() {
                 "Expected table with column name exact match first");
     }
 
+    @Test
+    public void testSortOrdering() {
+        String query = "unit_data";
+        SortCriterion criterion = new SortCriterion().setOrder(SortOrder.ASCENDING).setField("lastOperationTime");
+        SearchResult result = getSearchService().searchAcrossEntities(SEARCHABLE_ENTITIES, query, null, criterion, 0,
+            100, new SearchFlags().setFulltext(true).setSkipCache(true), null);
+        assertTrue(result.getEntities().size() > 2,
+            String.format("%s - Expected search results to have at least two results", query));
+    }
+
     private Stream<AnalyzeResponse.AnalyzeToken> getTokens(AnalyzeRequest request) throws IOException {
         return getSearchClient().indices().analyze(request, RequestOptions.DEFAULT).getTokens().stream();
     }

From 7855fb60a7e96e6d04d8d96f7505f8b4dd62a7c4 Mon Sep 17 00:00:00 2001
From: Indy Prentice <iprentic@users.noreply.github.com>
Date: Wed, 18 Oct 2023 17:19:10 -0300
Subject: [PATCH 069/259] fix(api): Add preceding / to get index sizes path
 (#9043)

Co-authored-by: Indy Prentice <indy@Indys-MacBook-Pro.local>
---
 .../ElasticSearchTimeseriesAspectService.java    |  2 +-
 .../search/TimeseriesAspectServiceTestBase.java  | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
index a496fc427138e..3e8f83a531b59 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java
@@ -169,7 +169,7 @@ public List<TimeseriesIndexSizeResult> getIndexSizes() {
     List<TimeseriesIndexSizeResult> res = new ArrayList<>();
     try {
       String indicesPattern = _indexConvention.getAllTimeseriesAspectIndicesPattern();
-      Response r = _searchClient.getLowLevelClient().performRequest(new Request("GET", indicesPattern + "/_stats"));
+      Response r = _searchClient.getLowLevelClient().performRequest(new Request("GET", "/" + indicesPattern + "/_stats"));
       JsonNode body = new ObjectMapper().readTree(r.getEntity().getContent());
       body.get("indices").fields().forEachRemaining(entry -> {
         TimeseriesIndexSizeResult elemResult = new TimeseriesIndexSizeResult();
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
index cc60ba8679e1f..f9b8f84b10ad2 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
@@ -45,6 +45,7 @@
 import com.linkedin.timeseries.GroupingBucket;
 import com.linkedin.timeseries.GroupingBucketType;
 import com.linkedin.timeseries.TimeWindowSize;
+import com.linkedin.timeseries.TimeseriesIndexSizeResult;
 import org.opensearch.client.RestHighLevelClient;
 import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
 import org.testng.annotations.BeforeClass;
@@ -884,4 +885,19 @@ public void testCountByFilterAfterDelete() throws InterruptedException {
         _elasticSearchTimeseriesAspectService.countByFilter(ENTITY_NAME, ASPECT_NAME, urnAndTimeFilter);
     assertEquals(count, 0L);
   }
+
+  @Test(groups = {"getAggregatedStats"}, dependsOnGroups = {"upsert"})
+  public void testGetIndexSizes() {
+    List<TimeseriesIndexSizeResult> result = _elasticSearchTimeseriesAspectService.getIndexSizes();
+    /*
+    Example result:
+    {aspectName=testentityprofile, sizeMb=52.234, indexName=es_timeseries_aspect_service_test_testentity_testentityprofileaspect_v1, entityName=testentity}
+    {aspectName=testentityprofile, sizeMb=0.208, indexName=es_timeseries_aspect_service_test_testentitywithouttests_testentityprofileaspect_v1, entityName=testentitywithouttests}
+     */
+    // There may be other indices in there from other tests, so just make sure that index for entity + aspect is in there
+    assertTrue(result.size() > 1);
+    assertTrue(
+        result.stream().anyMatch(idxSizeResult -> idxSizeResult.getIndexName().equals(
+            "es_timeseries_aspect_service_test_testentitywithouttests_testentityprofileaspect_v1")));
+  }
 }

From 409f981fd3e12a1d470a79cb091ac92e1a4a2c46 Mon Sep 17 00:00:00 2001
From: Indy Prentice <iprentic@users.noreply.github.com>
Date: Wed, 18 Oct 2023 18:25:54 -0300
Subject: [PATCH 070/259] fix(search): Apply SearchFlags passed in through to
 scroll queries (#9041)

Co-authored-by: Indy Prentice <indy@Indys-MacBook-Pro.local>
---
 .../client/CachingEntitySearchService.java    | 13 ++++++----
 .../elasticsearch/ElasticSearchService.java   | 13 ++++++----
 .../query/request/SearchRequestHandler.java   |  4 +++-
 .../search/LineageServiceTestBase.java        | 16 ++++++++++---
 .../request/SearchRequestHandlerTest.java     | 24 +++++++++++++++++++
 .../metadata/search/EntitySearchService.java  |  6 +++--
 6 files changed, 60 insertions(+), 16 deletions(-)

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
index 13a7d16b723a7..ceaf37a1289d9 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
@@ -256,13 +256,13 @@ public ScrollResult getCachedScrollResults(
         cacheAccess.stop();
         if (result == null) {
           Timer.Context cacheMiss = MetricUtils.timer(this.getClass(), "scroll_cache_miss").time();
-          result = getRawScrollResults(entities, query, filters, sortCriterion, scrollId, keepAlive, size, isFullText);
+          result = getRawScrollResults(entities, query, filters, sortCriterion, scrollId, keepAlive, size, isFullText, flags);
           cache.put(cacheKey, toJsonString(result));
           cacheMiss.stop();
           MetricUtils.counter(this.getClass(), "scroll_cache_miss_count").inc();
         }
       } else {
-        result = getRawScrollResults(entities, query, filters, sortCriterion, scrollId, keepAlive, size, isFullText);
+        result = getRawScrollResults(entities, query, filters, sortCriterion, scrollId, keepAlive, size, isFullText, flags);
       }
       return result;
     }
@@ -328,7 +328,8 @@ private ScrollResult getRawScrollResults(
       @Nullable final String scrollId,
       @Nullable final String keepAlive,
       final int count,
-      final boolean fulltext) {
+      final boolean fulltext,
+      @Nullable final SearchFlags searchFlags) {
     if (fulltext) {
       return entitySearchService.fullTextScroll(
           entities,
@@ -337,7 +338,8 @@ private ScrollResult getRawScrollResults(
           sortCriterion,
           scrollId,
           keepAlive,
-          count);
+          count,
+          searchFlags);
     } else {
       return entitySearchService.structuredScroll(entities,
           input,
@@ -345,7 +347,8 @@ private ScrollResult getRawScrollResults(
           sortCriterion,
           scrollId,
           keepAlive,
-          count);
+          count,
+          searchFlags);
     }
   }
 
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
index ef5a555e95ba8..024cf2b0abec2 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
@@ -175,23 +175,26 @@ public List<String> getBrowsePaths(@Nonnull String entityName, @Nonnull Urn urn)
   @Nonnull
   @Override
   public ScrollResult fullTextScroll(@Nonnull List<String> entities, @Nonnull String input, @Nullable Filter postFilters,
-      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size) {
+      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size, @Nullable SearchFlags searchFlags) {
     log.debug(String.format(
         "Scrolling Structured Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
         entities, input, postFilters, sortCriterion, scrollId, size));
+    SearchFlags flags = Optional.ofNullable(searchFlags).orElse(new SearchFlags());
+    flags.setFulltext(true);
     return esSearchDAO.scroll(entities, input, postFilters, sortCriterion, scrollId, keepAlive, size,
-            new SearchFlags().setFulltext(true));
+            flags);
   }
 
   @Nonnull
   @Override
   public ScrollResult structuredScroll(@Nonnull List<String> entities, @Nonnull String input, @Nullable Filter postFilters,
-      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size) {
+      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size, @Nullable SearchFlags searchFlags) {
     log.debug(String.format(
         "Scrolling FullText Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s",
         entities, input, postFilters, sortCriterion, scrollId, size));
-    return esSearchDAO.scroll(entities, input, postFilters, sortCriterion, scrollId, keepAlive, size,
-            new SearchFlags().setFulltext(false));
+    SearchFlags flags = Optional.ofNullable(searchFlags).orElse(new SearchFlags());
+    flags.setFulltext(false);
+    return esSearchDAO.scroll(entities, input, postFilters, sortCriterion, scrollId, keepAlive, size, flags);
   }
 
   public Optional<SearchResponse> raw(@Nonnull String indexName, @Nullable String jsonQuery) {
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
index c06907e800d5e..49571a60d5f21 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
@@ -242,7 +242,9 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
     BoolQueryBuilder filterQuery = getFilterQuery(filter);
     searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery));
     _aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
-    searchSourceBuilder.highlighter(getHighlights());
+    if (!finalSearchFlags.isSkipHighlighting()) {
+      searchSourceBuilder.highlighter(_highlights);
+    }
     ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
     searchRequest.source(searchSourceBuilder);
     log.debug("Search request is: " + searchRequest);
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java
index 461a146022446..696e3b62834bd 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java
@@ -47,8 +47,10 @@
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
 import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl;
 import org.junit.Assert;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.action.search.SearchRequest;
 import org.springframework.cache.CacheManager;
 import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
 import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
@@ -108,6 +110,7 @@ abstract public class LineageServiceTestBase extends AbstractTestNGSpringContext
   private GraphService _graphService;
   private CacheManager _cacheManager;
   private LineageSearchService _lineageSearchService;
+  private RestHighLevelClient _searchClientSpy;
 
   private static final String ENTITY_NAME = "testEntity";
   private static final Urn TEST_URN = TestEntityUtil.getTestEntityUrn();
@@ -162,10 +165,11 @@ private ElasticSearchService buildEntitySearchService() {
     EntityIndexBuilders indexBuilders =
         new EntityIndexBuilders(getIndexBuilder(), _entityRegistry,
             _indexConvention, _settingsBuilder);
-    ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, getSearchClient(), _indexConvention, false,
+    _searchClientSpy = spy(getSearchClient());
+    ESSearchDAO searchDAO = new ESSearchDAO(_entityRegistry, _searchClientSpy, _indexConvention, false,
         ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, getSearchConfiguration(), null);
-    ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, getSearchClient(), _indexConvention, getSearchConfiguration(), getCustomSearchConfiguration());
-    ESWriteDAO writeDAO = new ESWriteDAO(_entityRegistry, getSearchClient(), _indexConvention, getBulkProcessor(), 1);
+    ESBrowseDAO browseDAO = new ESBrowseDAO(_entityRegistry, _searchClientSpy, _indexConvention, getSearchConfiguration(), getCustomSearchConfiguration());
+    ESWriteDAO writeDAO = new ESWriteDAO(_entityRegistry, _searchClientSpy, _indexConvention, getBulkProcessor(), 1);
     return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO);
   }
 
@@ -246,9 +250,15 @@ public void testSearchService() throws Exception {
     _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString());
     syncAfterWrite(getBulkProcessor());
 
+    Mockito.reset(_searchClientSpy);
     searchResult = searchAcrossLineage(null, TEST1);
     assertEquals(searchResult.getNumEntities().intValue(), 1);
     assertEquals(searchResult.getEntities().get(0).getEntity(), urn);
+    // Verify that highlighting was turned off in the query
+    ArgumentCaptor<SearchRequest> searchRequestCaptor = ArgumentCaptor.forClass(SearchRequest.class);
+    Mockito.verify(_searchClientSpy, times(1)).search(searchRequestCaptor.capture(), any());
+    SearchRequest capturedRequest = searchRequestCaptor.getValue();
+    assertNull(capturedRequest.source().highlighter());
     clearCache(false);
 
     when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(),
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
index 90c6c523c588f..0ea035a10f91d 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
@@ -97,6 +97,30 @@ public void testDatasetFieldsAndHighlights() {
     ), "unexpected lineage fields in highlights: " + highlightFields);
   }
 
+  @Test
+  public void testSearchRequestHandlerHighlightingTurnedOff() {
+    SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null);
+    SearchRequest searchRequest = requestHandler.getSearchRequest("testQuery", null, null, 0,
+        10,  new SearchFlags().setFulltext(false).setSkipHighlighting(true), null);
+    SearchSourceBuilder sourceBuilder = searchRequest.source();
+    assertEquals(sourceBuilder.from(), 0);
+    assertEquals(sourceBuilder.size(), 10);
+    // Filters
+    Collection<AggregationBuilder> aggBuilders = sourceBuilder.aggregations().getAggregatorFactories();
+    // Expect 2 aggregations: textFieldOverride and _index
+    assertEquals(aggBuilders.size(), 2);
+    for (AggregationBuilder aggBuilder : aggBuilders) {
+      if (aggBuilder.getName().equals("textFieldOverride")) {
+        TermsAggregationBuilder filterPanelBuilder = (TermsAggregationBuilder) aggBuilder;
+        assertEquals(filterPanelBuilder.field(), "textFieldOverride.keyword");
+      } else if (!aggBuilder.getName().equals("_entityType")) {
+        fail("Found unexepected aggregation: " + aggBuilder.getName());
+      }
+    }
+    // Highlights should not be present
+    assertNull(sourceBuilder.highlighter());
+  }
+
   @Test
   public void testSearchRequestHandler() {
     SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null);
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
index a46b58aabfb0b..64f59780b887f 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
@@ -188,11 +188,12 @@ BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nullable
    * @param sortCriterion {@link SortCriterion} to be applied to search results
    * @param scrollId opaque scroll identifier to pass to search service
    * @param size the number of search hits to return
+   * @param searchFlags flags controlling search options
    * @return a {@link ScrollResult} that contains a list of matched documents and related search result metadata
    */
   @Nonnull
   ScrollResult fullTextScroll(@Nonnull List<String> entities, @Nonnull String input, @Nullable Filter postFilters,
-      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size);
+      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size, @Nullable SearchFlags searchFlags);
 
   /**
    * Gets a list of documents that match given search request. The results are aggregated and filters are applied to the
@@ -204,11 +205,12 @@ ScrollResult fullTextScroll(@Nonnull List<String> entities, @Nonnull String inpu
    * @param sortCriterion {@link SortCriterion} to be applied to search results
    * @param scrollId opaque scroll identifier to pass to search service
    * @param size the number of search hits to return
+   * @param searchFlags flags controlling search options
    * @return a {@link ScrollResult} that contains a list of matched documents and related search result metadata
    */
   @Nonnull
   ScrollResult structuredScroll(@Nonnull List<String> entities, @Nonnull String input, @Nullable Filter postFilters,
-      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size);
+      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size, @Nullable SearchFlags searchFlags);
 
   /**
    * Max result size returned by the underlying search backend

From 269c4eac7ef09d73224050e432bfbf60727e4d65 Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Thu, 19 Oct 2023 01:43:05 +0100
Subject: [PATCH 071/259] fix(ownership): Corrects validation of ownership type
 and makes it consistent across graphQL calls (#9044)

Co-authored-by: Ellie O'Neil <oneile729@gmail.com>
---
 .../resolvers/mutate/AddOwnerResolver.java    |  27 ++-
 .../resolvers/mutate/AddOwnersResolver.java   |   2 +-
 .../mutate/BatchAddOwnersResolver.java        |   3 +-
 .../resolvers/mutate/util/OwnerUtils.java     |  65 +++-----
 .../owner/AddOwnersResolverTest.java          | 157 ++++++++++++++++--
 5 files changed, 183 insertions(+), 71 deletions(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java
index 5ca7007d98e43..3f2dab0a5ba71 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java
@@ -2,14 +2,11 @@
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.CorpuserUrn;
-
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.AddOwnerInput;
-import com.linkedin.datahub.graphql.generated.OwnerEntityType;
 import com.linkedin.datahub.graphql.generated.OwnerInput;
-import com.linkedin.datahub.graphql.generated.OwnershipType;
 import com.linkedin.datahub.graphql.generated.ResourceRefInput;
 import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils;
 import com.linkedin.metadata.entity.EntityService;
@@ -20,7 +17,6 @@
 import lombok.extern.slf4j.Slf4j;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
-import static com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils.*;
 
 
 @Slf4j
@@ -32,30 +28,33 @@ public class AddOwnerResolver implements DataFetcher<CompletableFuture<Boolean>>
   @Override
   public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throws Exception {
     final AddOwnerInput input = bindArgument(environment.getArgument("input"), AddOwnerInput.class);
-
     Urn ownerUrn = Urn.createFromString(input.getOwnerUrn());
-    OwnerEntityType ownerEntityType = input.getOwnerEntityType();
-    OwnershipType type = input.getType() == null ? OwnershipType.NONE : input.getType();
-    String ownershipUrn = input.getOwnershipTypeUrn() == null ? mapOwnershipTypeToEntity(type.name()) : input.getOwnershipTypeUrn();
     Urn targetUrn = Urn.createFromString(input.getResourceUrn());
+    OwnerInput.Builder ownerInputBuilder = OwnerInput.builder();
+    ownerInputBuilder.setOwnerUrn(input.getOwnerUrn());
+    ownerInputBuilder.setOwnerEntityType(input.getOwnerEntityType());
+    if (input.getType() != null) {
+      ownerInputBuilder.setType(input.getType());
+    }
+    if (input.getOwnershipTypeUrn() != null) {
+      ownerInputBuilder.setOwnershipTypeUrn(input.getOwnershipTypeUrn());
+    }
 
+    OwnerInput ownerInput = ownerInputBuilder.build();
     if (!OwnerUtils.isAuthorizedToUpdateOwners(environment.getContext(), targetUrn)) {
       throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
     }
 
     return CompletableFuture.supplyAsync(() -> {
-      OwnerUtils.validateAddInput(
-          ownerUrn, input.getOwnershipTypeUrn(), ownerEntityType,
-          targetUrn,
-          _entityService
-      );
+      OwnerUtils.validateAddOwnerInput(ownerInput, ownerUrn, _entityService);
+
       try {
 
         log.debug("Adding Owner. input: {}", input);
 
         Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn());
         OwnerUtils.addOwnersToResources(
-            ImmutableList.of(new OwnerInput(input.getOwnerUrn(), ownerEntityType, type, ownershipUrn)),
+            ImmutableList.of(ownerInput),
             ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), null, null)),
             actor,
             _entityService
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java
index 06424efa83819..4e5b5bdb2a651 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java
@@ -39,7 +39,7 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
         throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
       }
 
-      OwnerUtils.validateAddInput(
+      OwnerUtils.validateAddOwnerInput(
           owners,
           targetUrn,
           _entityService
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java
index 019c044d81ab3..5beaeecae673f 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java
@@ -53,8 +53,7 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
 
   private void validateOwners(List<OwnerInput> owners) {
     for (OwnerInput ownerInput : owners) {
-      OwnerUtils.validateOwner(UrnUtils.getUrn(ownerInput.getOwnerUrn()), ownerInput.getOwnerEntityType(),
-          UrnUtils.getUrn(ownerInput.getOwnershipTypeUrn()), _entityService);
+      OwnerUtils.validateOwner(ownerInput, _entityService);
     }
   }
 
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java
index d2f7f896e5953..7233995804423 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java
@@ -50,7 +50,7 @@ public static void addOwnersToResources(
   ) {
     final List<MetadataChangeProposal> changes = new ArrayList<>();
     for (ResourceRefInput resource : resources) {
-      changes.add(buildAddOwnersProposal(owners, UrnUtils.getUrn(resource.getResourceUrn()), actor, entityService));
+      changes.add(buildAddOwnersProposal(owners, UrnUtils.getUrn(resource.getResourceUrn()), entityService));
     }
     EntityUtils.ingestChangeProposals(changes, entityService, actor, false);
   }
@@ -69,7 +69,7 @@ public static void removeOwnersFromResources(
   }
 
 
-  private static MetadataChangeProposal buildAddOwnersProposal(List<OwnerInput> owners, Urn resourceUrn, Urn actor, EntityService entityService) {
+  static MetadataChangeProposal buildAddOwnersProposal(List<OwnerInput> owners, Urn resourceUrn, EntityService entityService) {
     Ownership ownershipAspect = (Ownership) EntityUtils.getAspectFromEntity(
         resourceUrn.toString(),
         Constants.OWNERSHIP_ASPECT_NAME, entityService,
@@ -181,18 +181,13 @@ public static boolean isAuthorizedToUpdateOwners(@Nonnull QueryContext context,
         orPrivilegeGroups);
   }
 
-  public static Boolean validateAddInput(
+  public static Boolean validateAddOwnerInput(
       List<OwnerInput> owners,
       Urn resourceUrn,
       EntityService entityService
   ) {
     for (OwnerInput owner : owners) {
-      boolean result = validateAddInput(
-          UrnUtils.getUrn(owner.getOwnerUrn()),
-          owner.getOwnershipTypeUrn(),
-          owner.getOwnerEntityType(),
-          resourceUrn,
-          entityService);
+      boolean result = validateAddOwnerInput(owner, resourceUrn, entityService);
       if (!result) {
         return false;
       }
@@ -200,44 +195,29 @@ public static Boolean validateAddInput(
     return true;
   }
 
-  public static Boolean validateAddInput(
-      Urn ownerUrn,
-      String ownershipEntityUrn,
-      OwnerEntityType ownerEntityType,
+  public static Boolean validateAddOwnerInput(
+      OwnerInput owner,
       Urn resourceUrn,
       EntityService entityService
   ) {
 
-    if (OwnerEntityType.CORP_GROUP.equals(ownerEntityType) && !Constants.CORP_GROUP_ENTITY_NAME.equals(ownerUrn.getEntityType())) {
-      throw new IllegalArgumentException(String.format("Failed to change ownership for resource %s. Expected a corp group urn.", resourceUrn));
-    }
-
-    if (OwnerEntityType.CORP_USER.equals(ownerEntityType) && !Constants.CORP_USER_ENTITY_NAME.equals(ownerUrn.getEntityType())) {
-      throw new IllegalArgumentException(String.format("Failed to change ownership for resource %s. Expected a corp user urn.", resourceUrn));
-    }
-
     if (!entityService.exists(resourceUrn)) {
       throw new IllegalArgumentException(String.format("Failed to change ownership for resource %s. Resource does not exist.", resourceUrn));
     }
 
-    if (!entityService.exists(ownerUrn)) {
-      throw new IllegalArgumentException(String.format("Failed to change ownership for resource %s. Owner %s does not exist.", resourceUrn, ownerUrn));
-    }
-
-    if (ownershipEntityUrn != null && !entityService.exists(UrnUtils.getUrn(ownershipEntityUrn))) {
-      throw new IllegalArgumentException(String.format("Failed to change ownership type for resource %s. Ownership Type "
-          + "%s does not exist.", resourceUrn, ownershipEntityUrn));
-    }
+    validateOwner(owner, entityService);
 
     return true;
   }
 
   public static void validateOwner(
-      Urn ownerUrn,
-      OwnerEntityType ownerEntityType,
-      Urn ownershipEntityUrn,
+      OwnerInput owner,
       EntityService entityService
   ) {
+
+    OwnerEntityType ownerEntityType = owner.getOwnerEntityType();
+    Urn ownerUrn = UrnUtils.getUrn(owner.getOwnerUrn());
+
     if (OwnerEntityType.CORP_GROUP.equals(ownerEntityType) && !Constants.CORP_GROUP_ENTITY_NAME.equals(ownerUrn.getEntityType())) {
       throw new IllegalArgumentException(
           String.format("Failed to change ownership for resource(s). Expected a corp group urn, found %s", ownerUrn));
@@ -252,9 +232,14 @@ public static void validateOwner(
       throw new IllegalArgumentException(String.format("Failed to change ownership for resource(s). Owner with urn %s does not exist.", ownerUrn));
     }
 
-    if (!entityService.exists(ownershipEntityUrn)) {
-      throw new IllegalArgumentException(String.format("Failed to change ownership for resource(s). Ownership type with "
-          + "urn %s does not exist.", ownershipEntityUrn));
+    if (owner.getOwnershipTypeUrn() != null && !entityService.exists(UrnUtils.getUrn(owner.getOwnershipTypeUrn()))) {
+      throw new IllegalArgumentException(String.format("Failed to change ownership for resource(s). Custom Ownership type with "
+          + "urn %s does not exist.", owner.getOwnershipTypeUrn()));
+    }
+
+    if (owner.getType() == null && owner.getOwnershipTypeUrn() == null) {
+      throw new IllegalArgumentException("Failed to change ownership for resource(s). Expected either "
+          + "type or ownershipTypeUrn to be specified.");
     }
   }
 
@@ -269,11 +254,11 @@ public static Boolean validateRemoveInput(
   }
 
   public static void addCreatorAsOwner(
-    QueryContext context,
-    String urn,
-    OwnerEntityType ownerEntityType,
-    OwnershipType ownershipType,
-    EntityService entityService) {
+      QueryContext context,
+      String urn,
+      OwnerEntityType ownerEntityType,
+      OwnershipType ownershipType,
+      EntityService entityService) {
     try {
       Urn actorUrn = CorpuserUrn.createFromString(context.getActorUrn());
       String ownershipTypeUrn = mapOwnershipTypeToEntity(ownershipType.name());
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java
index efc0c5dfcf36d..329d71ec125db 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java
@@ -2,6 +2,11 @@
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.AuditStamp;
+import com.linkedin.common.Owner;
+import com.linkedin.common.OwnerArray;
+import com.linkedin.common.Ownership;
+import com.linkedin.common.OwnershipSource;
+import com.linkedin.common.OwnershipSourceType;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -28,6 +33,7 @@ public class AddOwnersResolverTest {
   private static final String TEST_ENTITY_URN = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)";
   private static final String TEST_OWNER_1_URN = "urn:li:corpuser:test-id-1";
   private static final String TEST_OWNER_2_URN = "urn:li:corpuser:test-id-2";
+  private static final String TEST_OWNER_3_URN = "urn:li:corpGroup:test-id-3";
 
   @Test
   public void testGetSuccessNoExistingOwners() throws Exception {
@@ -75,33 +81,41 @@ public void testGetSuccessNoExistingOwners() throws Exception {
   }
 
   @Test
-  public void testGetSuccessExistingOwners() throws Exception {
+  public void testGetSuccessExistingOwnerNewType() throws Exception {
     EntityService mockService = getMockEntityService();
 
+    com.linkedin.common.Ownership oldOwnership = new Ownership().setOwners(new OwnerArray(
+            ImmutableList.of(new Owner()
+                    .setOwner(UrnUtils.getUrn(TEST_OWNER_1_URN))
+                    .setType(com.linkedin.common.OwnershipType.NONE)
+                    .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL))
+            )));
+
     Mockito.when(mockService.getAspect(
-        Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)),
-        Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME),
-        Mockito.eq(0L)))
-        .thenReturn(null);
+                    Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)),
+                    Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME),
+                    Mockito.eq(0L)))
+            .thenReturn(oldOwnership);
 
     Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true);
     Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_1_URN))).thenReturn(true);
-    Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_2_URN))).thenReturn(true);
 
     Mockito.when(mockService.exists(Urn.createFromString(
-            OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name()))))
-        .thenReturn(true);
+                    OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name()))))
+            .thenReturn(true);
 
     AddOwnersResolver resolver = new AddOwnersResolver(mockService);
 
     // Execute resolver
     QueryContext mockContext = getMockAllowContext();
     DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+
     AddOwnersInput input = new AddOwnersInput(ImmutableList.of(
-        new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER,
-            OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name())),
-        new OwnerInput(TEST_OWNER_2_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER,
-            OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))
+            OwnerInput.builder()
+                    .setOwnerUrn(TEST_OWNER_1_URN)
+                    .setOwnershipTypeUrn(OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))
+                    .setOwnerEntityType(OwnerEntityType.CORP_USER)
+                    .build()
     ), TEST_ENTITY_URN);
     Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input);
     Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
@@ -111,11 +125,126 @@ public void testGetSuccessExistingOwners() throws Exception {
     verifyIngestProposal(mockService, 1);
 
     Mockito.verify(mockService, Mockito.times(1)).exists(
-        Mockito.eq(Urn.createFromString(TEST_OWNER_1_URN))
+            Mockito.eq(Urn.createFromString(TEST_OWNER_1_URN))
     );
+  }
+
+  @Test
+  public void testGetSuccessDeprecatedTypeToOwnershipType() throws Exception {
+    EntityService mockService = getMockEntityService();
+
+    com.linkedin.common.Ownership oldOwnership = new Ownership().setOwners(new OwnerArray(
+            ImmutableList.of(new Owner()
+                    .setOwner(UrnUtils.getUrn(TEST_OWNER_1_URN))
+                    .setType(com.linkedin.common.OwnershipType.TECHNICAL_OWNER)
+                    .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL))
+            )));
+
+    Mockito.when(mockService.getAspect(
+                    Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)),
+                    Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME),
+                    Mockito.eq(0L)))
+            .thenReturn(oldOwnership);
+
+    Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true);
+    Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_1_URN))).thenReturn(true);
+
+    Mockito.when(mockService.exists(Urn.createFromString(
+                    OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name()))))
+            .thenReturn(true);
+
+    AddOwnersResolver resolver = new AddOwnersResolver(mockService);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+
+    AddOwnersInput input = new AddOwnersInput(ImmutableList.of(OwnerInput.builder()
+                    .setOwnerUrn(TEST_OWNER_1_URN)
+                    .setOwnershipTypeUrn(OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))
+                    .setOwnerEntityType(OwnerEntityType.CORP_USER)
+                    .build()
+    ), TEST_ENTITY_URN);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+    assertTrue(resolver.get(mockEnv).get());
+
+    // Unable to easily validate exact payload due to the injected timestamp
+    verifyIngestProposal(mockService, 1);
 
     Mockito.verify(mockService, Mockito.times(1)).exists(
-        Mockito.eq(Urn.createFromString(TEST_OWNER_2_URN))
+            Mockito.eq(Urn.createFromString(TEST_OWNER_1_URN))
+    );
+  }
+
+  @Test
+  public void testGetSuccessMultipleOwnerTypes() throws Exception {
+    EntityService mockService = getMockEntityService();
+
+    com.linkedin.common.Ownership oldOwnership = new Ownership().setOwners(new OwnerArray(
+            ImmutableList.of(new Owner()
+                    .setOwner(UrnUtils.getUrn(TEST_OWNER_1_URN))
+                    .setType(com.linkedin.common.OwnershipType.NONE)
+                    .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL))
+            )));
+
+    Mockito.when(mockService.getAspect(
+                    Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)),
+                    Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME),
+                    Mockito.eq(0L)))
+            .thenReturn(oldOwnership);
+
+    Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true);
+    Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_1_URN))).thenReturn(true);
+    Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_2_URN))).thenReturn(true);
+    Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_3_URN))).thenReturn(true);
+
+    Mockito.when(mockService.exists(Urn.createFromString(
+                    OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name()))))
+            .thenReturn(true);
+    Mockito.when(mockService.exists(Urn.createFromString(
+                    OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))))
+            .thenReturn(true);
+
+    AddOwnersResolver resolver = new AddOwnersResolver(mockService);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+
+    AddOwnersInput input = new AddOwnersInput(ImmutableList.of(OwnerInput.builder()
+                    .setOwnerUrn(TEST_OWNER_1_URN)
+                    .setOwnershipTypeUrn(OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))
+                    .setOwnerEntityType(OwnerEntityType.CORP_USER)
+                    .build(),
+            OwnerInput.builder()
+                    .setOwnerUrn(TEST_OWNER_2_URN)
+                    .setOwnershipTypeUrn(OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.BUSINESS_OWNER.name()))
+                    .setOwnerEntityType(OwnerEntityType.CORP_USER)
+                    .build(),
+            OwnerInput.builder()
+                    .setOwnerUrn(TEST_OWNER_3_URN)
+                    .setOwnershipTypeUrn(OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))
+                    .setOwnerEntityType(OwnerEntityType.CORP_GROUP)
+                    .build()
+    ), TEST_ENTITY_URN);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+    assertTrue(resolver.get(mockEnv).get());
+
+    // Unable to easily validate exact payload due to the injected timestamp
+    verifyIngestProposal(mockService, 1);
+
+    Mockito.verify(mockService, Mockito.times(1)).exists(
+            Mockito.eq(Urn.createFromString(TEST_OWNER_1_URN))
+    );
+
+    Mockito.verify(mockService, Mockito.times(1)).exists(
+            Mockito.eq(Urn.createFromString(TEST_OWNER_2_URN))
+    );
+
+    Mockito.verify(mockService, Mockito.times(1)).exists(
+            Mockito.eq(Urn.createFromString(TEST_OWNER_3_URN))
     );
   }
 

From 75b36c41ee4fd74891b1bfe37885b4cd840e2906 Mon Sep 17 00:00:00 2001
From: Ellie O'Neil <110510035+eboneil@users.noreply.github.com>
Date: Thu, 19 Oct 2023 08:32:24 -0700
Subject: [PATCH 072/259] docs(protobuf) Update messaging around nesting
 messages (#9048)

---
 metadata-integration/java/datahub-protobuf/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-integration/java/datahub-protobuf/README.md b/metadata-integration/java/datahub-protobuf/README.md
index daea8d438679c..29b82aa3e68f5 100644
--- a/metadata-integration/java/datahub-protobuf/README.md
+++ b/metadata-integration/java/datahub-protobuf/README.md
@@ -1,6 +1,6 @@
 # Protobuf Schemas
 
-The `datahub-protobuf` module is designed to be used with the Java Emitter, the input is a compiled protobuf binary `*.protoc` files and optionally the corresponding `*.proto` source code. In addition, you can supply the root message in cases where a single protobuf source file includes multiple non-nested messages.
+The `datahub-protobuf` module is designed to be used with the Java Emitter, the input is a compiled protobuf binary `*.protoc` files and optionally the corresponding `*.proto` source code. You can supply a file with multiple nested messages to be processed. If you have a file with multiple non-nested messages, you will need to separate them out into different files or supply the root message, as otherwise we will only process the first one.
 
 ## Supported Features
 

From b1abd38a6b4aef3da0c50ecd23612cae7e3c5d28 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Thu, 19 Oct 2023 15:33:54 -0400
Subject: [PATCH 073/259] refactor(): Use data-testids for glossary_navigation
 and dataset_ownership tests (#9033)

---
 .../CreateGlossaryEntityModal.tsx             |  7 ++-
 .../shared/EntityDropdown/EntityDropdown.tsx  |  3 +-
 .../MoveGlossaryEntityModal.tsx               |  5 +-
 .../Ownership/sidebar/SidebarOwnerSection.tsx |  6 +-
 .../src/app/glossary/BusinessGlossaryPage.tsx |  4 +-
 .../src/app/glossary/GlossarySidebar.tsx      |  2 +-
 .../e2e/glossary/glossary_navigation.js       | 55 +++++++++++--------
 .../cypress/e2e/lineage/lineage_graph.js      |  2 -
 .../e2e/mutations/dataset_ownership.js        |  2 +-
 .../tests/cypress/cypress/support/commands.js |  1 +
 10 files changed, 54 insertions(+), 33 deletions(-)

diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx
index d48ead2f5863e..9788d36af2c65 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx
@@ -112,7 +112,11 @@ function CreateGlossaryEntityModal(props: Props) {
                     <Button onClick={onClose} type="text">
                         Cancel
                     </Button>
-                    <Button onClick={createGlossaryEntity} disabled={createButtonDisabled}>
+                    <Button
+                        onClick={createGlossaryEntity}
+                        disabled={createButtonDisabled}
+                        data-testid="glossary-entity-modal-create-button"
+                    >
                         Create
                     </Button>
                 </>
@@ -130,6 +134,7 @@ function CreateGlossaryEntityModal(props: Props) {
             >
                 <Form.Item label={<Typography.Text strong>Name</Typography.Text>}>
                     <StyledItem
+                        data-testid="create-glossary-entity-modal-name"
                         name="name"
                         rules={[
                             {
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx
index bfb7ff7e540c4..5d4f9d9f875cf 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx
@@ -199,11 +199,12 @@ function EntityDropdown(props: Props) {
                         )}
                         {!isDomainMoveHidden && menuItems.has(EntityMenuItems.MOVE) && (
                             <StyledMenuItem
+                                data-testid="entity-menu-move-button"
                                 key="4"
                                 disabled={isMoveDisabled(entityType, entityData, me.platformPrivileges)}
                                 onClick={() => setIsMoveModalVisible(true)}
                             >
-                                <MenuItem data-testid="entity-menu-move-button">
+                                <MenuItem>
                                     <FolderOpenOutlined /> &nbsp;Move
                                 </MenuItem>
                             </StyledMenuItem>
diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx
index 5352825708776..37a625f58100b 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx
@@ -64,6 +64,7 @@ function MoveGlossaryEntityModal(props: Props) {
 
     return (
         <Modal
+            data-testid="move-glossary-entity-modal"
             title="Move"
             visible
             onCancel={onClose}
@@ -72,7 +73,9 @@ function MoveGlossaryEntityModal(props: Props) {
                     <Button onClick={onClose} type="text">
                         Cancel
                     </Button>
-                    <Button onClick={moveGlossaryEntity}>Move</Button>
+                    <Button onClick={moveGlossaryEntity} data-testid="glossary-entity-modal-move-button">
+                        Move
+                    </Button>
                 </>
             }
         >
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection.tsx
index 57743d0531afe..aa9a337d4ba44 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection.tsx
@@ -70,7 +70,11 @@ export const SidebarOwnerSection = ({ properties, readOnly }: Props) => {
                     </Typography.Paragraph>
                 )}
                 {!readOnly && (
-                    <Button type={ownersEmpty ? 'default' : 'text'} onClick={() => setShowAddModal(true)}>
+                    <Button
+                        type={ownersEmpty ? 'default' : 'text'}
+                        onClick={() => setShowAddModal(true)}
+                        data-testid="add-owners-button"
+                    >
                         <PlusOutlined /> Add Owners
                     </Button>
                 )}
diff --git a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
index 11f54cb5078e6..a5262265fd23d 100644
--- a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
+++ b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx
@@ -92,11 +92,12 @@ function BusinessGlossaryPage() {
                 {(termsError || nodesError) && (
                     <Message type="error" content="Failed to load glossary! An unexpected error occurred." />
                 )}
-                <MainContentWrapper>
+                <MainContentWrapper data-testid="glossary-entities-list">
                     <HeaderWrapper>
                         <Typography.Title level={3}>Business Glossary</Typography.Title>
                         <div>
                             <Button
+                                data-testid="add-term-button"
                                 id={BUSINESS_GLOSSARY_CREATE_TERM_ID}
                                 disabled={!canManageGlossaries}
                                 type="text"
@@ -105,6 +106,7 @@ function BusinessGlossaryPage() {
                                 <PlusOutlined /> Add Term
                             </Button>
                             <Button
+                                data-testid="add-term-group-button"
                                 id={BUSINESS_GLOSSARY_CREATE_TERM_GROUP_ID}
                                 disabled={!canManageGlossaries}
                                 type="text"
diff --git a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx
index 2d620fb06df38..4126c8f2bb53f 100644
--- a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx
+++ b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx
@@ -12,7 +12,7 @@ export default function GlossarySidebar() {
 
     return (
         <>
-            <SidebarWrapper width={browserWidth}>
+            <SidebarWrapper width={browserWidth} data-testid="glossary-browser-sidebar">
                 <GlossarySearch />
                 <GlossaryBrowser openToEntity />
             </SidebarWrapper>
diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
index e0d2bf240d74d..f52e4d3984a88 100644
--- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
+++ b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
@@ -4,42 +4,48 @@ const glossaryParentGroup = "Cypress";
 
 describe("glossary sidebar navigation test", () => {
     it("create term and term parent group, move and delete term group", () => {
-        //create a new term group and term, move term to the group
+
+        // Create a new term group and term, move term to the group
         cy.loginWithCredentials();
         cy.goToGlossaryList();
-        cy.clickOptionWithText("Add Term Group");
+        cy.clickOptionWithTestId("add-term-group-button");
         cy.waitTextVisible("Create Term Group");
-        cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(glossaryTermGroup);
-        cy.get(".ant-modal-footer > button:last-child").click();
-        cy.get('*[class^="GlossaryBrowser"]').contains(glossaryTermGroup).should("be.visible");
-        cy.clickOptionWithText("Add Term");
+        cy.enterTextInTestId("create-glossary-entity-modal-name", glossaryTermGroup);
+        cy.clickOptionWithTestId("glossary-entity-modal-create-button");
+        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryTermGroup).should("be.visible");
+        cy.clickOptionWithTestId("add-term-button"); 
+        cy.waitTextVisible("Created Term Group!");
         cy.waitTextVisible("Create Glossary Term");
-        cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(glossaryTerm);
-        cy.get(".ant-modal-footer > button:last-child").click();
-        cy.get('*[class^="GlossaryBrowser"]').contains(glossaryTerm).click();
-        cy.waitTextVisible("No documentation yet");
+        cy.enterTextInTestId("create-glossary-entity-modal-name", glossaryTerm);
+        cy.clickOptionWithTestId("glossary-entity-modal-create-button");
+        cy.waitTextVisible("Created Glossary Term!");
+        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryTerm).click().wait(3000);
         cy.openThreeDotDropdown();
-        cy.clickOptionWithText("Move");
-        cy.get('[role="dialog"]').contains(glossaryTermGroup).click({force: true});
-        cy.get('[role="dialog"]').contains(glossaryTermGroup).should("be.visible");
-        cy.get("button").contains("Move").click();
+        cy.clickOptionWithTestId("entity-menu-move-button")
+        cy.get('[data-testid="move-glossary-entity-modal"]').contains(glossaryTermGroup).click({force: true});
+        cy.get('[data-testid="move-glossary-entity-modal"]').contains(glossaryTermGroup).should("be.visible");
+        cy.clickOptionWithTestId("glossary-entity-modal-move-button");
         cy.waitTextVisible("Moved Glossary Term!");
-        //ensure the new term is under the parent term group in the navigation sidebar
-        cy.get('*[class^="GlossaryBrowser"]').contains(glossaryTermGroup).click();
+
+        // Ensure the new term is under the parent term group in the navigation sidebar
+        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryTermGroup).click();
         cy.get('*[class^="GlossaryEntitiesList"]').contains(glossaryTerm).should("be.visible");
-        //move a term group from the root level to be under a parent term group
+
+        // Move a term group from the root level to be under a parent term group
         cy.goToGlossaryList();
         cy.clickOptionWithText(glossaryTermGroup);
         cy.openThreeDotDropdown();
         cy.clickOptionWithText("Move");
-        cy.get('[role="dialog"]').contains(glossaryParentGroup).click({force: true});
-        cy.get('[role="dialog"]').contains(glossaryParentGroup).should("be.visible");
-        cy.get("button").contains("Move").click();
+        cy.get('[data-testid="move-glossary-entity-modal"]').contains(glossaryParentGroup).click({force: true});
+        cy.get('[data-testid="move-glossary-entity-modal"]').contains(glossaryParentGroup).should("be.visible");
+        cy.clickOptionWithTestId("glossary-entity-modal-move-button");
         cy.waitTextVisible("Moved Term Group!");
-        //ensure it is no longer on the sidebar navigator at the top level but shows up under the new parent
-        cy.get('*[class^="GlossaryBrowser"]').contains(glossaryParentGroup).click();
+
+        // Ensure it is no longer on the sidebar navigator at the top level but shows up under the new parent
+        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryParentGroup).click();
         cy.get('*[class^="GlossaryEntitiesList"]').contains(glossaryTermGroup).should("be.visible");
-        //delete a term group
+
+        // Delete a term group
         cy.goToGlossaryList();
         cy.clickOptionWithText(glossaryParentGroup);
         cy.clickOptionWithText(glossaryTermGroup);
@@ -50,7 +56,8 @@ describe("glossary sidebar navigation test", () => {
         cy.clickOptionWithText(glossaryTermGroup).wait(3000);
         cy.deleteFromDropdown();
         cy.waitTextVisible("Deleted Term Group!");
-        //ensure it is no longer in the sidebar navigator
+
+        // Ensure it is no longer in the sidebar navigator
         cy.ensureTextNotPresent(glossaryTerm);
         cy.ensureTextNotPresent(glossaryTermGroup);
     });
diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js
index 9e035f7f89772..85db210649c27 100644
--- a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_graph.js
@@ -5,8 +5,6 @@ const TASKS_ENTITY_TYPE = 'tasks';
 const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)';
 const JAN_1_2021_TIMESTAMP = 1609553357755;
 const JAN_1_2022_TIMESTAMP = 1641089357755;
-const TIMESTAMP_MILLIS_EIGHT_DAYS_AGO = getTimestampMillisNumDaysAgo(8);
-const TIMESTAMP_MILLIS_ONE_DAY_AGO = getTimestampMillisNumDaysAgo(1);
 const TIMESTAMP_MILLIS_14_DAYS_AGO = getTimestampMillisNumDaysAgo(14);
 const TIMESTAMP_MILLIS_7_DAYS_AGO = getTimestampMillisNumDaysAgo(7);
 const TIMESTAMP_MILLIS_NOW = getTimestampMillisNumDaysAgo(0);
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js
index fcc0566f3f6ce..99ad9a68d35e1 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js
@@ -5,7 +5,7 @@ const password = "Example password";
 const group_name = `Test group ${test_id}`;
 
 const addOwner = (owner, type, elementId) => {
-    cy.clickOptionWithText("Add Owners");
+    cy.clickOptionWithTestId("add-owners-button");
     cy.contains("Search for users or groups...").click({ force: true });
     cy.focused().type(owner);
     cy.clickOptionWithText(owner);
diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js
index 64bc1253fc383..5e3664f944edf 100644
--- a/smoke-test/tests/cypress/cypress/support/commands.js
+++ b/smoke-test/tests/cypress/cypress/support/commands.js
@@ -66,6 +66,7 @@ Cypress.Commands.add("logout", () => {
 Cypress.Commands.add("goToGlossaryList", () => {
   cy.visit("/glossary");
   cy.waitTextVisible("Glossary");
+  cy.wait(3000);
 });
 
 Cypress.Commands.add("goToDomainList", () => {

From 35d7770462e74cd95cf80b6df986fec715b7ce2c Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Fri, 20 Oct 2023 01:40:23 -0400
Subject: [PATCH 074/259] test(ingest/delta-lake): Fix integration tests
 (#9056)

---
 .../delta_lake_minio_mces_golden.json         |  48 ++--
 .../local/golden_mces_allow_table.json        | 261 ++++++++++++------
 .../local/golden_mces_inner_table.json        | 261 ++++++++++++------
 .../local/golden_mces_relative_path.json      |  62 +++--
 .../local/golden_mces_single_table.json       | 102 ++++---
 5 files changed, 480 insertions(+), 254 deletions(-)

diff --git a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json
index 52e92d27549f0..ed65d74037796 100644
--- a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json
+++ b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json
@@ -136,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -156,7 +157,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -171,7 +173,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -186,7 +189,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -203,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -218,7 +223,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -238,7 +244,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -253,7 +260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -268,7 +276,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -285,7 +294,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -300,7 +310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -320,7 +331,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -335,7 +347,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -355,14 +368,16 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "version": "0"
             },
             "lastUpdatedTimestamp": 1655664815399
         }
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -386,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json
index 4dcdf71ce0095..6ec6eb2809a10 100644
--- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json
+++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json
@@ -94,7 +94,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -115,7 +116,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -130,7 +132,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +167,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -183,7 +188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,7 +210,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -219,7 +226,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -235,7 +243,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -252,7 +261,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -267,7 +277,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -291,7 +302,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -312,7 +324,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -327,7 +340,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -343,7 +357,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -360,7 +375,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -375,7 +391,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -403,7 +420,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -424,7 +442,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -439,7 +458,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -455,7 +475,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +493,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +509,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +542,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -540,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -555,7 +580,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -571,7 +597,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -588,7 +615,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -603,7 +631,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -639,7 +668,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -654,7 +684,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -674,14 +705,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831476907
+            "lastUpdatedTimestamp": 1655831477768
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -702,14 +736,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831477701
+            "lastUpdatedTimestamp": 1655831477745
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -730,14 +766,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831477726
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -758,14 +796,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831477745
+            "lastUpdatedTimestamp": 1655831477701
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -786,14 +826,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831477768
+            "lastUpdatedTimestamp": 1655831476907
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -833,7 +874,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -973,7 +1015,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -988,7 +1031,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1008,14 +1052,16 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "version": "0"
             },
             "lastUpdatedTimestamp": 1655664815399
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1055,7 +1101,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1199,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1167,7 +1215,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,14 +1236,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831649166
+            "lastUpdatedTimestamp": 1655831649788
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1215,14 +1267,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831649715
+            "lastUpdatedTimestamp": 1655831649754
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1243,14 +1297,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831649731
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1271,14 +1327,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831649754
+            "lastUpdatedTimestamp": 1655831649715
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1299,14 +1357,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831649788
+            "lastUpdatedTimestamp": 1655831649166
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1346,7 +1405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1444,7 +1504,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1465,7 +1526,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1480,7 +1542,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1496,7 +1559,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1513,7 +1577,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1528,7 +1593,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1568,7 +1634,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1583,7 +1650,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1603,14 +1671,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831865396
+            "lastUpdatedTimestamp": 1655831866541
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1631,14 +1702,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831866337
+            "lastUpdatedTimestamp": 1655831866447
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1659,14 +1732,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831866398
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1687,14 +1762,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831866447
+            "lastUpdatedTimestamp": 1655831866337
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1715,14 +1792,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831866541
+            "lastUpdatedTimestamp": 1655831865396
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1766,7 +1844,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "allow_table.json"
+        "runId": "allow_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json
index 901e4c1262d3f..715beebfe22fb 100644
--- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json
+++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json
@@ -94,7 +94,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +115,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -144,7 +147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +165,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -176,7 +181,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -196,7 +202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -211,7 +218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -226,7 +234,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -243,7 +252,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -258,7 +268,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -278,7 +289,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -298,7 +310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +326,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +342,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -345,7 +360,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -360,7 +376,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -404,7 +422,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -419,7 +438,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -434,7 +454,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +472,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -466,7 +488,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -494,7 +517,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -514,7 +538,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -529,7 +554,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -544,7 +570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -561,7 +588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -576,7 +604,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -608,7 +637,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -623,7 +653,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -643,14 +674,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831476907
+            "lastUpdatedTimestamp": 1655831477768
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -671,14 +705,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831477701
+            "lastUpdatedTimestamp": 1655831477745
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -699,14 +735,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831477726
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -727,14 +765,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831477745
+            "lastUpdatedTimestamp": 1655831477701
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -755,14 +795,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831477768
+            "lastUpdatedTimestamp": 1655831476907
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -798,7 +839,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -938,7 +980,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -953,7 +996,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -973,14 +1017,16 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "version": "0"
             },
             "lastUpdatedTimestamp": 1655664815399
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1016,7 +1062,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1113,7 +1160,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1128,7 +1176,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1148,14 +1197,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831649166
+            "lastUpdatedTimestamp": 1655831649788
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1176,14 +1228,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831649715
+            "lastUpdatedTimestamp": 1655831649754
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1204,14 +1258,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831649731
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1232,14 +1288,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831649754
+            "lastUpdatedTimestamp": 1655831649715
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1260,14 +1318,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831649788
+            "lastUpdatedTimestamp": 1655831649166
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1303,7 +1362,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1401,7 +1461,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1421,7 +1482,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1436,7 +1498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1451,7 +1514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1468,7 +1532,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1483,7 +1548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1519,7 +1585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1534,7 +1601,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1554,14 +1622,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831865396
+            "lastUpdatedTimestamp": 1655831866541
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1582,14 +1653,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831866337
+            "lastUpdatedTimestamp": 1655831866447
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1610,14 +1683,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831866398
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1638,14 +1713,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831866447
+            "lastUpdatedTimestamp": 1655831866337
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1666,14 +1743,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831866541
+            "lastUpdatedTimestamp": 1655831865396
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1713,7 +1791,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "inner_table.json"
+        "runId": "inner_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json
index 18474e819334e..2076ec4096f68 100644
--- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json
+++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json
@@ -94,7 +94,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +115,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -144,7 +147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +165,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -176,7 +181,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -191,7 +197,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -211,14 +218,17 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "readVersion": "3",
+                "version": "4"
             },
-            "lastUpdatedTimestamp": 1655831476907
+            "lastUpdatedTimestamp": 1655831477768
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -239,14 +249,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "2",
+                "version": "3"
             },
-            "lastUpdatedTimestamp": 1655831477701
+            "lastUpdatedTimestamp": 1655831477745
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -267,14 +279,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "1"
+                "readVersion": "1",
+                "version": "2"
             },
             "lastUpdatedTimestamp": 1655831477726
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -295,14 +309,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "2"
+                "readVersion": "0",
+                "version": "1"
             },
-            "lastUpdatedTimestamp": 1655831477745
+            "lastUpdatedTimestamp": 1655831477701
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -323,14 +339,15 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "version": "0"
             },
-            "lastUpdatedTimestamp": 1655831477768
+            "lastUpdatedTimestamp": 1655831476907
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -350,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json
index bb47a077e878b..42e3b19612c2b 100644
--- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json
+++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json
@@ -93,7 +93,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -113,7 +114,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -128,7 +130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -143,7 +146,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -160,7 +164,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -175,7 +180,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -195,7 +201,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +217,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +233,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -242,7 +251,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -257,7 +267,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -277,7 +288,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -297,7 +309,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -312,7 +325,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -327,7 +341,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -344,7 +359,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -359,7 +375,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -383,7 +400,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -403,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -418,7 +437,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -433,7 +453,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -450,7 +471,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -465,7 +487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -493,7 +516,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -513,7 +537,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -528,7 +553,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -543,7 +569,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -560,7 +587,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +603,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -607,7 +636,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +652,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -643,14 +674,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "readVersion": "3",
+                "version": "4"
             },
             "lastUpdatedTimestamp": 1655831477768
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -686,7 +719,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file

From 3360fa222db42d587a4ae8fbc046628424f51373 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Fri, 20 Oct 2023 16:25:06 -0400
Subject: [PATCH 075/259] test(): Ingestion source creation cypress test
 (#8850)

---
 .../app/ingest/source/IngestionSourceList.tsx |  7 +-
 .../source/IngestionSourceTableColumns.tsx    | 12 +++-
 .../source/builder/CreateScheduleStep.tsx     |  6 +-
 .../ingest/source/builder/NameSourceStep.tsx  |  1 +
 .../ingest/source/builder/RecipeBuilder.tsx   | 18 ++++-
 .../cypress/e2e/mutations/ingestion_source.js | 68 +++++++++++++++++++
 6 files changed, 105 insertions(+), 7 deletions(-)
 create mode 100644 smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
index 6c91a0f6f3f8f..0e341a5ff3a79 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -358,7 +358,12 @@ export const IngestionSourceList = () => {
             <SourceContainer>
                 <TabToolbar>
                     <div>
-                        <Button id={INGESTION_CREATE_SOURCE_ID} type="text" onClick={() => setIsBuildingSource(true)}>
+                        <Button
+                            id={INGESTION_CREATE_SOURCE_ID}
+                            type="text"
+                            onClick={() => setIsBuildingSource(true)}
+                            data-testid="create-ingestion-source-button"
+                        >
                             <PlusOutlined /> Create new source
                         </Button>
                         <Button id={INGESTION_REFRESH_SOURCES_ID} type="text" onClick={onRefresh}>
diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
index 2b8ecb860ed0c..c47d08d5b6003 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
@@ -119,7 +119,11 @@ export function LastStatusColumn({ status, record, setFocusExecutionUrn }: LastS
     return (
         <StatusContainer>
             {Icon && <Icon style={{ color, fontSize: 14 }} />}
-            <StatusButton type="link" onClick={() => setFocusExecutionUrn(record.lastExecUrn)}>
+            <StatusButton
+                data-testid="ingestion-source-table-status"
+                type="link"
+                onClick={() => setFocusExecutionUrn(record.lastExecUrn)}
+            >
                 <Typography.Text strong style={{ color, marginLeft: 8 }}>
                     {text || 'Pending...'}
                 </Typography.Text>
@@ -159,7 +163,11 @@ export function ActionsColumn({
                 </Tooltip>
             )}
             {!record.cliIngestion && (
-                <Button style={{ marginRight: 16 }} onClick={() => onEdit(record.urn)}>
+                <Button
+                    data-testid="ingestion-source-table-edit-button"
+                    style={{ marginRight: 16 }}
+                    onClick={() => onEdit(record.urn)}
+                >
                     EDIT
                 </Button>
             )}
diff --git a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx
index dba9b25e14e99..7a14b6a794189 100644
--- a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx
@@ -167,7 +167,11 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
             <ControlsContainer>
                 <Button onClick={prev}>Previous</Button>
                 <div>
-                    <Button disabled={!interval || interval.length === 0 || cronAsText.error} onClick={onClickNext}>
+                    <Button
+                        data-testid="ingestion-schedule-next-button"
+                        disabled={!interval || interval.length === 0 || cronAsText.error}
+                        onClick={onClickNext}
+                    >
                         Next
                     </Button>
                 </div>
diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
index 913f8253ece5a..992ebff643c31 100644
--- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -123,6 +123,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                 <Button onClick={prev}>Previous</Button>
                 <div>
                     <SaveButton
+                        data-testid="ingestion-source-save-button"
                         disabled={!(state.name !== undefined && state.name.length > 0)}
                         onClick={() => onClickCreate(false)}
                     >
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx
index 4ddeb7b492595..bee9b04cee100 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx
@@ -86,10 +86,20 @@ function RecipeBuilder(props: Props) {
                     {sourceConfigs?.displayName} Recipe
                 </Title>
                 <ButtonsWrapper>
-                    <StyledButton type="text" isSelected={isViewingForm} onClick={() => switchViews(true)}>
+                    <StyledButton
+                        type="text"
+                        isSelected={isViewingForm}
+                        onClick={() => switchViews(true)}
+                        data-testid="recipe-builder-form-button"
+                    >
                         <FormOutlined /> Form
                     </StyledButton>
-                    <StyledButton type="text" isSelected={!isViewingForm} onClick={() => switchViews(false)}>
+                    <StyledButton
+                        type="text"
+                        isSelected={!isViewingForm}
+                        onClick={() => switchViews(false)}
+                        data-testid="recipe-builder-yaml-button"
+                    >
                         <CodeOutlined /> YAML
                     </StyledButton>
                 </ButtonsWrapper>
@@ -114,7 +124,9 @@ function RecipeBuilder(props: Props) {
                         <Button disabled={isEditing} onClick={goToPrevious}>
                             Previous
                         </Button>
-                        <Button onClick={onClickNext}>Next</Button>
+                        <Button data-testid="recipe-builder-next-button" onClick={onClickNext}>
+                            Next
+                        </Button>
                     </ControlsContainer>
                 </>
             )}
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js b/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js
new file mode 100644
index 0000000000000..6c5dd77810644
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js
@@ -0,0 +1,68 @@
+
+const number = Math.floor(Math.random() * 100000);
+const accound_id = `account${number}`;
+const warehouse_id = `warehouse${number}`;
+const username = `user${number}`;
+const password = `password${number}`;
+const role = `role${number}`;
+const ingestion_source_name = `ingestion source ${number}`;
+
+describe("ingestion source creation flow", () => {
+    it("create a ingestion source using ui, verify ingestion source details saved correctly, remove ingestion source", () => {
+      // Go to ingestion page, create a snowflake source
+      cy.loginWithCredentials();
+      cy.goToIngestionPage();
+      cy.clickOptionWithTestId("create-ingestion-source-button");
+      cy.clickOptionWithText("Snowflake");
+      cy.waitTextVisible("Snowflake Recipe");
+      cy.get("#account_id").type(accound_id);
+      cy.get("#warehouse").type(warehouse_id);
+      cy.get("#username").type(username);
+      cy.get("#password").type(password);
+      cy.focused().blur();
+      cy.get("#role").type(role);
+
+      // Verify yaml recipe is generated correctly
+      cy.clickOptionWithTestId("recipe-builder-yaml-button");
+      cy.waitTextVisible("account_id");
+      cy.waitTextVisible(accound_id);
+      cy.waitTextVisible(warehouse_id);
+      cy.waitTextVisible(username);
+      cy.waitTextVisible(password);
+      cy.waitTextVisible(role);
+
+      // Finish creating source
+      cy.clickOptionWithTestId("recipe-builder-next-button");
+      cy.waitTextVisible("Configure an Ingestion Schedule");
+      cy.clickOptionWithTestId("ingestion-schedule-next-button");
+      cy.waitTextVisible("Give this ingestion source a name."); 
+      cy.get('[data-testid="source-name-input"]').type(ingestion_source_name);
+      cy.clickOptionWithTestId("ingestion-source-save-button");
+      cy.waitTextVisible("Successfully created ingestion source!").wait(5000)
+      cy.waitTextVisible(ingestion_source_name);
+      cy.get('[data-testid="ingestion-source-table-status"]').contains("Pending...").should("be.visible");
+
+      // Verify ingestion source details are saved correctly
+      cy.get('[data-testid="ingestion-source-table-edit-button"]').first().click();
+      cy.waitTextVisible("Edit Ingestion Source");
+      cy.get("#account_id").should("have.value", accound_id);
+      cy.get("#warehouse").should("have.value", warehouse_id);
+      cy.get("#username").should("have.value", username);
+      cy.get("#password").should("have.value", password);
+      cy.get("#role").should("have.value", role);
+      cy.get("button").contains("Next").click();
+      cy.waitTextVisible("Configure an Ingestion Schedule");
+      cy.clickOptionWithTestId("ingestion-schedule-next-button");
+      cy.get('[data-testid="source-name-input"]').clear().type(ingestion_source_name + " EDITED");
+      cy.clickOptionWithTestId("ingestion-source-save-button");
+      cy.waitTextVisible("Successfully updated ingestion source!");
+      cy.waitTextVisible(ingestion_source_name + " EDITED");
+
+      // Remove ingestion source
+      cy.get('[data-testid="delete-button"]').first().click();
+      cy.waitTextVisible("Confirm Ingestion Source Removal");
+      cy.get("button").contains("Yes").click();
+      cy.waitTextVisible("Removed ingestion source.");
+      cy.ensureTextNotPresent(ingestion_source_name + " EDITED")
+    })
+});
\ No newline at end of file

From 2fea466d48c856f5c469af6f611990a200e5bece Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Fri, 20 Oct 2023 13:47:52 -0700
Subject: [PATCH 076/259] docs: fix lineage capability annotations (#8954)

---
 .../src/datahub/ingestion/source/aws/glue.py           |  1 +
 .../datahub/ingestion/source/bigquery_v2/bigquery.py   |  1 +
 .../src/datahub/ingestion/source/kafka_connect.py      |  1 +
 .../datahub/ingestion/source/looker/looker_source.py   |  6 +++++-
 .../src/datahub/ingestion/source/metabase.py           |  1 +
 .../src/datahub/ingestion/source/metadata/lineage.py   | 10 +++++++++-
 .../src/datahub/ingestion/source/mode.py               |  1 +
 .../src/datahub/ingestion/source/nifi.py               |  4 +++-
 .../src/datahub/ingestion/source/powerbi/powerbi.py    |  5 ++++-
 .../src/datahub/ingestion/source/sql_queries.py        | 10 +++++++++-
 .../src/datahub/ingestion/source/superset.py           |  1 +
 .../src/datahub/ingestion/source/tableau.py            |  6 +++++-
 12 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
index e5dff786b71d1..aa7e5aa352a3e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
@@ -221,6 +221,7 @@ def report_table_dropped(self, table: str) -> None:
     SourceCapability.DELETION_DETECTION,
     "Enabled by default when stateful ingestion is turned on.",
 )
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class GlueSource(StatefulIngestionSourceBase):
     """
     Note: if you also have files in S3 that you'd like to ingest, we recommend you use Glue's built-in data catalog. See [here](../../../../docs/generated/ingestion/sources/s3.md) for a quick guide on how to set up a crawler on Glue and ingest the outputs with DataHub.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index 552612f877b9a..692d8c4f81bb6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -153,6 +153,7 @@ def cleanup(config: BigQueryV2Config) -> None:
 )
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
 @capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration")
+@capability(SourceCapability.LINEAGE_FINE, "Optionally enabled via configuration")
 @capability(
     SourceCapability.USAGE_STATS,
     "Enabled by default, can be disabled via configuration `include_usage_statistics`",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py
index 5fae0ee5215a3..1a1e012e80633 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py
@@ -1096,6 +1096,7 @@ def transform_connector_config(
 @config_class(KafkaConnectSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class KafkaConnectSource(StatefulIngestionSourceBase):
     config: KafkaConnectSourceConfig
     report: KafkaConnectSourceReport
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
index 8297a0aa8efa7..a3df977582ca4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -103,6 +103,11 @@
 @capability(
     SourceCapability.OWNERSHIP, "Enabled by default, configured using `extract_owners`"
 )
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default, configured using `extract_column_level_lineage`",
+)
 @capability(
     SourceCapability.USAGE_STATS,
     "Enabled by default, configured using `extract_usage_history`",
@@ -1128,7 +1133,6 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
     def emit_independent_looks_mcp(
         self, dashboard_element: LookerDashboardElement
     ) -> Iterable[MetadataWorkUnit]:
-
         yield from auto_workunit(
             stream=self._make_chart_metadata_events(
                 dashboard_element=dashboard_element,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py
index fb4512893feb1..24145d60210ff 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py
@@ -80,6 +80,7 @@ def remove_trailing_slash(cls, v):
 @config_class(MetabaseConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
 class MetabaseSource(Source):
     """
     This plugin extracts Charts, dashboards, and associated metadata. This plugin is in beta and has only been tested
diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
index 1c0c809c16a60..f33c6e0edae3d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py
@@ -23,11 +23,17 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    Source,
+    SourceCapability,
+    SourceReport,
+)
 from datahub.ingestion.api.source_helpers import (
     auto_status_aspect,
     auto_workunit_reporter,
@@ -121,6 +127,8 @@ def version_must_be_1(cls, v):
 @platform_name("File Based Lineage")
 @config_class(LineageFileSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.LINEAGE_COARSE, "Specified in the lineage file.")
+@capability(SourceCapability.LINEAGE_FINE, "Specified in the lineage file.")
 @dataclass
 class LineageFileSource(Source):
     """
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py
index a000c66a406c2..c46b56da422d9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mode.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py
@@ -98,6 +98,7 @@ class HTTPError429(HTTPError):
 @config_class(ModeConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
 class ModeSource(Source):
     """
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
index ac1e03812db3b..bc05edbb3c623 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -26,11 +26,12 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceCapability, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.schema_classes import (
     DataFlowInfoClass,
@@ -360,6 +361,7 @@ def report_dropped(self, ent_name: str) -> None:
 @platform_name("NiFi", id="nifi")
 @config_class(NifiSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.LINEAGE_COARSE, "Supported. See docs for limitations")
 class NifiSource(Source):
     """
     This plugin extracts the following:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
index 52bcef66658c8..4611a8eed4782 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -264,7 +264,6 @@ def extract_lineage(
                 )
 
         if len(upstream) > 0:
-
             upstream_lineage_class: UpstreamLineageClass = UpstreamLineageClass(
                 upstreams=upstream,
                 fineGrainedLineages=cll_lineage or None,
@@ -1139,6 +1138,10 @@ def report_to_datahub_work_units(
     SourceCapability.OWNERSHIP,
     "Disabled by default, configured using `extract_ownership`",
 )
+@capability(
+    SourceCapability.LINEAGE_COARSE,
+    "Enabled by default, configured using `extract_lineage`.",
+)
 @capability(
     SourceCapability.LINEAGE_FINE,
     "Disabled by default, configured using `extract_column_level_lineage`. ",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py
index bce4d1ec76e6e..fcf97e461967c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py
@@ -20,11 +20,17 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    Source,
+    SourceCapability,
+    SourceReport,
+)
 from datahub.ingestion.api.source_helpers import auto_workunit_reporter
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.graph.client import DataHubGraph
@@ -83,6 +89,8 @@ def compute_stats(self) -> None:
 @platform_name("SQL Queries")
 @config_class(SqlQueriesSourceConfig)
 @support_status(SupportStatus.TESTING)
+@capability(SourceCapability.LINEAGE_COARSE, "Parsed from SQL queries")
+@capability(SourceCapability.LINEAGE_FINE, "Parsed from SQL queries")
 class SqlQueriesSource(Source):
     # TODO: Documentation
     urns: Optional[Set[str]]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py
index 14bc4242d2a91..e491a1e8b82fa 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/superset.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py
@@ -142,6 +142,7 @@ def get_filter_name(filter_obj):
 @capability(
     SourceCapability.DELETION_DETECTION, "Optionally enabled via stateful_ingestion"
 )
+@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
 class SupersetSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
index bad7ae49d325e..4bc40b0aac964 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
@@ -452,6 +452,10 @@ class TableauSourceReport(StaleEntityRemovalSourceReport):
 @capability(SourceCapability.OWNERSHIP, "Requires recipe configuration")
 @capability(SourceCapability.TAGS, "Requires recipe configuration")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default, configure using `extract_column_level_lineage`",
+)
 class TableauSource(StatefulIngestionSourceBase):
     platform = "tableau"
 
@@ -533,7 +537,7 @@ def fetch_projects():
                     path=[],
                 )
             # Set parent project name
-            for project_id, project in all_project_map.items():
+            for _project_id, project in all_project_map.items():
                 if (
                     project.parent_id is not None
                     and project.parent_id in all_project_map

From 4d35a254cabb3a6241af8857c7d63298783ebaa7 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Fri, 20 Oct 2023 17:09:14 -0400
Subject: [PATCH 077/259] =?UTF-8?q?Added=20more=20data-testid=20usage=20fo?=
 =?UTF-8?q?r=20edit=5Fdocumentation=20and=20managing=5Fsecr=E2=80=A6=20(#9?=
 =?UTF-8?q?060)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../components/legacy/DescriptionModal.tsx    |  6 ++-
 .../shared/components/styled/AddLinkModal.tsx |  6 ++-
 .../tabs/Documentation/DocumentationTab.tsx   |  1 +
 .../components/DescriptionEditorToolbar.tsx   |  2 +-
 .../app/ingest/secret/SecretBuilderModal.tsx  |  4 ++
 .../src/app/ingest/secret/SecretsList.tsx     |  6 ++-
 .../e2e/glossary/glossary_navigation.js       |  3 +-
 .../e2e/mutations/edit_documentation.js       | 42 +++++++--------
 .../cypress/e2e/mutations/managing_secrets.js | 51 ++++++++++---------
 9 files changed, 70 insertions(+), 51 deletions(-)

diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx
index 579b8c9905da0..cb37c44a36caa 100644
--- a/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/legacy/DescriptionModal.tsx
@@ -41,7 +41,11 @@ export default function UpdateDescriptionModal({ title, description, original, o
             footer={
                 <>
                     <Button onClick={onClose}>Cancel</Button>
-                    <Button onClick={() => onSubmit(updatedDesc)} disabled={updatedDesc === description}>
+                    <Button
+                        onClick={() => onSubmit(updatedDesc)}
+                        disabled={updatedDesc === description}
+                        data-testid="description-modal-update-button"
+                    >
                         Update
                     </Button>
                 </>
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx
index 34d4f0cb3fe91..68a8cf4094362 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx
@@ -57,7 +57,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
 
     return (
         <>
-            <Button icon={<PlusOutlined />} onClick={showModal} {...buttonProps}>
+            <Button data-testid="add-link-button" icon={<PlusOutlined />} onClick={showModal} {...buttonProps}>
                 Add Link
             </Button>
             <Modal
@@ -69,13 +69,14 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                     <Button type="text" onClick={handleClose}>
                         Cancel
                     </Button>,
-                    <Button form="addLinkForm" key="submit" htmlType="submit">
+                    <Button data-testid="add-link-modal-add-button" form="addLinkForm" key="submit" htmlType="submit">
                         Add
                     </Button>,
                 ]}
             >
                 <Form form={form} name="addLinkForm" onFinish={handleAdd} layout="vertical">
                     <Form.Item
+                        data-testid="add-link-modal-url"
                         name="url"
                         label="URL"
                         rules={[
@@ -93,6 +94,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                         <Input placeholder="https://" autoFocus />
                     </Form.Item>
                     <Form.Item
+                        data-testid="add-link-modal-label"
                         name="label"
                         label="Label"
                         rules={[
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx
index de065d23e56e7..344c2aef87175 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/DocumentationTab.tsx
@@ -60,6 +60,7 @@ export const DocumentationTab = ({ properties }: { properties?: Props }) => {
                     <TabToolbar>
                         <div>
                             <Button
+                                data-testid="edit-documentation-button"
                                 type="text"
                                 onClick={() => routeToTab({ tabName: 'Documentation', tabParams: { editing: true } })}
                             >
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx
index 6128a5f277c85..07e197049ccc8 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/DescriptionEditorToolbar.tsx
@@ -15,7 +15,7 @@ export const DescriptionEditorToolbar = ({ disableSave, onClose, onSave }: Descr
             <Button type="text" onClick={onClose}>
                 Back
             </Button>
-            <Button onClick={onSave} disabled={disableSave}>
+            <Button data-testid="description-editor-save-button" onClick={onSave} disabled={disableSave}>
                 <CheckOutlined /> Save
             </Button>
         </TabToolbar>
diff --git a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
index 539eef972608c..30f04d61b8fc9 100644
--- a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
+++ b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx
@@ -40,6 +40,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         Cancel
                     </Button>
                     <Button
+                        data-testid="secret-modal-create-button"
                         id="createSecretButton"
                         onClick={() =>
                             onSubmit?.(
@@ -71,6 +72,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         Give your secret a name. This is what you&apos;ll use to reference the secret from your recipes.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-name-input"
                         name={NAME_FIELD_NAME}
                         rules={[
                             {
@@ -91,6 +93,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         The value of your secret, which will be encrypted and stored securely within DataHub.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-value-input"
                         name={VALUE_FIELD_NAME}
                         rules={[
                             {
@@ -110,6 +113,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel }
                         An optional description to help keep track of your secret.
                     </Typography.Paragraph>
                     <Form.Item
+                        data-testid="secret-modal-description-input"
                         name={DESCRIPTION_FIELD_NAME}
                         rules={[{ whitespace: true }, { min: 1, max: 500 }]}
                         hasFeedback
diff --git a/datahub-web-react/src/app/ingest/secret/SecretsList.tsx b/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
index 69e11d1471db3..8e5b601e2a809 100644
--- a/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
+++ b/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
@@ -176,7 +176,11 @@ export const SecretsList = () => {
             <div>
                 <TabToolbar>
                     <div>
-                        <Button type="text" onClick={() => setIsCreatingSecret(true)}>
+                        <Button
+                            data-testid="create-secret-button"
+                            type="text"
+                            onClick={() => setIsCreatingSecret(true)}
+                        >
                             <PlusOutlined /> Create new secret
                         </Button>
                     </div>
diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
index f52e4d3984a88..aeceaf99be889 100644
--- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
+++ b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
@@ -17,8 +17,7 @@ describe("glossary sidebar navigation test", () => {
         cy.waitTextVisible("Created Term Group!");
         cy.waitTextVisible("Create Glossary Term");
         cy.enterTextInTestId("create-glossary-entity-modal-name", glossaryTerm);
-        cy.clickOptionWithTestId("glossary-entity-modal-create-button");
-        cy.waitTextVisible("Created Glossary Term!");
+        cy.clickOptionWithTestId("glossary-entity-modal-create-button").wait(3000);
         cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryTerm).click().wait(3000);
         cy.openThreeDotDropdown();
         cy.clickOptionWithTestId("entity-menu-move-button")
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js b/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js
index 83b66e2cb2549..5f9758a35ca0e 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js
@@ -10,20 +10,20 @@ describe("edit documentation and link to dataset", () => {
     cy.visit(
       "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema"
     );
-    cy.get("[role='tab']").contains("Documentation").click();
+    cy.openEntityTab("Documentation");
     cy.waitTextVisible("my hive dataset");
     cy.waitTextVisible("Sample doc");
-    cy.clickOptionWithText("Edit");
+    cy.clickOptionWithTestId("edit-documentation-button");
     cy.focused().clear();
     cy.focused().type(documentation_edited);
-    cy.get("button").contains("Save").click();
+    cy.clickOptionWithTestId("description-editor-save-button");
     cy.waitTextVisible("Description Updated");
     cy.waitTextVisible(documentation_edited);
     //return documentation to original state
-    cy.clickOptionWithText("Edit");
+    cy.clickOptionWithTestId("edit-documentation-button");
     cy.focused().clear().wait(1000);
     cy.focused().type("my hive dataset");
-    cy.get("button").contains("Save").click();
+    cy.clickOptionWithTestId("description-editor-save-button");
     cy.waitTextVisible("Description Updated");
     cy.waitTextVisible("my hive dataset");
   });
@@ -33,21 +33,21 @@ describe("edit documentation and link to dataset", () => {
     cy.visit(
       "/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema"
     );
-    cy.get("[role='tab']").contains("Documentation").click();
+    cy.openEntityTab("Documentation");
     cy.contains("Sample doc").trigger("mouseover", { force: true });
     cy.get('[data-icon="delete"]').click();
     cy.waitTextVisible("Link Removed");
-    cy.get("button").contains("Add Link").click().wait(1000);
-    cy.get('[role="dialog"] #addLinkForm_url').type(wrong_url);
+    cy.clickOptionWithTestId("add-link-button").wait(1000);
+    cy.enterTextInTestId("add-link-modal-url", wrong_url);
     cy.waitTextVisible("This field must be a valid url.");
     cy.focused().clear();
     cy.waitTextVisible("A URL is required.");
-    cy.focused().type(correct_url);
+    cy.enterTextInTestId("add-link-modal-url", correct_url);
     cy.ensureTextNotPresent("This field must be a valid url.");
-    cy.get("#addLinkForm_label").type("Sample doc");
-    cy.get('[role="dialog"] button').contains("Add").click();
+    cy.enterTextInTestId("add-link-modal-label", "Sample doc");
+    cy.clickOptionWithTestId("add-link-modal-add-button");
     cy.waitTextVisible("Link Added");
-    cy.get("[role='tab']").contains("Documentation").click();
+    cy.openEntityTab("Documentation");
     cy.get(`[href='${correct_url}']`).should("be.visible");
   });
 
@@ -55,18 +55,18 @@ describe("edit documentation and link to dataset", () => {
     cy.loginWithCredentials();
     cy.visit("/domain/urn:li:domain:marketing/Entities");
     cy.waitTextVisible("SampleCypressKafkaDataset");
-    cy.get("button").contains("Add Link").click().wait(1000);
-    cy.get('[role="dialog"] #addLinkForm_url').type(wrong_url);
+    cy.clickOptionWithTestId("add-link-button").wait(1000);
+    cy.enterTextInTestId("add-link-modal-url", wrong_url);
     cy.waitTextVisible("This field must be a valid url.");
     cy.focused().clear();
     cy.waitTextVisible("A URL is required.");
-    cy.focused().type(correct_url);
+    cy.enterTextInTestId("add-link-modal-url", correct_url);
     cy.ensureTextNotPresent("This field must be a valid url.");
-    cy.get("#addLinkForm_label").type("Sample doc");
-    cy.get('[role="dialog"] button').contains("Add").click();
+    cy.enterTextInTestId("add-link-modal-label", "Sample doc");
+    cy.clickOptionWithTestId("add-link-modal-add-button");
     cy.waitTextVisible("Link Added");
-    cy.get("[role='tab']").contains("Documentation").click();
-    cy.waitTextVisible("Edit");
+    cy.openEntityTab("Documentation");
+    cy.get("[data-testid='edit-documentation-button']").should("be.visible");
     cy.get(`[href='${correct_url}']`).should("be.visible");
     cy.contains("Sample doc").trigger("mouseover", { force: true });
     cy.get('[data-icon="delete"]').click();
@@ -83,14 +83,14 @@ describe("edit documentation and link to dataset", () => {
     cy.waitTextVisible("Foo field description has changed");
     cy.focused().clear().wait(1000);
     cy.focused().type(documentation_edited);
-    cy.get("button").contains("Update").click();
+    cy.clickOptionWithTestId("description-modal-update-button");
     cy.waitTextVisible("Updated!");
     cy.waitTextVisible(documentation_edited);
     cy.waitTextVisible("(edited)");
     cy.get("tbody [data-icon='edit']").first().click({ force: true });
     cy.focused().clear().wait(1000);
     cy.focused().type("Foo field description has changed");
-    cy.get("button").contains("Update").click();
+    cy.clickOptionWithTestId("description-modal-update-button");
     cy.waitTextVisible("Updated!");
     cy.waitTextVisible("Foo field description has changed");
     cy.waitTextVisible("(edited)");
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js
index 466bb2ef0757e..77fd63b9cae02 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js
@@ -8,23 +8,24 @@ const ingestion_source_name = `ingestion source ${number}`;
 
 describe("managing secrets for ingestion creation", () => {
     it("create a secret, create ingestion source using a secret, remove a secret", () => {
+      // Navigate to the manage ingestion page → secrets
       cy.loginWithCredentials();
-      //navigate to the manage ingestion page → secrets
       cy.goToIngestionPage();
-      cy.clickOptionWithText("Secrets");
-      //create a new secret
-      cy.clickOptionWithText("Create new secret");
-      cy.get('[role="dialog"]').contains("Create a new Secret").should("be.visible");
-      cy.get('[role="dialog"] #name').type(`secretname${number}`);
-      cy.get('[role="dialog"] #value').type(`secretvalue${number}`);
-      cy.get('[role="dialog"] #description').type(`secretdescription${number}`);
-      cy.get('#createSecretButton').click();
+      cy.openEntityTab("Secrets");
+
+      // Create a new secret
+      cy.clickOptionWithTestId("create-secret-button");
+      cy.enterTextInTestId('secret-modal-name-input', `secretname${number}`);
+      cy.enterTextInTestId('secret-modal-value-input', `secretvalue${number}`);
+      cy.enterTextInTestId('secret-modal-description-input', `secretdescription${number}`);
+      cy.clickOptionWithTestId("secret-modal-create-button");
       cy.waitTextVisible("Successfully created Secret!");
       cy.waitTextVisible(`secretname${number}`);
-      cy.waitTextVisible(`secretdescription${number}`).wait(5000)//prevent issue with missing secret
-      //create an ingestion source using a secret
+      cy.waitTextVisible(`secretdescription${number}`).wait(5000)
+
+      // Create an ingestion source using a secret
       cy.goToIngestionPage();
-      cy.clickOptionWithText("Create new source");
+      cy.get("#ingestion-create-source").click();
       cy.clickOptionWithText("Snowflake");
       cy.waitTextVisible("Snowflake Recipe");
       cy.get("#account_id").type(accound_id);
@@ -40,11 +41,12 @@ describe("managing secrets for ingestion creation", () => {
       cy.waitTextVisible("Give this ingestion source a name."); 
       cy.get('[data-testid="source-name-input"]').type(ingestion_source_name);
       cy.get("button").contains("Save").click();
-      cy.waitTextVisible("Successfully created ingestion source!").wait(5000)//prevent issue with missing form data
+      cy.waitTextVisible("Successfully created ingestion source!").wait(5000)
       cy.waitTextVisible(ingestion_source_name);
       cy.get("button").contains("Pending...").should("be.visible");
-      //remove a secret
-      cy.clickOptionWithText("Secrets");
+
+      // Remove a secret
+      cy.openEntityTab("Secrets");
       cy.waitTextVisible(`secretname${number}`);
       cy.get('[data-icon="delete"]').first().click();
       cy.waitTextVisible("Confirm Secret Removal");
@@ -52,14 +54,16 @@ describe("managing secrets for ingestion creation", () => {
       cy.waitTextVisible("Removed secret.");
       cy.ensureTextNotPresent(`secretname${number}`);
       cy.ensureTextNotPresent(`secretdescription${number}`);
-      //remove ingestion source
+
+      // Remove ingestion source
       cy.goToIngestionPage();
       cy.get('[data-testid="delete-button"]').first().click();
       cy.waitTextVisible("Confirm Ingestion Source Removal");
       cy.get("button").contains("Yes").click();
       cy.waitTextVisible("Removed ingestion source.");
       cy.ensureTextNotPresent(ingestion_source_name)
-      //verify secret is not present during ingestion source creation for password dropdown
+
+      // Verify secret is not present during ingestion source creation for password dropdown
       cy.clickOptionWithText("Create new source");
       cy.clickOptionWithText("Snowflake");
       cy.waitTextVisible("Snowflake Recipe");
@@ -68,13 +72,13 @@ describe("managing secrets for ingestion creation", () => {
       cy.get("#username").type(username);
       cy.get("#password").click().wait(1000);
       cy.ensureTextNotPresent(`secretname${number}`);
-      //verify secret can be added during ingestion source creation and used successfully
+
+      // Verify secret can be added during ingestion source creation and used successfully
       cy.clickOptionWithText("Create Secret");    
-      cy.get('[role="dialog"]').contains("Create a new Secret").should("be.visible");
-      cy.get('[role="dialog"] #name').type(`secretname${number}`);
-      cy.get('[role="dialog"] #value').type(`secretvalue${number}`);
-      cy.get('[role="dialog"] #description').type(`secretdescription${number}`);
-      cy.get('#createSecretButton').click();
+      cy.enterTextInTestId('secret-modal-name-input', `secretname${number}`)
+      cy.enterTextInTestId('secret-modal-value-input', `secretvalue${number}`)
+      cy.enterTextInTestId('secret-modal-description-input', `secretdescription${number}`)
+      cy.clickOptionWithTestId("secret-modal-create-button");
       cy.waitTextVisible("Created secret!");
       cy.get("#role").type(role);
       cy.get("button").contains("Next").click();
@@ -86,6 +90,7 @@ describe("managing secrets for ingestion creation", () => {
       cy.waitTextVisible("Successfully created ingestion source!").wait(5000)//prevent issue with missing form data
       cy.waitTextVisible(ingestion_source_name);
       cy.get("button").contains("Pending...").should("be.visible");
+
       //Remove ingestion source and secret
       cy.goToIngestionPage();
       cy.get('[data-testid="delete-button"]').first().click();

From 63599c95553b89304b656efb2c208c9084d60717 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Sat, 21 Oct 2023 03:17:28 -0700
Subject: [PATCH 078/259] fix(search): fix mapping builder bug (#9062)

---
 .../search/elasticsearch/indexbuilder/MappingsBuilder.java  | 2 +-
 .../timeseries/search/TimeseriesAspectServiceTestBase.java  | 6 ++++--
 .../io/datahubproject/test/search/SearchTestContainer.java  | 2 +-
 smoke-test/tests/containers/containers_test.py              | 1 +
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
index 1edc77bbd214c..35cef71edd953 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
@@ -133,7 +133,7 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
     } else if (fieldType == FieldType.DATETIME) {
       mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
     } else if (fieldType == FieldType.OBJECT) {
-      mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
+      mappingForField.put(TYPE, ESUtils.OBJECT_FIELD_TYPE);
     } else {
       log.info("FieldType {} has no mappings implemented", fieldType);
     }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
index f9b8f84b10ad2..b19d2026fbfc4 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
@@ -889,15 +889,17 @@ public void testCountByFilterAfterDelete() throws InterruptedException {
   @Test(groups = {"getAggregatedStats"}, dependsOnGroups = {"upsert"})
   public void testGetIndexSizes() {
     List<TimeseriesIndexSizeResult> result = _elasticSearchTimeseriesAspectService.getIndexSizes();
+    //CHECKSTYLE:OFF
     /*
     Example result:
     {aspectName=testentityprofile, sizeMb=52.234, indexName=es_timeseries_aspect_service_test_testentity_testentityprofileaspect_v1, entityName=testentity}
     {aspectName=testentityprofile, sizeMb=0.208, indexName=es_timeseries_aspect_service_test_testentitywithouttests_testentityprofileaspect_v1, entityName=testentitywithouttests}
      */
     // There may be other indices in there from other tests, so just make sure that index for entity + aspect is in there
-    assertTrue(result.size() > 1);
+    //CHECKSTYLE:ON
+    assertTrue(result.size() > 0);
     assertTrue(
         result.stream().anyMatch(idxSizeResult -> idxSizeResult.getIndexName().equals(
-            "es_timeseries_aspect_service_test_testentitywithouttests_testentityprofileaspect_v1")));
+            "es_timeseries_aspect_service_test_testentity_testentityprofileaspect_v1")));
   }
 }
diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java
index 67e1ee368f513..4c1555fc510e6 100644
--- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java
+++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java
@@ -5,7 +5,7 @@
 import java.time.Duration;
 
 public interface SearchTestContainer {
-    String SEARCH_JAVA_OPTS = "-Xms64m -Xmx384m -XX:MaxDirectMemorySize=368435456";
+    String SEARCH_JAVA_OPTS = "-Xms446m -Xmx446m -XX:MaxDirectMemorySize=368435456";
     Duration STARTUP_TIMEOUT = Duration.ofMinutes(5); // usually < 1min
 
     GenericContainer<?> startContainer();
diff --git a/smoke-test/tests/containers/containers_test.py b/smoke-test/tests/containers/containers_test.py
index 05a45239dabf8..227645a87d30a 100644
--- a/smoke-test/tests/containers/containers_test.py
+++ b/smoke-test/tests/containers/containers_test.py
@@ -227,6 +227,7 @@ def test_update_container(frontend_session, ingest_cleanup_data):
                 "ownerUrn": new_owner,
                 "resourceUrn": container_urn,
                 "ownerEntityType": "CORP_USER",
+                "ownershipTypeUrn": "urn:li:ownershipType:__system__technical_owner"
             }
         },
     }

From 86e0023a4e158467130f7337478a48bf98fb344b Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Sat, 21 Oct 2023 16:20:59 +0100
Subject: [PATCH 079/259] feat(ingestion): Adds more advanced configurations
 for runtime debugging (#8998)

---
 .../ingest/IngestionResolverUtils.java        |  10 ++
 ...eateIngestionExecutionRequestResolver.java |   3 +
 .../source/UpsertIngestionSourceResolver.java |  10 ++
 .../src/main/resources/ingestion.graphql      |  10 ++
 .../UpsertIngestionSourceResolverTest.java    |   2 +-
 .../app/ingest/source/IngestionSourceList.tsx |   8 +-
 .../ingest/source/builder/NameSourceStep.tsx  | 123 +++++++++++++++++-
 .../src/app/ingest/source/builder/types.ts    |  17 +++
 .../src/graphql/ingestion.graphql             |   8 ++
 docker/build.gradle                           |  12 +-
 docs/ui-ingestion.md                          |  20 ++-
 .../docs/dev_guides/profiling_ingestions.md   |  39 ++++++
 .../TimeseriesAspectServiceTestBase.java      |   6 +-
 .../test/search/SearchTestContainer.java      |   2 +
 .../ingestion/DataHubIngestionSourceInfo.pdl  |  13 +-
 15 files changed, 267 insertions(+), 16 deletions(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java
index 7db0b6f826a04..1140c031f1d35 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java
@@ -5,6 +5,7 @@
 import com.linkedin.datahub.graphql.generated.IngestionConfig;
 import com.linkedin.datahub.graphql.generated.IngestionSchedule;
 import com.linkedin.datahub.graphql.generated.IngestionSource;
+import com.linkedin.datahub.graphql.generated.StringMapEntry;
 import com.linkedin.datahub.graphql.generated.StructuredReport;
 import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper;
 import com.linkedin.entity.EntityResponse;
@@ -21,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -143,6 +145,14 @@ public static IngestionConfig mapIngestionSourceConfig(final DataHubIngestionSou
     result.setVersion(config.getVersion());
     result.setExecutorId(config.getExecutorId());
     result.setDebugMode(config.isDebugMode());
+    if (config.getExtraArgs() != null) {
+      List<StringMapEntry> extraArgs = config.getExtraArgs()
+          .keySet()
+          .stream()
+          .map(key -> new StringMapEntry(key, config.getExtraArgs().get(key)))
+          .collect(Collectors.toList());
+      result.setExtraArgs(extraArgs);
+    }
     return result;
   }
 
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java
index e5064e6620526..ea20b837e0a1f 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java
@@ -117,6 +117,9 @@ public CompletableFuture<String> get(final DataFetchingEnvironment environment)
           if (ingestionSourceInfo.getConfig().hasDebugMode()) {
             debugMode = ingestionSourceInfo.getConfig().isDebugMode() ? "true" : "false";
           }
+          if (ingestionSourceInfo.getConfig().hasExtraArgs()) {
+            arguments.putAll(ingestionSourceInfo.getConfig().getExtraArgs());
+          }
           arguments.put(DEBUG_MODE_ARG_NAME, debugMode);
           execInput.setArgs(new StringMap(arguments));
 
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java
index 2ce394ad5ba84..68e334bd976f8 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java
@@ -1,10 +1,12 @@
 package com.linkedin.datahub.graphql.resolvers.ingest.source;
 
 import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.StringMap;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode;
 import com.linkedin.datahub.graphql.exception.DataHubGraphQLException;
+import com.linkedin.datahub.graphql.generated.StringMapEntryInput;
 import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceConfigInput;
 import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceInput;
 import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceScheduleInput;
@@ -17,6 +19,8 @@
 import com.linkedin.mxe.MetadataChangeProposal;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.Map;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 
 import java.net.URISyntaxException;
@@ -108,6 +112,12 @@ private DataHubIngestionSourceConfig mapConfig(final UpdateIngestionSourceConfig
     if (input.getDebugMode() != null) {
       result.setDebugMode(input.getDebugMode());
     }
+    if (input.getExtraArgs() != null) {
+      Map<String, String> extraArgs = input.getExtraArgs()
+          .stream()
+          .collect(Collectors.toMap(StringMapEntryInput::getKey, StringMapEntryInput::getValue));
+      result.setExtraArgs(new StringMap(extraArgs));
+    }
     return result;
   }
 
diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql
index 69c8aff124583..21f9fb2633119 100644
--- a/datahub-graphql-core/src/main/resources/ingestion.graphql
+++ b/datahub-graphql-core/src/main/resources/ingestion.graphql
@@ -332,6 +332,11 @@ type IngestionConfig {
   Advanced: Whether or not to run ingestion in debug mode
   """
   debugMode: Boolean
+
+  """
+  Advanced: Extra arguments for the ingestion run.
+  """
+  extraArgs: [StringMapEntry!]
 }
 
 """
@@ -483,6 +488,11 @@ input UpdateIngestionSourceConfigInput {
   Whether or not to run ingestion in debug mode
   """
   debugMode: Boolean
+
+  """
+  Extra arguments for the ingestion run.
+  """
+  extraArgs: [StringMapEntryInput!]
 }
 
 """
diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java
index 2538accc694fb..16d8da9169a8f 100644
--- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java
+++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java
@@ -26,7 +26,7 @@ public class UpsertIngestionSourceResolverTest {
       "Test source",
       "mysql", "Test source description",
       new UpdateIngestionSourceScheduleInput("* * * * *", "UTC"),
-      new UpdateIngestionSourceConfigInput("my test recipe", "0.8.18", "executor id", false)
+      new UpdateIngestionSourceConfigInput("my test recipe", "0.8.18", "executor id", false, null)
   );
 
   @Test
diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
index 0e341a5ff3a79..13af19b0b6ac2 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -15,7 +15,7 @@ import { Message } from '../../shared/Message';
 import TabToolbar from '../../entity/shared/components/styled/TabToolbar';
 import { IngestionSourceBuilderModal } from './builder/IngestionSourceBuilderModal';
 import { addToListIngestionSourcesCache, CLI_EXECUTOR_ID, removeFromListIngestionSourcesCache } from './utils';
-import { DEFAULT_EXECUTOR_ID, SourceBuilderState } from './builder/types';
+import { DEFAULT_EXECUTOR_ID, SourceBuilderState, StringMapEntryInput } from './builder/types';
 import { IngestionSource, UpdateIngestionSourceInput } from '../../../types.generated';
 import { SearchBar } from '../../search/SearchBar';
 import { useEntityRegistry } from '../../useEntityRegistry';
@@ -173,6 +173,11 @@ export const IngestionSourceList = () => {
         setFocusSourceUrn(undefined);
     };
 
+    const formatExtraArgs = (extraArgs): StringMapEntryInput[] => {
+        if (extraArgs === null || extraArgs === undefined) return [];
+        return extraArgs.map((entry) => ({ key: entry.key, value: entry.value }));
+    };
+
     const createOrUpdateIngestionSource = (
         input: UpdateIngestionSourceInput,
         resetState: () => void,
@@ -294,6 +299,7 @@ export const IngestionSourceList = () => {
                             (recipeBuilderState.config?.executorId as string)) ||
                         DEFAULT_EXECUTOR_ID,
                     debugMode: recipeBuilderState.config?.debugMode || false,
+                    extraArgs: formatExtraArgs(recipeBuilderState.config?.extraArgs || []),
                 },
                 schedule: recipeBuilderState.schedule && {
                     interval: recipeBuilderState.schedule?.interval as string,
diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
index 992ebff643c31..f4c048bcaf0d2 100644
--- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -1,7 +1,7 @@
 import { Button, Checkbox, Collapse, Form, Input, Typography } from 'antd';
 import React from 'react';
 import styled from 'styled-components';
-import { SourceBuilderState, StepProps } from './types';
+import { SourceBuilderState, StepProps, StringMapEntryInput } from './types';
 
 const ControlsContainer = styled.div`
     display: flex;
@@ -13,6 +13,10 @@ const SaveButton = styled(Button)`
     margin-right: 15px;
 `;
 
+const ExtraEnvKey = 'extra_env_vars';
+const ExtraReqKey = 'extra_pip_requirements';
+const ExtraPluginKey = 'extra_pip_plugins';
+
 export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps) => {
     const setName = (stagedName: string) => {
         const newState: SourceBuilderState = {
@@ -55,6 +59,90 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
         updateState(newState);
     };
 
+    const retrieveExtraEnvs = () => {
+        const extraArgs: StringMapEntryInput[] = state.config?.extraArgs ? state.config?.extraArgs : [];
+        const index: number = extraArgs.findIndex((entry) => entry.key === ExtraEnvKey) as number;
+        if (index > -1) {
+            return extraArgs[index].value;
+        }
+        return '';
+    };
+
+    const setExtraEnvs = (envs: string) => {
+        let extraArgs: StringMapEntryInput[] = state.config?.extraArgs ? state.config?.extraArgs : [];
+        const indxOfEnvVars: number = extraArgs.findIndex((entry) => entry.key === ExtraEnvKey) as number;
+        const value = { key: ExtraEnvKey, value: envs };
+        if (indxOfEnvVars > -1) {
+            extraArgs[indxOfEnvVars] = value;
+        } else {
+            extraArgs = [...extraArgs, value];
+        }
+        const newState: SourceBuilderState = {
+            ...state,
+            config: {
+                ...state.config,
+                extraArgs,
+            },
+        };
+        updateState(newState);
+    };
+
+    const retrieveExtraDataHubPlugins = () => {
+        const extraArgs: StringMapEntryInput[] = state.config?.extraArgs ? state.config?.extraArgs : [];
+        const index: number = extraArgs.findIndex((entry) => entry.key === ExtraPluginKey) as number;
+        if (index > -1) {
+            return extraArgs[index].value;
+        }
+        return '';
+    };
+
+    const setExtraDataHubPlugins = (plugins: string) => {
+        let extraArgs: StringMapEntryInput[] = state.config?.extraArgs ? state.config?.extraArgs : [];
+        const indxOfPlugins: number = extraArgs.findIndex((entry) => entry.key === ExtraPluginKey) as number;
+        const value = { key: ExtraPluginKey, value: plugins };
+        if (indxOfPlugins > -1) {
+            extraArgs[indxOfPlugins] = value;
+        } else {
+            extraArgs = [...extraArgs, value];
+        }
+        const newState: SourceBuilderState = {
+            ...state,
+            config: {
+                ...state.config,
+                extraArgs,
+            },
+        };
+        updateState(newState);
+    };
+
+    const retrieveExtraReqs = () => {
+        const extraArgs: StringMapEntryInput[] = state.config?.extraArgs ? state.config?.extraArgs : [];
+        const index: number = extraArgs.findIndex((entry) => entry.key === ExtraReqKey) as number;
+        if (index > -1) {
+            return extraArgs[index].value;
+        }
+        return '';
+    };
+
+    const setExtraReqs = (reqs: string) => {
+        let extraArgs: StringMapEntryInput[] = state.config?.extraArgs ? state.config?.extraArgs : [];
+        const indxOfReqs: number = extraArgs.findIndex((entry) => entry.key === ExtraReqKey) as number;
+        const value = { key: ExtraReqKey, value: reqs };
+        if (indxOfReqs > -1) {
+            extraArgs[indxOfReqs] = value;
+        } else {
+            extraArgs = [...extraArgs, value];
+        }
+        const newState: SourceBuilderState = {
+            ...state,
+            config: {
+                ...state.config,
+                extraArgs,
+            },
+        };
+        updateState(newState);
+    };
+
     const onClickCreate = (shouldRun?: boolean) => {
         if (state.name !== undefined && state.name.length > 0) {
             submit(shouldRun);
@@ -116,6 +204,39 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 onChange={(event) => setDebugMode(event.target.checked)}
                             />
                         </Form.Item>
+                        <Form.Item label={<Typography.Text strong>Extra Enviroment Variables</Typography.Text>}>
+                            <Typography.Paragraph>
+                                Advanced: Set extra environment variables to an ingestion execution
+                            </Typography.Paragraph>
+                            <Input
+                                data-testid="extra-args-input"
+                                placeholder='{"MY_CUSTOM_ENV": "my_custom_value2"}'
+                                value={retrieveExtraEnvs()}
+                                onChange={(event) => setExtraEnvs(event.target.value)}
+                            />
+                        </Form.Item>
+                        <Form.Item label={<Typography.Text strong>Extra DataHub plugins</Typography.Text>}>
+                            <Typography.Paragraph>
+                                Advanced: Set extra DataHub plugins for an ingestion execution
+                            </Typography.Paragraph>
+                            <Input
+                                data-testid="extra-pip-plugin-input"
+                                placeholder='["debug"]'
+                                value={retrieveExtraDataHubPlugins()}
+                                onChange={(event) => setExtraDataHubPlugins(event.target.value)}
+                            />
+                        </Form.Item>
+                        <Form.Item label={<Typography.Text strong>Extra Pip Libraries</Typography.Text>}>
+                            <Typography.Paragraph>
+                                Advanced: Add extra pip libraries for an ingestion execution
+                            </Typography.Paragraph>
+                            <Input
+                                data-testid="extra-pip-reqs-input"
+                                placeholder='["sqlparse==0.4.3"]'
+                                value={retrieveExtraReqs()}
+                                onChange={(event) => setExtraReqs(event.target.value)}
+                            />
+                        </Form.Item>
                     </Collapse.Panel>
                 </Collapse>
             </Form>
diff --git a/datahub-web-react/src/app/ingest/source/builder/types.ts b/datahub-web-react/src/app/ingest/source/builder/types.ts
index cfe0f27ae7dbe..2df467b7beba1 100644
--- a/datahub-web-react/src/app/ingest/source/builder/types.ts
+++ b/datahub-web-react/src/app/ingest/source/builder/types.ts
@@ -34,6 +34,18 @@ export type StepProps = {
     ingestionSources: SourceConfig[];
 };
 
+export type StringMapEntryInput = {
+    /**
+     * The key of the map entry
+     */
+    key: string;
+
+    /**
+     * The value fo the map entry
+     */
+    value: string;
+};
+
 /**
  * The object represents the state of the Ingestion Source Builder form.
  */
@@ -91,5 +103,10 @@ export interface SourceBuilderState {
          * Advanced: Whether or not to run this ingestion source in debug mode
          */
         debugMode?: boolean | null;
+
+        /**
+         * Advanced: Extra arguments for the ingestion run.
+         */
+        extraArgs?: StringMapEntryInput[] | null;
     };
 }
diff --git a/datahub-web-react/src/graphql/ingestion.graphql b/datahub-web-react/src/graphql/ingestion.graphql
index c127e9ec03f9a..1767fe34bfef0 100644
--- a/datahub-web-react/src/graphql/ingestion.graphql
+++ b/datahub-web-react/src/graphql/ingestion.graphql
@@ -12,6 +12,10 @@ query listIngestionSources($input: ListIngestionSourcesInput!) {
                 version
                 executorId
                 debugMode
+                extraArgs {
+                    key
+                    value
+                }
             }
             schedule {
                 interval
@@ -51,6 +55,10 @@ query getIngestionSource($urn: String!, $runStart: Int, $runCount: Int) {
             version
             executorId
             debugMode
+            extraArgs {
+                key
+                value
+            }
         }
         schedule {
             interval
diff --git a/docker/build.gradle b/docker/build.gradle
index c8fdbc86b18b7..56634a5fe0c67 100644
--- a/docker/build.gradle
+++ b/docker/build.gradle
@@ -97,10 +97,20 @@ task quickstartDebug(type: Exec, dependsOn: ':metadata-ingestion:install') {
     dependsOn(debug_modules.collect { it + ':dockerTagDebug' })
     shouldRunAfter ':metadata-ingestion:clean', 'quickstartNuke'
 
-    environment "DATAHUB_PRECREATE_TOPICS", "true"
     environment "DATAHUB_TELEMETRY_ENABLED", "false"
     environment "DOCKER_COMPOSE_BASE", "file://${rootProject.projectDir}"
 
+    // Elastic
+    // environment "DATAHUB_SEARCH_IMAGE", 'elasticsearch'
+    // environment "DATAHUB_SEARCH_TAG", '7.10.1'
+
+    // OpenSearch
+    environment "DATAHUB_SEARCH_IMAGE", 'opensearchproject/opensearch'
+    environment "DATAHUB_SEARCH_TAG", '2.9.0'
+    environment "XPACK_SECURITY_ENABLED", 'plugins.security.disabled=true'
+    environment "USE_AWS_ELASTICSEARCH", 'true'
+
+
     def cmd = [
             'source ../metadata-ingestion/venv/bin/activate && ',
             'datahub docker quickstart',
diff --git a/docs/ui-ingestion.md b/docs/ui-ingestion.md
index db2007e1e19a9..438ddd8823b7e 100644
--- a/docs/ui-ingestion.md
+++ b/docs/ui-ingestion.md
@@ -1,5 +1,12 @@
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Ingestion
 
+<FeatureAvailability/>
+
 ## Introduction 
 
 Starting in version `0.8.25`, DataHub supports creating, configuring, scheduling, & executing batch metadata ingestion using the DataHub user interface. This makes
@@ -173,28 +180,29 @@ Finally, give your Ingestion Source a name.
 Once you're happy with your configurations, click 'Done' to save your changes.
 
 
-##### Advanced: Running with a specific CLI version 
+##### Advanced ingestion configs:
 
-DataHub comes pre-configured to use the latest version of the DataHub CLI ([acryl-datahub](https://pypi.org/project/acryl-datahub/)) that is compatible
+DataHub's Managed Ingestion UI comes pre-configured to use the latest version of the DataHub CLI ([acryl-datahub](https://pypi.org/project/acryl-datahub/)) that is compatible
 with the server. However, you can override the default package version using the 'Advanced' source configurations.
 
 To do so, simply click 'Advanced', then change the 'CLI Version' text box to contain the exact version
 of the DataHub CLI you'd like to use.
 
-
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/custom-ingestion-cli-version.png"/>
 </p>
 
 _Pinning the CLI version to version `0.8.23.2`_
 
+Other advanced options include specifying **environment variables**, **DataHub plugins** or **python packages at runtime**. 
+
 Once you're happy with your changes, simply click 'Done' to save. 
 
    </TabItem>
    <TabItem value="cli" label="CLI" default>
 
 You can upload and even update recipes using the cli as mentioned in the [cli documentation for uploading ingestion recipes](./cli.md#ingest-deploy).
-An example execution would look something like:
+An example execution for a given `recipe.yaml` file, would look something like:
 
 ```bash
 datahub ingest deploy --name "My Test Ingestion Source" --schedule "5 * * * *" --time-zone "UTC" -c recipe.yaml
@@ -330,8 +338,8 @@ for the `datahub-actions` container and running `docker logs <container-id>`.
 There are valid cases for ingesting metadata without the UI-based ingestion scheduler. For example,
 
 - You have written a custom ingestion Source
-- Your data sources are not reachable on the network where DataHub is deployed
-- Your ingestion source requires context from a local filesystem (e.g. input files, environment variables, etc)
+- Your data sources are not reachable on the network where DataHub is deployed. Managed DataHub users can use a [remote executor](managed-datahub/operator-guide/setting-up-remote-ingestion-executor-on-aws.md) for remote UI-based ingestion.
+- Your ingestion source requires context from a local filesystem (e.g. input files)
 - You want to distribute metadata ingestion among multiple producers / environments
 
 ### How do I attach policies to the actions pod to give it permissions to pull metadata from various sources?
diff --git a/metadata-ingestion/docs/dev_guides/profiling_ingestions.md b/metadata-ingestion/docs/dev_guides/profiling_ingestions.md
index d876d99b494f8..77cc2f456aa2d 100644
--- a/metadata-ingestion/docs/dev_guides/profiling_ingestions.md
+++ b/metadata-ingestion/docs/dev_guides/profiling_ingestions.md
@@ -13,6 +13,35 @@ This page documents how to perform memory profiles of ingestion runs.
 It is useful when trying to size the amount of resources necessary to ingest some source or when developing new features or sources.
 
 ## How to use
+
+<Tabs>
+<TabItem value="ui" label="UI" default>
+
+Create an ingestion as specified in the [Ingestion guide](../../../docs/ui-ingestion.md).
+
+Add a flag to your ingestion recipe to generate a memray memory dump of your ingestion:
+```yaml
+source:
+  ...
+
+sink:
+  ...
+
+flags:
+  generate_memory_profiles: "<path to folder where dumps will be written to>"
+```
+
+In the final panel, under the advanced section, add the `debug` datahub package under the **Extra DataHub Plugins** section.
+As seen below:
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion-advanced-extra-datahub-plugin.png"/>
+</p>
+
+Finally, save and run the ingestion process.
+
+</TabItem>
+<TabItem value="cli" label="CLI" default>
 Install the `debug` plugin for DataHub's CLI wherever the ingestion runs:
 
 ```bash
@@ -33,6 +62,16 @@ flags:
   generate_memory_profiles: "<path to folder where dumps will be written to>"
 ```
 
+Finally run the ingestion recipe
+
+```bash
+$ datahub ingest -c recipe.yaml
+```
+
+</TabItem>
+</Tabs>
+
+
 Once the ingestion run starts a binary file will be created and appended to during the execution of the ingestion. 
 
 These files follow the pattern `file-<ingestion-run-urn>.bin` for a unique identification.
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
index b19d2026fbfc4..1362a0f69eff2 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java
@@ -892,8 +892,10 @@ public void testGetIndexSizes() {
     //CHECKSTYLE:OFF
     /*
     Example result:
-    {aspectName=testentityprofile, sizeMb=52.234, indexName=es_timeseries_aspect_service_test_testentity_testentityprofileaspect_v1, entityName=testentity}
-    {aspectName=testentityprofile, sizeMb=0.208, indexName=es_timeseries_aspect_service_test_testentitywithouttests_testentityprofileaspect_v1, entityName=testentitywithouttests}
+    {aspectName=testentityprofile, sizeMb=52.234,
+    indexName=es_timeseries_aspect_service_test_testentity_testentityprofileaspect_v1, entityName=testentity}
+    {aspectName=testentityprofile, sizeMb=0.208,
+    indexName=es_timeseries_aspect_service_test_testentitywithouttests_testentityprofileaspect_v1, entityName=testentitywithouttests}
      */
     // There may be other indices in there from other tests, so just make sure that index for entity + aspect is in there
     //CHECKSTYLE:ON
diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java
index 4c1555fc510e6..34aa6978f742f 100644
--- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java
+++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestContainer.java
@@ -5,7 +5,9 @@
 import java.time.Duration;
 
 public interface SearchTestContainer {
+
     String SEARCH_JAVA_OPTS = "-Xms446m -Xmx446m -XX:MaxDirectMemorySize=368435456";
+
     Duration STARTUP_TIMEOUT = Duration.ofMinutes(5); // usually < 1min
 
     GenericContainer<?> startContainer();
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
index b3e237202fc2f..f777b5d6e12e7 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
@@ -37,10 +37,10 @@ record DataHubIngestionSourceInfo {
   * Parameters associated with the Ingestion Source
   */
   config: record DataHubIngestionSourceConfig {
-    /**
-     * The JSON recipe to use for ingestion
-     */
-     recipe: string
+     /**
+      * The JSON recipe to use for ingestion
+      */
+      recipe: string
 
      /**
       * The PyPI version of the datahub CLI to use when executing a recipe
@@ -56,5 +56,10 @@ record DataHubIngestionSourceInfo {
       * Whether or not to run this ingestion source in debug mode
       */
       debugMode: optional boolean
+
+     /**
+      * Extra arguments for the ingestion run.
+      */
+      extraArgs: optional map[string, string]
   }
 }
\ No newline at end of file

From 04216e30bb2a1eab1993e48276d9f8e52d6b3121 Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Sat, 21 Oct 2023 17:21:54 +0200
Subject: [PATCH 080/259] feat(ingest/s3): S3 add partition to schema (#8900)

Co-authored-by: Pedro Silva <pedro@acryl.io>
---
 .../src/datahub/ingestion/source/s3/config.py |  5 ++-
 .../src/datahub/ingestion/source/s3/source.py | 33 +++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
index 9b5296f0b9dd5..3ef6476078f6f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
@@ -75,7 +75,10 @@ class DataLakeSourceConfig(
         default=100,
         description="Maximum number of rows to use when inferring schemas for TSV and CSV files.",
     )
-
+    add_partition_columns_to_schema: bool = Field(
+        default=False,
+        description="Whether to add partition fields to the schema.",
+    )
     verify_ssl: Union[bool, str] = Field(
         default=True,
         description="Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
index eb49fcbb268c0..94c571eabad11 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
@@ -78,6 +78,7 @@
     NullTypeClass,
     NumberTypeClass,
     RecordTypeClass,
+    SchemaField,
     SchemaFieldDataType,
     SchemaMetadata,
     StringTypeClass,
@@ -90,6 +91,7 @@
     OperationClass,
     OperationTypeClass,
     OtherSchemaClass,
+    SchemaFieldDataTypeClass,
     _Aspect,
 )
 from datahub.telemetry import stats, telemetry
@@ -458,8 +460,39 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List:
         logger.debug(f"Extracted fields in schema: {fields}")
         fields = sorted(fields, key=lambda f: f.fieldPath)
 
+        if self.source_config.add_partition_columns_to_schema:
+            self.add_partition_columns_to_schema(
+                fields=fields, path_spec=path_spec, full_path=table_data.full_path
+            )
+
         return fields
 
+    def add_partition_columns_to_schema(
+        self, path_spec: PathSpec, full_path: str, fields: List[SchemaField]
+    ) -> None:
+        is_fieldpath_v2 = False
+        for field in fields:
+            if field.fieldPath.startswith("[version=2.0]"):
+                is_fieldpath_v2 = True
+                break
+        vars = path_spec.get_named_vars(full_path)
+        if vars is not None and "partition_key" in vars:
+            for partition_key in vars["partition_key"].values():
+                fields.append(
+                    SchemaField(
+                        fieldPath=f"{partition_key}"
+                        if not is_fieldpath_v2
+                        else f"[version=2.0].[type=string].{partition_key}",
+                        nativeDataType="string",
+                        type=SchemaFieldDataType(StringTypeClass())
+                        if not is_fieldpath_v2
+                        else SchemaFieldDataTypeClass(type=StringTypeClass()),
+                        isPartitioningKey=True,
+                        nullable=True,
+                        recursive=False,
+                    )
+                )
+
     def get_table_profile(
         self, table_data: TableData, dataset_urn: str
     ) -> Iterable[MetadataWorkUnit]:

From 633e6d6f779dc1e09bdfd10a160889684c485706 Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Mon, 23 Oct 2023 17:15:44 +0100
Subject: [PATCH 081/259] feat(frontend): Remove debug flag from start script
 (#9075)

---
 docker/datahub-frontend/start.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/datahub-frontend/start.sh b/docker/datahub-frontend/start.sh
index 430982aa2456b..9dc1514144bb1 100755
--- a/docker/datahub-frontend/start.sh
+++ b/docker/datahub-frontend/start.sh
@@ -50,7 +50,6 @@ export JAVA_OPTS="-Xms512m \
    -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf \
    -Dlogback.configurationFile=datahub-frontend/conf/logback.xml \
    -Dlogback.debug=false \
-   -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 \
    ${PROMETHEUS_AGENT:-} ${OTEL_AGENT:-} \
    ${TRUSTSTORE_FILE:-} ${TRUSTSTORE_TYPE:-} ${TRUSTSTORE_PASSWORD:-} \
    ${HTTP_PROXY:-} ${HTTPS_PROXY:-} ${NO_PROXY:-} \

From 8fb95e88a17260d0d6727f4d5e09636b128faf47 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 23 Oct 2023 12:40:42 -0700
Subject: [PATCH 082/259] feat(sqlparser): parse create DDL statements (#9002)

---
 .../goldens/v2_sqlite_operator.json           | 162 +++++++++++++++---
 .../v2_sqlite_operator_no_dag_listener.json   | 162 +++++++++++++++---
 .../datahub/emitter/sql_parsing_builder.py    |   9 +-
 .../testing/check_sql_parser_result.py        |   9 +
 .../src/datahub/utilities/sqlglot_lineage.py  |  92 ++++++++--
 .../test_bigquery_create_view_with_cte.json   |   8 +-
 ..._bigquery_from_sharded_table_wildcard.json |   4 +-
 .../test_bigquery_nested_subqueries.json      |   4 +-
 ..._bigquery_sharded_table_normalization.json |   4 +-
 .../test_bigquery_star_with_replace.json      |   6 +-
 .../test_bigquery_view_from_union.json        |   4 +-
 .../goldens/test_create_table_ddl.json        |  55 +++++-
 .../goldens/test_create_view_as_select.json   |   2 +-
 .../test_select_from_struct_subfields.json    |   2 +-
 .../test_select_with_full_col_name.json       |   2 +-
 .../test_teradata_default_normalization.json  |   2 +
 16 files changed, 430 insertions(+), 97 deletions(-)

diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
index 1a32b38ce055d..81d0a71b651d9 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
@@ -74,9 +74,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -98,7 +96,44 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -157,7 +192,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:24.632190+00:00",
+                "start_date": "2023-10-15 20:29:10.262813+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -172,7 +207,7 @@
             "name": "sqlite_operator_create_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056984632,
+                "time": 1697401750262,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -221,7 +256,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056984632,
+            "timestampMillis": 1697401750262,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -251,9 +286,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -275,7 +308,80 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -331,7 +437,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056984947,
+            "timestampMillis": 1697401750651,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -447,7 +553,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:28.605901+00:00",
+                "start_date": "2023-10-15 20:29:15.013834+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -462,7 +568,7 @@
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056988605,
+                "time": 1697401755013,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -511,7 +617,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056988605,
+            "timestampMillis": 1697401755013,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -621,7 +727,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056989098,
+            "timestampMillis": 1697401755600,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -807,7 +913,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:32.888165+00:00",
+                "start_date": "2023-10-15 20:29:20.216818+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -822,7 +928,7 @@
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056992888,
+                "time": 1697401760216,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -895,7 +1001,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056992888,
+            "timestampMillis": 1697401760216,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1131,7 +1237,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056993744,
+            "timestampMillis": 1697401761237,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1249,7 +1355,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:37.745717+00:00",
+                "start_date": "2023-10-15 20:29:26.243934+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1264,7 +1370,7 @@
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056997745,
+                "time": 1697401766243,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1313,7 +1419,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056997745,
+            "timestampMillis": 1697401766243,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1425,7 +1531,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056998672,
+            "timestampMillis": 1697401767373,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1543,7 +1649,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:56:42.645806+00:00",
+                "start_date": "2023-10-15 20:29:32.075613+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1558,7 +1664,7 @@
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057002645,
+                "time": 1697401772075,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1607,7 +1713,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057002645,
+            "timestampMillis": 1697401772075,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1719,7 +1825,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057003759,
+            "timestampMillis": 1697401773454,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
index c082be693e30c..96a0f02ccec17 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
@@ -74,9 +74,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -98,7 +96,44 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -157,7 +192,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:00:45.832554+00:00",
+                "start_date": "2023-10-15 20:27:26.883178+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -172,7 +207,7 @@
             "name": "sqlite_operator_create_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057245832,
+                "time": 1697401646883,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -221,7 +256,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057245832,
+            "timestampMillis": 1697401646883,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -251,9 +286,7 @@
                 "downstream_task_ids": "['populate_cost_table']",
                 "inlets": "[]",
                 "outlets": "[]",
-                "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}",
-                "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not <class 'sqlglot.expressions.Create'> (outer statement type: <class 'sqlglot.expressions.Create'>)\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE TABLE IF NOT EXISTS costs (\\n            id INTEGER PRIMARY KEY,\\n            month TEXT NOT NULL,\\n            total_cost REAL NOT NULL,\\n            area REAL NOT NULL\\n        )\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table",
             "name": "create_cost_table",
@@ -275,7 +308,80 @@
                 "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)"
             ],
             "inputDatajobs": [],
-            "fineGrainedLineages": []
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
         }
     }
 },
@@ -331,7 +437,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057246734,
+            "timestampMillis": 1697401647826,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -502,7 +608,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:00:49.653938+00:00",
+                "start_date": "2023-10-15 20:27:31.398799+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -517,7 +623,7 @@
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057249653,
+                "time": 1697401651398,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -566,7 +672,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057249653,
+            "timestampMillis": 1697401651398,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -676,7 +782,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057250831,
+            "timestampMillis": 1697401652651,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -917,7 +1023,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:00:53.989264+00:00",
+                "start_date": "2023-10-15 20:27:37.697995+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -932,7 +1038,7 @@
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057253989,
+                "time": 1697401657697,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1005,7 +1111,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057253989,
+            "timestampMillis": 1697401657697,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1241,7 +1347,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057255628,
+            "timestampMillis": 1697401659496,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1414,7 +1520,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:01:00.421177+00:00",
+                "start_date": "2023-10-15 20:27:45.670215+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1429,7 +1535,7 @@
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057260421,
+                "time": 1697401665670,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1478,7 +1584,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057260421,
+            "timestampMillis": 1697401665670,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1590,7 +1696,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057262258,
+            "timestampMillis": 1697401667670,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1763,7 +1869,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 07:01:05.540192+00:00",
+                "start_date": "2023-10-15 20:27:51.559194+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1778,7 +1884,7 @@
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057265540,
+                "time": 1697401671559,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1827,7 +1933,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057265540,
+            "timestampMillis": 1697401671559,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1939,7 +2045,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057267631,
+            "timestampMillis": 1697401673788,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
index 071d590f270f8..dedcfa0385f75 100644
--- a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
@@ -179,15 +179,16 @@ def add_lineage(
 
     def gen_workunits(self) -> Iterable[MetadataWorkUnit]:
         if self.generate_lineage:
-            yield from self._gen_lineage_workunits()
+            for mcp in self._gen_lineage_mcps():
+                yield mcp.as_workunit()
         if self.generate_usage_statistics:
             yield from self._gen_usage_statistics_workunits()
 
-    def _gen_lineage_workunits(self) -> Iterable[MetadataWorkUnit]:
+    def _gen_lineage_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
         for downstream_urn in self._lineage_map:
             upstreams: List[UpstreamClass] = []
             fine_upstreams: List[FineGrainedLineageClass] = []
-            for upstream_urn, edge in self._lineage_map[downstream_urn].items():
+            for edge in self._lineage_map[downstream_urn].values():
                 upstreams.append(edge.gen_upstream_aspect())
                 fine_upstreams.extend(edge.gen_fine_grained_lineage_aspects())
 
@@ -201,7 +202,7 @@ def _gen_lineage_workunits(self) -> Iterable[MetadataWorkUnit]:
             )
             yield MetadataChangeProposalWrapper(
                 entityUrn=downstream_urn, aspect=upstream_lineage
-            ).as_workunit()
+            )
 
     def _gen_usage_statistics_workunits(self) -> Iterable[MetadataWorkUnit]:
         yield from self._usage_aggregator.generate_workunits(
diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
index b3b1331db768b..2b610947e9043 100644
--- a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
+++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py
@@ -24,6 +24,7 @@ def assert_sql_result_with_resolver(
     *,
     expected_file: pathlib.Path,
     schema_resolver: SchemaResolver,
+    allow_table_error: bool = False,
     **kwargs: Any,
 ) -> None:
     # HACK: Our BigQuery source overwrites this value and doesn't undo it.
@@ -36,6 +37,14 @@ def assert_sql_result_with_resolver(
         **kwargs,
     )
 
+    if res.debug_info.table_error:
+        if allow_table_error:
+            logger.info(
+                f"SQL parser table error: {res.debug_info.table_error}",
+                exc_info=res.debug_info.table_error,
+            )
+        else:
+            raise res.debug_info.table_error
     if res.debug_info.column_error:
         logger.warning(
             f"SQL parser column error: {res.debug_info.column_error}",
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index c830ec8c02fd4..97121b368f507 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -241,9 +241,9 @@ class SqlParsingResult(_ParserBaseModel):
     )
 
 
-def _parse_statement(sql: str, dialect: str) -> sqlglot.Expression:
-    statement = sqlglot.parse_one(
-        sql, read=dialect, error_level=sqlglot.ErrorLevel.RAISE
+def _parse_statement(sql: sqlglot.exp.ExpOrStr, dialect: str) -> sqlglot.Expression:
+    statement: sqlglot.Expression = sqlglot.maybe_parse(
+        sql, dialect=dialect, error_level=sqlglot.ErrorLevel.RAISE
     )
     return statement
 
@@ -467,14 +467,20 @@ def _column_level_lineage(  # noqa: C901
     default_db: Optional[str],
     default_schema: Optional[str],
 ) -> List[_ColumnLineageInfo]:
-    if not isinstance(
-        statement,
-        _SupportedColumnLineageTypesTuple,
+    is_create_ddl = _is_create_table_ddl(statement)
+    if (
+        not isinstance(
+            statement,
+            _SupportedColumnLineageTypesTuple,
+        )
+        and not is_create_ddl
     ):
         raise UnsupportedStatementTypeError(
             f"Can only generate column-level lineage for select-like inner statements, not {type(statement)}"
         )
 
+    column_lineage: List[_ColumnLineageInfo] = []
+
     use_case_insensitive_cols = dialect in {
         # Column identifiers are case-insensitive in BigQuery, so we need to
         # do a normalization step beforehand to make sure it's resolved correctly.
@@ -580,6 +586,38 @@ def _schema_aware_fuzzy_column_resolve(
         ) from e
     logger.debug("Qualified sql %s", statement.sql(pretty=True, dialect=dialect))
 
+    # Handle the create DDL case.
+    if is_create_ddl:
+        assert (
+            output_table is not None
+        ), "output_table must be set for create DDL statements"
+
+        create_schema: sqlglot.exp.Schema = statement.this
+        sqlglot_columns = create_schema.expressions
+
+        for column_def in sqlglot_columns:
+            if not isinstance(column_def, sqlglot.exp.ColumnDef):
+                # Ignore things like constraints.
+                continue
+
+            output_col = _schema_aware_fuzzy_column_resolve(
+                output_table, column_def.name
+            )
+            output_col_type = column_def.args.get("kind")
+
+            column_lineage.append(
+                _ColumnLineageInfo(
+                    downstream=_DownstreamColumnRef(
+                        table=output_table,
+                        column=output_col,
+                        column_type=output_col_type,
+                    ),
+                    upstreams=[],
+                )
+            )
+
+        return column_lineage
+
     # Try to figure out the types of the output columns.
     try:
         statement = sqlglot.optimizer.annotate_types.annotate_types(
@@ -589,8 +627,6 @@ def _schema_aware_fuzzy_column_resolve(
         # This is not a fatal error, so we can continue.
         logger.debug("sqlglot failed to annotate types: %s", e)
 
-    column_lineage = []
-
     try:
         assert isinstance(statement, _SupportedColumnLineageTypesTuple)
 
@@ -599,7 +635,6 @@ def _schema_aware_fuzzy_column_resolve(
             (select_col.alias_or_name, select_col) for select_col in statement.selects
         ]
         logger.debug("output columns: %s", [col[0] for col in output_columns])
-        output_col: str
         for output_col, original_col_expression in output_columns:
             if output_col == "*":
                 # If schema information is available, the * will be expanded to the actual columns.
@@ -628,7 +663,7 @@ def _schema_aware_fuzzy_column_resolve(
 
             # Generate SELECT lineage.
             # Using a set here to deduplicate upstreams.
-            direct_col_upstreams: Set[_ColumnRef] = set()
+            direct_raw_col_upstreams: Set[_ColumnRef] = set()
             for node in lineage_node.walk():
                 if node.downstream:
                     # We only want the leaf nodes.
@@ -643,8 +678,9 @@ def _schema_aware_fuzzy_column_resolve(
                     if node.subfield:
                         normalized_col = f"{normalized_col}.{node.subfield}"
 
-                    col = _schema_aware_fuzzy_column_resolve(table_ref, normalized_col)
-                    direct_col_upstreams.add(_ColumnRef(table=table_ref, column=col))
+                    direct_raw_col_upstreams.add(
+                        _ColumnRef(table=table_ref, column=normalized_col)
+                    )
                 else:
                     # This branch doesn't matter. For example, a count(*) column would go here, and
                     # we don't get any column-level lineage for that.
@@ -665,7 +701,16 @@ def _schema_aware_fuzzy_column_resolve(
             if original_col_expression.type:
                 output_col_type = original_col_expression.type
 
-            if not direct_col_upstreams:
+            # Fuzzy resolve upstream columns.
+            direct_resolved_col_upstreams = {
+                _ColumnRef(
+                    table=edge.table,
+                    column=_schema_aware_fuzzy_column_resolve(edge.table, edge.column),
+                )
+                for edge in direct_raw_col_upstreams
+            }
+
+            if not direct_resolved_col_upstreams:
                 logger.debug(f'  "{output_col}" has no upstreams')
             column_lineage.append(
                 _ColumnLineageInfo(
@@ -674,12 +719,12 @@ def _schema_aware_fuzzy_column_resolve(
                         column=output_col,
                         column_type=output_col_type,
                     ),
-                    upstreams=sorted(direct_col_upstreams),
+                    upstreams=sorted(direct_resolved_col_upstreams),
                     # logic=column_logic.sql(pretty=True, dialect=dialect),
                 )
             )
 
-        # TODO: Also extract referenced columns (e.g. non-SELECT lineage)
+        # TODO: Also extract referenced columns (aka auxillary / non-SELECT lineage)
     except (sqlglot.errors.OptimizeError, ValueError) as e:
         raise SqlUnderstandingError(
             f"sqlglot failed to compute some lineage: {e}"
@@ -700,6 +745,12 @@ def _extract_select_from_create(
         return statement
 
 
+def _is_create_table_ddl(statement: sqlglot.exp.Expression) -> bool:
+    return isinstance(statement, sqlglot.exp.Create) and isinstance(
+        statement.this, sqlglot.exp.Schema
+    )
+
+
 def _try_extract_select(
     statement: sqlglot.exp.Expression,
 ) -> sqlglot.exp.Expression:
@@ -766,6 +817,7 @@ def _translate_sqlglot_type(
 def _translate_internal_column_lineage(
     table_name_urn_mapping: Dict[_TableName, str],
     raw_column_lineage: _ColumnLineageInfo,
+    dialect: str,
 ) -> ColumnLineageInfo:
     downstream_urn = None
     if raw_column_lineage.downstream.table:
@@ -779,7 +831,9 @@ def _translate_internal_column_lineage(
             )
             if raw_column_lineage.downstream.column_type
             else None,
-            native_column_type=raw_column_lineage.downstream.column_type.sql()
+            native_column_type=raw_column_lineage.downstream.column_type.sql(
+                dialect=dialect
+            )
             if raw_column_lineage.downstream.column_type
             and raw_column_lineage.downstream.column_type.this
             != sqlglot.exp.DataType.Type.UNKNOWN
@@ -800,12 +854,14 @@ def _get_dialect(platform: str) -> str:
     # TODO: convert datahub platform names to sqlglot dialect
     if platform == "presto-on-hive":
         return "hive"
+    if platform == "mssql":
+        return "tsql"
     else:
         return platform
 
 
 def _sqlglot_lineage_inner(
-    sql: str,
+    sql: sqlglot.exp.ExpOrStr,
     schema_resolver: SchemaResolver,
     default_db: Optional[str] = None,
     default_schema: Optional[str] = None,
@@ -918,7 +974,7 @@ def _sqlglot_lineage_inner(
     if column_lineage:
         column_lineage_urns = [
             _translate_internal_column_lineage(
-                table_name_urn_mapping, internal_col_lineage
+                table_name_urn_mapping, internal_col_lineage, dialect=dialect
             )
             for internal_col_lineage in column_lineage
         ]
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
index f0175b4dc8892..d610b0a83f229 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json
@@ -18,7 +18,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -36,7 +36,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -54,7 +54,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -72,7 +72,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
index b7df5444987f2..2d3d188d28316 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -32,7 +32,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
index 67e306bebf545..41ae0885941b0 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -32,7 +32,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
index b7df5444987f2..2d3d188d28316 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -32,7 +32,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
index b393b2445d6c4..26f8f8f59a3ff 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json
@@ -16,7 +16,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -34,7 +34,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -52,7 +52,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
index 53fb94300e804..83365c09f69c2 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
@@ -17,7 +17,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
@@ -39,7 +39,7 @@
                         "com.linkedin.pegasus2avro.schema.StringType": {}
                     }
                 },
-                "native_column_type": "TEXT"
+                "native_column_type": "STRING"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json
index 4773974545bfa..cf31b71cb50f6 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_table_ddl.json
@@ -4,5 +4,58 @@
     "out_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)"
     ],
-    "column_lineage": null
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "id",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INTEGER"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "month",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "TEXT"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "total_cost",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "REAL"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:sqlite,costs,PROD)",
+                "column": "area",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "REAL"
+            },
+            "upstreams": []
+        }
+    ]
 }
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
index ff452467aa5bd..8a6b60d0f1bde 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json
@@ -30,7 +30,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "BIGINT"
+                "native_column_type": "NUMBER"
             },
             "upstreams": []
         },
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
index 5ad847e252497..2424fcda34752 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "NUMERIC"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
index 6ee3d2e61c39b..8dd2633eff612 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json
@@ -14,7 +14,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "NUMERIC"
             },
             "upstreams": [
                 {
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
index b0351a7e07ad2..ee80285d87f60 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_default_normalization.json
@@ -12,6 +12,7 @@
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)",
                 "column": "PatientId",
+                "column_type": null,
                 "native_column_type": "INTEGER()"
             },
             "upstreams": [
@@ -25,6 +26,7 @@
             "downstream": {
                 "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,myteradata.demo_user.test_lineage2,PROD)",
                 "column": "BMI",
+                "column_type": null,
                 "native_column_type": "FLOAT()"
             },
             "upstreams": [

From 10456c5e3cdaad14927b89bb9deee1a6df0ce92c Mon Sep 17 00:00:00 2001
From: Ellie O'Neil <110510035+eboneil@users.noreply.github.com>
Date: Mon, 23 Oct 2023 14:53:07 -0700
Subject: [PATCH 083/259] docs(ingest): update to get_workunits_internal
 (#9054)

---
 metadata-ingestion/adding-source.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/adding-source.md b/metadata-ingestion/adding-source.md
index e4fc950a7cdbd..a0930102c6827 100644
--- a/metadata-ingestion/adding-source.md
+++ b/metadata-ingestion/adding-source.md
@@ -62,7 +62,7 @@ Some sources use the default `SourceReport` class, but others inherit and extend
 
 ### 3. Implement the source itself
 
-The core for the source is the `get_workunits` method, which produces a stream of metadata events (typically MCP objects) wrapped up in a MetadataWorkUnit.
+The core for the source is the `get_workunits_internal` method, which produces a stream of metadata events (typically MCP objects) wrapped up in a MetadataWorkUnit.
 The [file source](./src/datahub/ingestion/source/file.py) is a good and simple example.
 
 The MetadataChangeEventClass is defined in the metadata models which are generated

From a0ce4f333e1cbbc544a650ec3e8012a1f10aef2b Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Mon, 23 Oct 2023 21:21:21 -0400
Subject: [PATCH 084/259] Column level lineage and path test (#8822)

---
 .../preview/EntityPaths/EntityPathsModal.tsx  |  1 +
 .../e2e/lineage/lineage_column_path.js        | 68 +++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js

diff --git a/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx b/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx
index d5722429aaf6b..2bb76714d6119 100644
--- a/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx
+++ b/datahub-web-react/src/app/preview/EntityPaths/EntityPathsModal.tsx
@@ -39,6 +39,7 @@ export default function EntityPathsModal({ paths, resultEntityUrn, hideModal }:
 
     return (
         <StyledModal
+            data-testid="entity-paths-modal"
             title={
                 <Header>
                     Column path{paths.length > 1 && 's'} from{' '}
diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js
new file mode 100644
index 0000000000000..37ca62c8d1229
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/lineage_column_path.js
@@ -0,0 +1,68 @@
+import { aliasQuery } from "../utils";
+const DATASET_ENTITY_TYPE = 'dataset';
+const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)';
+const DOWNSTREAM_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)";
+const upstreamColumn = '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)-Upstream"] text';
+const downstreamColumn = '[data-testid="node-urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)-Downstream"] text';
+
+const verifyColumnPathModal = (from, to) => {
+  cy.get('[data-testid="entity-paths-modal"]').contains(from).should("be.visible");
+  cy.get('[data-testid="entity-paths-modal"]').contains(to).should("be.visible");
+};
+
+describe("column-Level lineage and impact analysis path test", () => {
+    beforeEach(() => {
+        cy.on('uncaught:exception', (err, runnable) => { return false; });
+        cy.intercept("POST", "/api/v2/graphql", (req) => {
+          aliasQuery(req, "appConfig");
+        });
+      });
+
+    it("verify column-level lineage path at lineage praph and impact analysis ", () => {
+      // Open dataset with column-level lineage configured an navigate to lineage tab -> visualize lineage
+      cy.loginWithCredentials();
+      cy.goToEntityLineageGraph(DATASET_ENTITY_TYPE, DATASET_URN);
+
+      // Enable “show columns” toggle
+      cy.waitTextVisible("SampleCypressHdfs");
+      cy.clickOptionWithTestId("column-toggle");
+      cy.waitTextVisible("shipment_info");
+
+      // Verify functionality of column lineage
+      cy.get(upstreamColumn).eq(3).click();
+      cy.get(upstreamColumn).eq(3).prev().should('not.have.attr', 'fill', 'white');
+      cy.get(downstreamColumn).eq(2).prev().should('not.have.attr', 'stroke', 'transparent');
+      cy.get(downstreamColumn).eq(2).click();
+      cy.get(downstreamColumn).eq(2).prev().should('not.have.attr', 'fill', 'white');
+      cy.get(upstreamColumn).eq(3).prev().should('not.have.attr', 'stroke', 'transparent');
+
+      // Open dataset impact analysis view, enable column lineage
+      cy.goToDataset(DATASET_URN, "SampleCypressHdfsDataset");
+      cy.openEntityTab("Lineage");
+      cy.clickOptionWithText("Column Lineage");
+      cy.clickOptionWithText("Downstream");
+
+      // Verify upstream column lineage, test column path modal
+      cy.clickOptionWithText("Upstream");
+      cy.waitTextVisible("SampleCypressKafkaDataset");
+      cy.ensureTextNotPresent("field_bar");
+      cy.contains("Select column").click({ force: true}).wait(1000);
+      cy.get(".rc-virtual-list").contains("shipment_info").click(); 
+      cy.waitTextVisible("field_bar");
+      cy.clickOptionWithText("field_bar");
+      verifyColumnPathModal("shipment_info", "field_bar");
+      cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click();
+    
+      // Verify downstream column lineage, test column path modal
+      cy.goToDataset(DOWNSTREAM_DATASET_URN, "SampleCypressKafkaDataset");
+      cy.openEntityTab("Lineage");
+      cy.clickOptionWithText("Column Lineage");
+      cy.ensureTextNotPresent("shipment_info");
+      cy.contains("Select column").click({ force: true}).wait(1000);
+      cy.get(".rc-virtual-list").contains("field_bar").click(); 
+      cy.waitTextVisible("shipment_info");
+      cy.clickOptionWithText("shipment_info");
+      verifyColumnPathModal("shipment_info", "field_bar");
+      cy.get('[data-testid="entity-paths-modal"] [data-icon="close"]').click();
+    });
+}); 
\ No newline at end of file

From adf8c8db38c56250cb612b208f6e59b04c7258c6 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 24 Oct 2023 02:59:56 -0400
Subject: [PATCH 085/259] refactor(ingest): Move sqlalchemy import out of
 sql_types.py (#9065)

---
 .../src/datahub/ingestion/source/sql/athena.py           | 2 +-
 .../src/datahub/ingestion/source/sql/sql_common.py       | 2 +-
 .../src/datahub/ingestion/source/sql/sql_types.py        | 9 +--------
 .../src/datahub/utilities/sqlalchemy_type_converter.py   | 6 +++++-
 metadata-ingestion/tests/unit/test_athena_source.py      | 2 +-
 .../unit/utilities/test_sqlalchemy_type_converter.py     | 2 +-
 6 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
index dad61e5173166..06b9ad92677a2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
@@ -31,7 +31,6 @@
     register_custom_type,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, make_sqlalchemy_uri
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     gen_database_container,
@@ -41,6 +40,7 @@
 from datahub.metadata.schema_classes import RecordTypeClass
 from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
 from datahub.utilities.sqlalchemy_type_converter import (
+    MapType,
     get_schema_fields_for_sqlalchemy_column,
 )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index 6524eea8222d4..be03858ec3ef9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -37,7 +37,6 @@
     DatasetSubTypes,
 )
 from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.ingestion.source.sql.sql_utils import (
     add_table_to_schema_container,
     downgrade_schema_from_v2,
@@ -91,6 +90,7 @@
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.registries.domain_registry import DomainRegistry
 from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport
+from datahub.utilities.sqlalchemy_type_converter import MapType
 
 if TYPE_CHECKING:
     from datahub.ingestion.source.ge_data_profiler import (
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
index 51626891e9fef..ae47623188f42 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
@@ -1,8 +1,6 @@
 import re
 from typing import Any, Dict, ValuesView
 
-from sqlalchemy import types
-
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     ArrayType,
     BooleanType,
@@ -17,6 +15,7 @@
     TimeType,
     UnionType,
 )
+from datahub.utilities.sqlalchemy_type_converter import MapType
 
 # these can be obtained by running `select format_type(oid, null),* from pg_type;`
 # we've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.)
@@ -369,12 +368,6 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "array": ArrayType,
 }
 
-
-class MapType(types.TupleType):
-    # Wrapper class around SQLalchemy's TupleType to increase compatibility with DataHub
-    pass
-
-
 # https://docs.aws.amazon.com/athena/latest/ug/data-types.html
 # https://github.com/dbt-athena/dbt-athena/tree/main
 ATHENA_SQL_TYPES_MAP: Dict[str, Any] = {
diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
index a431f262a85fd..1d5ec5dae3519 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
@@ -7,13 +7,17 @@
 from sqlalchemy_bigquery import STRUCT
 
 from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
 from datahub.metadata.schema_classes import NullTypeClass, SchemaFieldDataTypeClass
 
 logger = logging.getLogger(__name__)
 
 
+class MapType(types.TupleType):
+    # Wrapper class around SQLalchemy's TupleType to increase compatibility with DataHub
+    pass
+
+
 class SqlAlchemyColumnToAvroConverter:
     """Helper class that collects some methods to convert SQLalchemy columns to Avro schema."""
 
diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py
index 6d3ed20eafde2..23dd7dd5a6e45 100644
--- a/metadata-ingestion/tests/unit/test_athena_source.py
+++ b/metadata-ingestion/tests/unit/test_athena_source.py
@@ -9,7 +9,7 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.aws.s3_util import make_s3_urn
 from datahub.ingestion.source.sql.athena import CustomAthenaRestDialect
-from datahub.ingestion.source.sql.sql_types import MapType
+from datahub.utilities.sqlalchemy_type_converter import MapType
 
 FROZEN_TIME = "2020-04-14 07:00:00"
 
diff --git a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
index 959da0987a825..6c719d351c4c2 100644
--- a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
+++ b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py
@@ -3,7 +3,6 @@
 from sqlalchemy import types
 from sqlalchemy_bigquery import STRUCT
 
-from datahub.ingestion.source.sql.sql_types import MapType
 from datahub.metadata.schema_classes import (
     ArrayTypeClass,
     MapTypeClass,
@@ -12,6 +11,7 @@
     RecordTypeClass,
 )
 from datahub.utilities.sqlalchemy_type_converter import (
+    MapType,
     get_schema_fields_for_sqlalchemy_column,
 )
 

From c849246e63284bc73768ed58a22be62b708a6c48 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 24 Oct 2023 00:09:41 -0700
Subject: [PATCH 086/259] fix(ingest): add releases link (#9014)

---
 metadata-ingestion/setup.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index c46409ecbf52f..417588a433655 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -282,7 +282,8 @@
     # Source plugins
     # sqlalchemy-bigquery is included here since it provides an implementation of
     # a SQLalchemy-conform STRUCT type definition
-    "athena": sql_common | {"PyAthena[SQLAlchemy]>=2.6.0,<3.0.0", "sqlalchemy-bigquery>=1.4.1"},
+    "athena": sql_common
+    | {"PyAthena[SQLAlchemy]>=2.6.0,<3.0.0", "sqlalchemy-bigquery>=1.4.1"},
     "azure-ad": set(),
     "bigquery": sql_common
     | bigquery_common
@@ -354,7 +355,11 @@
     | {"psycopg2-binary", "pymysql>=1.0.2"},
     "pulsar": {"requests"},
     "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib,
-    "redshift": sql_common | redshift_common | usage_common | sqlglot_lib | {"redshift-connector"},
+    "redshift": sql_common
+    | redshift_common
+    | usage_common
+    | sqlglot_lib
+    | {"redshift-connector"},
     "redshift-legacy": sql_common | redshift_common,
     "redshift-usage-legacy": sql_common | usage_common | redshift_common,
     "s3": {*s3_base, *data_lake_profiling},
@@ -435,7 +440,9 @@
 deepdiff_dep = "deepdiff"
 test_api_requirements = {pytest_dep, deepdiff_dep, "PyYAML"}
 
-debug_requirements = {"memray"}
+debug_requirements = {
+    "memray",
+}
 
 base_dev_requirements = {
     *base_requirements,
@@ -668,6 +675,7 @@
         "Documentation": "https://datahubproject.io/docs/",
         "Source": "https://github.com/datahub-project/datahub",
         "Changelog": "https://github.com/datahub-project/datahub/releases",
+        "Releases": "https://github.com/acryldata/datahub/releases",
     },
     license="Apache License 2.0",
     description="A CLI to work with DataHub metadata",

From eb0b03d2f2f2c9ce88562c32d968d095a59f8547 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 24 Oct 2023 10:45:09 -0400
Subject: [PATCH 087/259] fix(ingest/bigquery): Correctly apply table pattern
 to read events; fix end time calculation; deprecate
 match_fully_qualified_names (#9077)

---
 .../ingestion/source/bigquery_v2/bigquery_config.py    |  7 +++----
 .../datahub/ingestion/source/bigquery_v2/lineage.py    |  2 +-
 .../src/datahub/ingestion/source/bigquery_v2/usage.py  | 10 +++++++---
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index 944814b6936a4..a6a740385cf5c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -119,8 +119,8 @@ class BigQueryV2Config(
     )
 
     match_fully_qualified_names: bool = Field(
-        default=False,
-        description="Whether `dataset_pattern` is matched against fully qualified dataset name `<project_id>.<dataset_name>`.",
+        default=True,
+        description="[deprecated] Whether `dataset_pattern` is matched against fully qualified dataset name `<project_id>.<dataset_name>`.",
     )
 
     include_external_url: bool = Field(
@@ -327,8 +327,7 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
         ):
             logger.warning(
                 "Please update `dataset_pattern` to match against fully qualified schema name `<project_id>.<dataset_name>` and set config `match_fully_qualified_names : True`."
-                "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. "
-                "The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`."
+                "The config option `match_fully_qualified_names` is deprecated and will be removed in a future release."
             )
         return values
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
index 98c8cbaf85eec..aa462435b8105 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
@@ -548,7 +548,7 @@ def _get_parsed_audit_log_events(self, project_id: str) -> Iterable[QueryEvent]:
         # handle the case where the read happens within our time range but the query
         # completion event is delayed and happens after the configured end time.
         corrected_start_time = self.start_time - self.config.max_query_duration
-        corrected_end_time = self.end_time + -self.config.max_query_duration
+        corrected_end_time = self.end_time + self.config.max_query_duration
         self.report.log_entry_start_time = corrected_start_time
         self.report.log_entry_end_time = corrected_end_time
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
index 201567e104a51..7fc38991e5928 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
@@ -335,8 +335,12 @@ def get_time_window(self) -> Tuple[datetime, datetime]:
     def _is_table_allowed(self, table_ref: Optional[BigQueryTableRef]) -> bool:
         return (
             table_ref is not None
-            and self.config.dataset_pattern.allowed(table_ref.table_identifier.dataset)
-            and self.config.table_pattern.allowed(table_ref.table_identifier.table)
+            and self.config.dataset_pattern.allowed(
+                f"{table_ref.table_identifier.project_id}.{table_ref.table_identifier.dataset}"
+                if self.config.match_fully_qualified_names
+                else table_ref.table_identifier.dataset
+            )
+            and self.config.table_pattern.allowed(str(table_ref.table_identifier))
         )
 
     def _should_ingest_usage(self) -> bool:
@@ -844,7 +848,7 @@ def _get_parsed_bigquery_log_events(
         # handle the case where the read happens within our time range but the query
         # completion event is delayed and happens after the configured end time.
         corrected_start_time = self.start_time - self.config.max_query_duration
-        corrected_end_time = self.end_time + -self.config.max_query_duration
+        corrected_end_time = self.end_time + self.config.max_query_duration
         self.report.audit_start_time = corrected_start_time
         self.report.audit_end_time = corrected_end_time
 

From d13553f53ad9e7592256cd88e78eef0ca95832e4 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 24 Oct 2023 12:24:50 -0700
Subject: [PATCH 088/259] feat(sqlparser): extract CLL from `update`s (#9078)

---
 .../src/datahub/utilities/sqlglot_lineage.py  | 68 +++++++++++--
 .../test_snowflake_update_from_table.json     | 56 +++++++++++
 .../test_snowflake_update_hardcoded.json      | 35 +++++++
 .../unit/sql_parsing/test_sqlglot_lineage.py  | 96 +++++++++++++++++++
 4 files changed, 246 insertions(+), 9 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json

diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 97121b368f507..526d90b2a1bfa 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -745,6 +745,47 @@ def _extract_select_from_create(
         return statement
 
 
+_UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT: Set[str] = set(
+    sqlglot.exp.Update.arg_types.keys()
+) - set(sqlglot.exp.Select.arg_types.keys())
+
+
+def _extract_select_from_update(
+    statement: sqlglot.exp.Update,
+) -> sqlglot.exp.Select:
+    statement = statement.copy()
+
+    # The "SET" expressions need to be converted.
+    # For the update command, it'll be a list of EQ expressions, but the select
+    # should contain aliased columns.
+    new_expressions = []
+    for expr in statement.expressions:
+        if isinstance(expr, sqlglot.exp.EQ) and isinstance(
+            expr.left, sqlglot.exp.Column
+        ):
+            new_expressions.append(
+                sqlglot.exp.Alias(
+                    this=expr.right,
+                    alias=expr.left.this,
+                )
+            )
+        else:
+            # If we don't know how to convert it, just leave it as-is. If this causes issues,
+            # they'll get caught later.
+            new_expressions.append(expr)
+
+    return sqlglot.exp.Select(
+        **{
+            **{
+                k: v
+                for k, v in statement.args.items()
+                if k not in _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
+            },
+            "expressions": new_expressions,
+        }
+    )
+
+
 def _is_create_table_ddl(statement: sqlglot.exp.Expression) -> bool:
     return isinstance(statement, sqlglot.exp.Create) and isinstance(
         statement.this, sqlglot.exp.Schema
@@ -767,6 +808,9 @@ def _try_extract_select(
     elif isinstance(statement, sqlglot.exp.Insert):
         # TODO Need to map column renames in the expressions part of the statement.
         statement = statement.expression
+    elif isinstance(statement, sqlglot.exp.Update):
+        # Assumption: the output table is already captured in the modified tables list.
+        statement = _extract_select_from_update(statement)
     elif isinstance(statement, sqlglot.exp.Create):
         # TODO May need to map column renames.
         # Assumption: the output table is already captured in the modified tables list.
@@ -942,19 +986,25 @@ def _sqlglot_lineage_inner(
     )
 
     # Simplify the input statement for column-level lineage generation.
-    select_statement = _try_extract_select(statement)
+    try:
+        select_statement = _try_extract_select(statement)
+    except Exception as e:
+        logger.debug(f"Failed to extract select from statement: {e}", exc_info=True)
+        debug_info.column_error = e
+        select_statement = None
 
     # Generate column-level lineage.
     column_lineage: Optional[List[_ColumnLineageInfo]] = None
     try:
-        column_lineage = _column_level_lineage(
-            select_statement,
-            dialect=dialect,
-            input_tables=table_name_schema_mapping,
-            output_table=downstream_table,
-            default_db=default_db,
-            default_schema=default_schema,
-        )
+        if select_statement is not None:
+            column_lineage = _column_level_lineage(
+                select_statement,
+                dialect=dialect,
+                input_tables=table_name_schema_mapping,
+                output_table=downstream_table,
+                default_db=default_db,
+                default_schema=default_schema,
+            )
     except UnsupportedStatementTypeError as e:
         # Inject details about the outer statement type too.
         e.args = (f"{e.args[0]} (outer statement type: {type(statement)})",)
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
new file mode 100644
index 0000000000000..e2baa34e7fe28
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
@@ -0,0 +1,56 @@
+{
+    "query_type": "UPDATE",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)"
+    ],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+                    "column": "col1"
+                },
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+                    "column": "col2"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+                    "column": "col1"
+                },
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)",
+                    "column": "col2"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
new file mode 100644
index 0000000000000..b41ed61b37cdb
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
@@ -0,0 +1,35 @@
+{
+    "query_type": "UPDATE",
+    "in_tables": [],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                "column": "orderkey",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": []
+        },
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                "column": "totalprice",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": []
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index 059add8db67e4..dfc5b486abd35 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -3,6 +3,7 @@
 import pytest
 
 from datahub.testing.check_sql_parser_result import assert_sql_result
+from datahub.utilities.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
 
 RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens"
 
@@ -672,3 +673,98 @@ def test_teradata_default_normalization():
         },
         expected_file=RESOURCE_DIR / "test_teradata_default_normalization.json",
     )
+
+
+def test_snowflake_update_hardcoded():
+    assert_sql_result(
+        """
+UPDATE snowflake_sample_data.tpch_sf1.orders
+SET orderkey = 1, totalprice = 2
+WHERE orderkey = 3
+""",
+        dialect="snowflake",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": {
+                "orderkey": "NUMBER(38,0)",
+                "totalprice": "NUMBER(12,2)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_update_hardcoded.json",
+    )
+
+
+def test_update_from_select():
+    assert _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT == {"returning", "this"}
+
+
+def test_snowflake_update_from_table():
+    # Can create these tables with the following SQL:
+    """
+    -- Create or replace my_table
+    CREATE OR REPLACE TABLE my_table (
+        id INT IDENTITY PRIMARY KEY,
+        col1 VARCHAR(50),
+        col2 VARCHAR(50)
+    );
+
+    -- Create or replace table1
+    CREATE OR REPLACE TABLE table1 (
+        id INT IDENTITY PRIMARY KEY,
+        col1 VARCHAR(50),
+        col2 VARCHAR(50)
+    );
+
+    -- Create or replace table2
+    CREATE OR REPLACE TABLE table2 (
+        id INT IDENTITY PRIMARY KEY,
+        col2 VARCHAR(50)
+    );
+
+    -- Insert data into my_table
+    INSERT INTO my_table (col1, col2)
+    VALUES ('foo', 'bar'),
+           ('baz', 'qux');
+
+    -- Insert data into table1
+    INSERT INTO table1 (col1, col2)
+    VALUES ('foo', 'bar'),
+           ('baz', 'qux');
+
+    -- Insert data into table2
+    INSERT INTO table2 (col2)
+    VALUES ('bar'),
+           ('qux');
+    """
+
+    assert_sql_result(
+        """
+UPDATE my_table
+SET
+    col1 = t1.col1 || t1.col2,
+    col2 = t1.col1 || t2.col2
+FROM table1 t1
+JOIN table2 t2 ON t1.id = t2.id
+WHERE my_table.id = t1.id;
+""",
+        dialect="snowflake",
+        default_db="my_db",
+        default_schema="my_schema",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)": {
+                "id": "NUMBER(38,0)",
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)": {
+                "id": "NUMBER(38,0)",
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)": {
+                "id": "NUMBER(38,0)",
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_update_from_table.json",
+    )

From 378d84a346cff4061f795dd1b296bde3ea5313c1 Mon Sep 17 00:00:00 2001
From: skrydal <piotr.skrydalewicz@gmail.com>
Date: Tue, 24 Oct 2023 22:12:11 +0200
Subject: [PATCH 089/259] fix(ui): Fixes handling of resources filters in UI
 (#9087)

---
 .../app/permissions/policy/PolicyDetailsModal.tsx |  4 ++--
 .../permissions/policy/PolicyPrivilegeForm.tsx    | 15 ++++++---------
 .../src/app/permissions/policy/policyUtils.ts     |  4 ++--
 docs/authorization/policies.md                    |  8 ++++----
 metadata-ingestion/tests/unit/serde/test_serde.py |  8 ++++----
 5 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
index 68e91983babdb..d3e01df3a66e8 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx
@@ -67,8 +67,8 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege
     const isMetadataPolicy = policy?.type === PolicyType.Metadata;
 
     const resources = convertLegacyResourceFilter(policy?.resources);
-    const resourceTypes = getFieldValues(resources?.filter, 'RESOURCE_TYPE') || [];
-    const resourceEntities = getFieldValues(resources?.filter, 'RESOURCE_URN') || [];
+    const resourceTypes = getFieldValues(resources?.filter, 'TYPE') || [];
+    const resourceEntities = getFieldValues(resources?.filter, 'URN') || [];
     const domains = getFieldValues(resources?.filter, 'DOMAIN') || [];
 
     const {
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
index 1520388a5033a..b8e1505fceaec 100644
--- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
+++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
@@ -67,8 +67,8 @@ export default function PolicyPrivilegeForm({
     } = useAppConfig();
 
     const resources: ResourceFilter = convertLegacyResourceFilter(maybeResources) || EMPTY_POLICY.resources;
-    const resourceTypes = getFieldValues(resources.filter, 'RESOURCE_TYPE') || [];
-    const resourceEntities = getFieldValues(resources.filter, 'RESOURCE_URN') || [];
+    const resourceTypes = getFieldValues(resources.filter, 'TYPE') || [];
+    const resourceEntities = getFieldValues(resources.filter, 'URN') || [];
 
     const getDisplayName = (entity) => {
         if (!entity) {
@@ -145,10 +145,7 @@ export default function PolicyPrivilegeForm({
         };
         setResources({
             ...resources,
-            filter: setFieldValues(filter, 'RESOURCE_TYPE', [
-                ...resourceTypes,
-                createCriterionValue(selectedResourceType),
-            ]),
+            filter: setFieldValues(filter, 'TYPE', [...resourceTypes, createCriterionValue(selectedResourceType)]),
         });
     };
 
@@ -160,7 +157,7 @@ export default function PolicyPrivilegeForm({
             ...resources,
             filter: setFieldValues(
                 filter,
-                'RESOURCE_TYPE',
+                'TYPE',
                 resourceTypes?.filter((criterionValue) => criterionValue.value !== deselectedResourceType),
             ),
         });
@@ -173,7 +170,7 @@ export default function PolicyPrivilegeForm({
         };
         setResources({
             ...resources,
-            filter: setFieldValues(filter, 'RESOURCE_URN', [
+            filter: setFieldValues(filter, 'URN', [
                 ...resourceEntities,
                 createCriterionValueWithEntity(
                     resource,
@@ -192,7 +189,7 @@ export default function PolicyPrivilegeForm({
             ...resources,
             filter: setFieldValues(
                 filter,
-                'RESOURCE_URN',
+                'URN',
                 resourceEntities?.filter((criterionValue) => criterionValue.value !== resource),
             ),
         });
diff --git a/datahub-web-react/src/app/permissions/policy/policyUtils.ts b/datahub-web-react/src/app/permissions/policy/policyUtils.ts
index c7af7342f6efa..2f178fcdeb5c3 100644
--- a/datahub-web-react/src/app/permissions/policy/policyUtils.ts
+++ b/datahub-web-react/src/app/permissions/policy/policyUtils.ts
@@ -99,10 +99,10 @@ export const convertLegacyResourceFilter = (resourceFilter: Maybe<ResourceFilter
     }
     const criteria = new Array<PolicyMatchCriterion>();
     if (resourceFilter.type) {
-        criteria.push(createCriterion('RESOURCE_TYPE', [createCriterionValue(resourceFilter.type)]));
+        criteria.push(createCriterion('TYPE', [createCriterionValue(resourceFilter.type)]));
     }
     if (resourceFilter.resources && resourceFilter.resources.length > 0) {
-        criteria.push(createCriterion('RESOURCE_URN', resourceFilter.resources.map(createCriterionValue)));
+        criteria.push(createCriterion('URN', resourceFilter.resources.map(createCriterionValue)));
     }
     return {
         filter: {
diff --git a/docs/authorization/policies.md b/docs/authorization/policies.md
index e3606f2a3e48d..63aa6688d3eec 100644
--- a/docs/authorization/policies.md
+++ b/docs/authorization/policies.md
@@ -137,7 +137,7 @@ We currently support the following:
 #### Resources
 
 Resource filter defines the set of resources that the policy applies to is defined using a list of criteria. Each
-criterion defines a field type (like resource_type, resource_urn, domain), a list of field values to compare, and a
+criterion defines a field type (like type, urn, domain), a list of field values to compare, and a
 condition (like EQUALS). It essentially checks whether the field of a certain resource matches any of the input values.
 Note, that if there are no criteria or resource is not set, policy is applied to ALL resources.
 
@@ -149,7 +149,7 @@ For example, the following resource filter will apply the policy to datasets, ch
       "filter": {
         "criteria": [
           {
-            "field": "RESOURCE_TYPE",
+            "field": "TYPE",
             "condition": "EQUALS",
             "values": [
               "dataset",
@@ -175,8 +175,8 @@ Supported fields are as follows
 
 | Field Type    | Description            | Example                 |
 |---------------|------------------------|-------------------------|
-| resource_type | Type of the resource   | dataset, chart, dataJob |
-| resource_urn  | Urn of the resource    | urn:li:dataset:...      |
+| type | Type of the resource   | dataset, chart, dataJob |
+| urn  | Urn of the resource    | urn:li:dataset:...      |
 | domain        | Domain of the resource | urn:li:domain:domainX   |
 
 ## Managing Policies
diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py
index d116f1f5473fa..d2d6a0bdda5b9 100644
--- a/metadata-ingestion/tests/unit/serde/test_serde.py
+++ b/metadata-ingestion/tests/unit/serde/test_serde.py
@@ -238,7 +238,7 @@ def test_missing_optional_simple() -> None:
                 "criteria": [
                     {
                         "condition": "EQUALS",
-                        "field": "RESOURCE_TYPE",
+                        "field": "TYPE",
                         "values": ["notebook", "dataset", "dashboard"],
                     }
                 ]
@@ -252,7 +252,7 @@ def test_missing_optional_simple() -> None:
             "criteria": [
                 {
                     "condition": "EQUALS",
-                    "field": "RESOURCE_TYPE",
+                    "field": "TYPE",
                     "values": ["notebook", "dataset", "dashboard"],
                 }
             ]
@@ -267,13 +267,13 @@ def test_missing_optional_simple() -> None:
 def test_missing_optional_in_union() -> None:
     # This one doesn't contain any optional fields and should work fine.
     revised_json = json.loads(
-        '{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}'
+        '{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}'
     )
     revised = models.DataHubPolicyInfoClass.from_obj(revised_json)
 
     # This one is missing the optional filters.allResources field.
     original_json = json.loads(
-        '{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}'
+        '{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}'
     )
     original = models.DataHubPolicyInfoClass.from_obj(original_json)
 

From edb82ad91fba8a401c56b82bc4c2916a39a6a6dd Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 24 Oct 2023 18:56:14 -0400
Subject: [PATCH 090/259] docs(ingest/bigquery): Add docs for breaking change:
 match_fully_qualified_names (#9094)

---
 docs/how/updating-datahub.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 3af3b2bdda215..7d8c25b06255a 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -11,11 +11,17 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
   by Looker and LookML source connectors.
 - #8853 - The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7. See the docs for more details.
 - #8853 - Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include `pip install 'acryl-datahub-airflow-plugin[plugin-v2]'`. To continue using the v1 plugin, set the `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN` environment variable to `true`.
-- #8943 The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
+- #8943 - The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled.
 This is currently enabled by default to preserve compatibility, but will be disabled by default and then removed in the future.
 If stateful ingestion is enabled, simply setting `include_metastore: false` will perform all required cleanup.
 Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
 `datahub delete --platform databricks --soft` and then reingesting with `include_metastore: false`.
+- #9077 - The BigQuery ingestion source by default sets `match_fully_qualified_names: true`.
+This means that any `dataset_pattern` or `schema_pattern` specified will be matched on the fully
+qualified dataset name, i.e. `<project_name>.<dataset_name>`. If this is not the case, please
+update your pattern (e.g. prepend your old dataset pattern with `.*\.` which matches the project part), 
+or set `match_fully_qualified_names: false` in your recipe. However, note that
+setting this to `false` is deprecated and this flag will be removed entirely in a future release.
 
 ### Potential Downtime
 

From fe18532b29e35af1cd3007e6affc102042b1af61 Mon Sep 17 00:00:00 2001
From: skrydal <piotr.skrydalewicz@gmail.com>
Date: Wed, 25 Oct 2023 00:58:56 +0200
Subject: [PATCH 091/259] docs(update): Added info on breaking change for
 policies (#9093)

Co-authored-by: Pedro Silva <pedro@acryl.io>
---
 docs/how/updating-datahub.md | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 7d8c25b06255a..57193ea69f2be 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -16,6 +16,39 @@ This is currently enabled by default to preserve compatibility, but will be disa
 If stateful ingestion is enabled, simply setting `include_metastore: false` will perform all required cleanup.
 Otherwise, we recommend soft deleting all databricks data via the DataHub CLI:
 `datahub delete --platform databricks --soft` and then reingesting with `include_metastore: false`.
+- #8846 - Changed enum values in resource filters used by policies. `RESOURCE_TYPE` became `TYPE` and `RESOURCE_URN` became `URN`.
+Any existing policies using these filters (i.e. defined for particular `urns` or `types` such as `dataset`) need to be upgraded
+manually, for example by retrieving their respective `dataHubPolicyInfo` aspect and changing part using filter i.e.
+```yaml
+   "resources": {
+     "filter": {
+       "criteria": [
+         {
+           "field": "RESOURCE_TYPE",
+           "condition": "EQUALS",
+           "values": [
+             "dataset"
+           ]
+         }
+       ]
+     }
+```
+into
+```yaml
+   "resources": {
+     "filter": {
+       "criteria": [
+         {
+           "field": "TYPE",
+           "condition": "EQUALS",
+           "values": [
+             "dataset"
+           ]
+         }
+       ]
+     }
+```
+for example, using `datahub put` command. Policies can be also removed and re-created via UI.
 - #9077 - The BigQuery ingestion source by default sets `match_fully_qualified_names: true`.
 This means that any `dataset_pattern` or `schema_pattern` specified will be matched on the fully
 qualified dataset name, i.e. `<project_name>.<dataset_name>`. If this is not the case, please

From ca331f58bd24187f9f0ca317216837178e9f41fa Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Wed, 25 Oct 2023 09:39:57 +0900
Subject: [PATCH 092/259] docs: add luckyorange script to head (#9080)

---
 docs-website/docusaurus.config.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
index 68ea1ebffa6c9..259ef970d818e 100644
--- a/docs-website/docusaurus.config.js
+++ b/docs-website/docusaurus.config.js
@@ -13,6 +13,13 @@ module.exports = {
   projectName: "datahub", // Usually your repo name.
   staticDirectories: ["static", "genStatic"],
   stylesheets: ["https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap"],
+  scripts: [
+    {
+      src: "https://tools.luckyorange.com/core/lo.js?site-id=28ea8a38",
+      async: true,
+      defer: true,
+    },
+  ],
   noIndex: isSaas,
   customFields: {
     isSaas: isSaas,

From 9a59c452bf36d750964f6d7f78df84a8c0c5eb66 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Wed, 25 Oct 2023 09:40:28 +0900
Subject: [PATCH 093/259] design: refactor docs navbar (#8975)

Co-authored-by: Jeff Merrick <jeff@wireform.io>
---
 docs-website/docusaurus.config.js             | 61 +++++++++----------
 docs-website/src/styles/global.scss           | 27 +++++---
 .../DocsVersionDropdownNavbarItem.js          |  4 ++
 .../src/theme/NavbarItem/styles.module.scss   |  8 +++
 4 files changed, 59 insertions(+), 41 deletions(-)
 create mode 100644 docs-website/src/theme/NavbarItem/styles.module.scss

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
index 259ef970d818e..506e263933394 100644
--- a/docs-website/docusaurus.config.js
+++ b/docs-website/docusaurus.config.js
@@ -57,44 +57,41 @@ module.exports = {
           position: "right",
         },
         {
-          to: "https://demo.datahubproject.io/",
-          label: "Demo",
-          position: "right",
-        },
-        {
-          href: "https://blog.datahubproject.io/",
-          label: "Blog",
-          position: "right",
-        },
-        {
-          href: "https://feature-requests.datahubproject.io/roadmap",
-          label: "Roadmap",
+          type: "dropdown",
+          label: "Resources",
           position: "right",
+          items: [
+            {
+              href: "https://demo.datahubproject.io/",
+              label: "Demo",
+            },
+            {
+              href: "https://blog.datahubproject.io/",
+              label: "Blog",
+            },
+            {
+              href: "https://feature-requests.datahubproject.io/roadmap",
+              label: "Roadmap",
+            },
+            {
+              href: "https://slack.datahubproject.io",
+              label: "Slack",
+            },
+            {
+              href: "https://github.com/datahub-project/datahub",
+              label: "GitHub",
+            },
+            {
+              href: "https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w",
+              label: "YouTube",
+            },
+          ],
         },
         {
           type: "docsVersionDropdown",
-          position: "right",
+          position: "left",
           dropdownActiveClassDisabled: true,
         },
-        {
-          href: "https://slack.datahubproject.io",
-          "aria-label": "Slack",
-          position: "right",
-          className: "item__icon item__slack",
-        },
-        {
-          href: "https://github.com/datahub-project/datahub",
-          "aria-label": "GitHub",
-          position: "right",
-          className: "item__icon item__github",
-        },
-
-        {
-          href: "https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w",
-          "aria-label": "YouTube",
-          position: "right",
-          className: "item__icon item__youtube",
-        },
       ],
     },
     footer: {
diff --git a/docs-website/src/styles/global.scss b/docs-website/src/styles/global.scss
index 55a54876b41ac..16e3893ed08b7 100644
--- a/docs-website/src/styles/global.scss
+++ b/docs-website/src/styles/global.scss
@@ -144,20 +144,29 @@ div[class^="announcementBar"] {
 
 /** Navbar */
 
-@media only screen and (max-width: 1050px) {
-  .navbar__toggle {
-    display: inherit;
-  }
-  .navbar__item {
-    display: none;
-  }
-}
-
 .navbar {
   .navbar__logo {
     height: 3rem;
   }
+
+  .navbar__link {
+    align-items: center;
+    margin: 0 1rem 0;
+    padding: 0;
+    border-bottom: 2px solid transparent;
+  }
+
+  .dropdown > .navbar__link:after {
+    top: -1px;
+    border-width: 0.3em 0.3em 0;
+    margin-left: 0.4em;
+  }
+
+  .navbar__link--active {
+    border-bottom-color: var(--ifm-navbar-link-hover-color);
+  }
   .navbar__item {
+    padding: 0.25rem 0;
     svg[class*="iconExternalLink"] {
       display: none;
     }
diff --git a/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js b/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js
index cc04ab23d3cf3..661d64392e67f 100644
--- a/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js
+++ b/docs-website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js
@@ -6,6 +6,9 @@ import { translate } from "@docusaurus/Translate";
 import { useLocation } from "@docusaurus/router";
 import DefaultNavbarItem from "@theme/NavbarItem/DefaultNavbarItem";
 import DropdownNavbarItem from "@theme/NavbarItem/DropdownNavbarItem";
+
+import styles from "./styles.module.scss";
+
 const getVersionMainDoc = (version) => version.docs.find((doc) => doc.id === version.mainDocId);
 export default function DocsVersionDropdownNavbarItem({
   mobile,
@@ -60,6 +63,7 @@ export default function DocsVersionDropdownNavbarItem({
   return (
     <DropdownNavbarItem
       {...props}
+      className={styles.versionNavItem}
       mobile={mobile}
       label={dropdownLabel}
       to={false} // This component is Swizzled to disable the link here
diff --git a/docs-website/src/theme/NavbarItem/styles.module.scss b/docs-website/src/theme/NavbarItem/styles.module.scss
new file mode 100644
index 0000000000000..d533115a9c9f0
--- /dev/null
+++ b/docs-website/src/theme/NavbarItem/styles.module.scss
@@ -0,0 +1,8 @@
+.versionNavItem {
+  margin-left: 0 !important;
+  padding: 0.2em 1em !important;
+  display: block;
+  border-radius: var(--ifm-button-border-radius) !important;
+  color: var(--ifm-menu-color-active);
+  background: var(--ifm-menu-color-background-active);
+}

From 916235d31a7d914b51a44ee02374270e5a4ea1d3 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 24 Oct 2023 19:59:42 -0700
Subject: [PATCH 094/259] fix(ingest): update athena type mapping (#9061)

---
 .../src/datahub/ingestion/source/sql/athena.py      |  4 +++-
 .../src/datahub/ingestion/source/sql/sql_common.py  |  5 +----
 .../src/datahub/ingestion/source/sql/sql_types.py   |  5 ++---
 .../datahub/utilities/sqlalchemy_type_converter.py  | 13 ++++++++++---
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
index 06b9ad92677a2..75e8fe1d6f7a6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
@@ -37,7 +37,7 @@
     gen_database_key,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
-from datahub.metadata.schema_classes import RecordTypeClass
+from datahub.metadata.schema_classes import MapTypeClass, RecordTypeClass
 from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
 from datahub.utilities.sqlalchemy_type_converter import (
     MapType,
@@ -46,7 +46,9 @@
 
 logger = logging.getLogger(__name__)
 
+assert STRUCT, "required type modules are not available"
 register_custom_type(STRUCT, RecordTypeClass)
+register_custom_type(MapType, MapTypeClass)
 
 
 class CustomAthenaRestDialect(AthenaRestDialect):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index be03858ec3ef9..fad9b9e8018a5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -80,7 +80,6 @@
     DatasetLineageTypeClass,
     DatasetPropertiesClass,
     GlobalTagsClass,
-    MapTypeClass,
     SubTypesClass,
     TagAssociationClass,
     UpstreamClass,
@@ -90,7 +89,6 @@
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.registries.domain_registry import DomainRegistry
 from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport
-from datahub.utilities.sqlalchemy_type_converter import MapType
 
 if TYPE_CHECKING:
     from datahub.ingestion.source.ge_data_profiler import (
@@ -140,6 +138,7 @@ class SqlWorkUnit(MetadataWorkUnit):
 
 
 _field_type_mapping: Dict[Type[TypeEngine], Type] = {
+    # Note: to add dialect-specific types to this mapping, use the `register_custom_type` function.
     types.Integer: NumberTypeClass,
     types.Numeric: NumberTypeClass,
     types.Boolean: BooleanTypeClass,
@@ -156,8 +155,6 @@ class SqlWorkUnit(MetadataWorkUnit):
     types.DATETIME: TimeTypeClass,
     types.TIMESTAMP: TimeTypeClass,
     types.JSON: RecordTypeClass,
-    # additional type definitions that are used by the Athena source
-    MapType: MapTypeClass,  # type: ignore
     # Because the postgresql dialect is used internally by many other dialects,
     # we add some postgres types here. This is ok to do because the postgresql
     # dialect is built-in to sqlalchemy.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
index ae47623188f42..3b4a7e1dc0287 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
@@ -7,7 +7,7 @@
     BytesType,
     DateType,
     EnumType,
-    MapType as MapTypeAvro,
+    MapType,
     NullType,
     NumberType,
     RecordType,
@@ -15,7 +15,6 @@
     TimeType,
     UnionType,
 )
-from datahub.utilities.sqlalchemy_type_converter import MapType
 
 # these can be obtained by running `select format_type(oid, null),* from pg_type;`
 # we've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.)
@@ -364,7 +363,7 @@ def resolve_vertica_modified_type(type_string: str) -> Any:
     "time": TimeType,
     "timestamp": TimeType,
     "row": RecordType,
-    "map": MapTypeAvro,
+    "map": MapType,
     "array": ArrayType,
 }
 
diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
index 1d5ec5dae3519..5d2fc6872c7bd 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py
@@ -4,7 +4,6 @@
 from typing import Any, Dict, List, Optional, Type, Union
 
 from sqlalchemy import types
-from sqlalchemy_bigquery import STRUCT
 
 from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields
 from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField
@@ -12,6 +11,12 @@
 
 logger = logging.getLogger(__name__)
 
+try:
+    # This is used for both BigQuery and Athena.
+    from sqlalchemy_bigquery import STRUCT
+except ImportError:
+    STRUCT = None
+
 
 class MapType(types.TupleType):
     # Wrapper class around SQLalchemy's TupleType to increase compatibility with DataHub
@@ -42,7 +47,9 @@ def get_avro_type(
     ) -> Dict[str, Any]:
         """Determines the concrete AVRO schema type for a SQLalchemy-typed column"""
 
-        if type(column_type) in cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE.keys():
+        if isinstance(
+            column_type, tuple(cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE.keys())
+        ):
             return {
                 "type": cls.PRIMITIVE_SQL_ALCHEMY_TYPE_TO_AVRO_TYPE[type(column_type)],
                 "native_data_type": str(column_type),
@@ -88,7 +95,7 @@ def get_avro_type(
                 "key_type": cls.get_avro_type(column_type=key_type, nullable=nullable),
                 "key_native_data_type": str(key_type),
             }
-        if isinstance(column_type, STRUCT):
+        if STRUCT and isinstance(column_type, STRUCT):
             fields = []
             for field_def in column_type._STRUCT_fields:
                 field_name, field_type = field_def

From 2d1584b12fe4a40a077457e618f0937132763586 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Tue, 24 Oct 2023 23:08:24 -0400
Subject: [PATCH 095/259] feat(ingest/datahub-source): Allow ingesting aspects
 from the entitiesV2 API (#9089)

---
 .../ingestion/source/datahub/config.py        | 19 ++++++-
 .../source/datahub/datahub_api_reader.py      | 49 +++++++++++++++++++
 .../source/datahub/datahub_source.py          | 16 ++++++
 3 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py

diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
index 053d136305527..83958dc76754f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
@@ -1,3 +1,4 @@
+import os
 from typing import Optional
 
 from pydantic import Field, root_validator
@@ -67,9 +68,25 @@ class DataHubSourceConfig(StatefulIngestionConfigBase):
         ),
     )
 
+    pull_from_datahub_api: bool = Field(
+        default=False,
+        description="Use the DataHub API to fetch versioned aspects.",
+        hidden_from_docs=True,
+    )
+
+    max_workers: int = Field(
+        default=5 * (os.cpu_count() or 4),
+        description="Number of worker threads to use for datahub api ingestion.",
+        hidden_from_docs=True,
+    )
+
     @root_validator
     def check_ingesting_data(cls, values):
-        if not values.get("database_connection") and not values.get("kafka_connection"):
+        if (
+            not values.get("database_connection")
+            and not values.get("kafka_connection")
+            and not values.get("pull_from_datahub_api")
+        ):
             raise ValueError(
                 "Your current config will not ingest any data."
                 " Please specify at least one of `database_connection` or `kafka_connection`, ideally both."
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
new file mode 100644
index 0000000000000..7ee36736723b2
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
@@ -0,0 +1,49 @@
+import logging
+from concurrent import futures
+from typing import Dict, Iterable, List
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.graph.filters import RemovedStatusFilter
+from datahub.ingestion.source.datahub.config import DataHubSourceConfig
+from datahub.ingestion.source.datahub.report import DataHubSourceReport
+from datahub.metadata._schema_classes import _Aspect
+
+logger = logging.getLogger(__name__)
+
+# Should work for at least mysql, mariadb, postgres
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
+
+
+class DataHubApiReader:
+    def __init__(
+        self,
+        config: DataHubSourceConfig,
+        report: DataHubSourceReport,
+        graph: DataHubGraph,
+    ):
+        self.config = config
+        self.report = report
+        self.graph = graph
+
+    def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
+        urns = self.graph.get_urns_by_filter(
+            status=RemovedStatusFilter.ALL,
+            batch_size=self.config.database_query_batch_size,
+        )
+        tasks: List[futures.Future[Iterable[MetadataChangeProposalWrapper]]] = []
+        with futures.ThreadPoolExecutor(
+            max_workers=self.config.max_workers
+        ) as executor:
+            for urn in urns:
+                tasks.append(executor.submit(self._get_aspects_for_urn, urn))
+            for task in futures.as_completed(tasks):
+                yield from task.result()
+
+    def _get_aspects_for_urn(self, urn: str) -> Iterable[MetadataChangeProposalWrapper]:
+        aspects: Dict[str, _Aspect] = self.graph.get_entity_semityped(urn)  # type: ignore
+        for aspect in aspects.values():
+            yield MetadataChangeProposalWrapper(
+                entityUrn=urn,
+                aspect=aspect,
+            )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
index 2368febe1ff57..a2f43b8cc62cb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
@@ -15,6 +15,7 @@
 from datahub.ingestion.api.source_helpers import auto_workunit_reporter
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.datahub.config import DataHubSourceConfig
+from datahub.ingestion.source.datahub.datahub_api_reader import DataHubApiReader
 from datahub.ingestion.source.datahub.datahub_database_reader import (
     DataHubDatabaseReader,
 )
@@ -58,6 +59,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         logger.info(f"Ingesting DataHub metadata up until {self.report.stop_time}")
         state = self.stateful_ingestion_handler.get_last_run_state()
 
+        if self.config.pull_from_datahub_api:
+            yield from self._get_api_workunits()
+
         if self.config.database_connection is not None:
             yield from self._get_database_workunits(
                 from_createdon=state.database_createdon_datetime
@@ -139,6 +143,18 @@ def _get_kafka_workunits(
                     )
                 self._commit_progress(i)
 
+    def _get_api_workunits(self) -> Iterable[MetadataWorkUnit]:
+        if self.ctx.graph is None:
+            self.report.report_failure(
+                "datahub_api",
+                "Specify datahub_api on your ingestion recipe to ingest from the DataHub API",
+            )
+            return
+
+        reader = DataHubApiReader(self.config, self.report, self.ctx.graph)
+        for mcp in reader.get_aspects():
+            yield mcp.as_workunit()
+
     def _commit_progress(self, i: Optional[int] = None) -> None:
         """Commit progress to stateful storage, if there have been no errors.
 

From b612545220d9329696eaa26d6b42439cdf01fb95 Mon Sep 17 00:00:00 2001
From: siddiquebagwan-gslab <mohdsiddique.bagwan@gslab.com>
Date: Wed, 25 Oct 2023 15:26:06 +0530
Subject: [PATCH 096/259] feat(ingestion/redshift): support
 auto_incremental_lineage (#9010)

---
 docs/how/updating-datahub.md                      |  2 ++
 metadata-ingestion/setup.py                       | 10 +++-------
 .../datahub/ingestion/source/redshift/config.py   |  8 +++++++-
 .../datahub/ingestion/source/redshift/redshift.py | 15 +++++++++++++--
 .../tests/unit/test_redshift_config.py            |  6 ++++++
 5 files changed, 31 insertions(+), 10 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/test_redshift_config.py

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 57193ea69f2be..8813afee65be9 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -4,6 +4,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ## Next
 
+- #9010 - In Redshift source's config `incremental_lineage` is set default to off.
+
 ### Breaking Changes
 
 - #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 417588a433655..72b0e776a0da5 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -355,13 +355,9 @@
     | {"psycopg2-binary", "pymysql>=1.0.2"},
     "pulsar": {"requests"},
     "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib,
-    "redshift": sql_common
-    | redshift_common
-    | usage_common
-    | sqlglot_lib
-    | {"redshift-connector"},
-    "redshift-legacy": sql_common | redshift_common,
-    "redshift-usage-legacy": sql_common | usage_common | redshift_common,
+    "redshift": sql_common | redshift_common | usage_common | {"redshift-connector"} | sqlglot_lib,
+    "redshift-legacy": sql_common | redshift_common | sqlglot_lib,
+    "redshift-usage-legacy": sql_common | redshift_common | sqlglot_lib | usage_common,
     "s3": {*s3_base, *data_lake_profiling},
     "gcs": {*s3_base, *data_lake_profiling},
     "sagemaker": aws_common,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
index 2789b800940db..79b044841e054 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -133,7 +133,13 @@ class RedshiftConfig(
     )
 
     extract_column_level_lineage: bool = Field(
-        default=True, description="Whether to extract column level lineage."
+        default=True,
+        description="Whether to extract column level lineage. This config works with rest-sink only.",
+    )
+
+    incremental_lineage: bool = Field(
+        default=False,
+        description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.  This config works with rest-sink only.",
     )
 
     @root_validator(pre=True)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
index a1b6333a3775d..26237a6ce12e0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -1,5 +1,6 @@
 import logging
 from collections import defaultdict
+from functools import partial
 from typing import Dict, Iterable, List, Optional, Type, Union
 
 import humanfriendly
@@ -25,6 +26,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
 from datahub.ingestion.api.source import (
     CapabilityReport,
     MetadataWorkUnitProcessor,
@@ -369,6 +371,11 @@ def gen_database_container(self, database: str) -> Iterable[MetadataWorkUnit]:
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
+            partial(
+                auto_incremental_lineage,
+                self.ctx.graph,
+                self.config.incremental_lineage,
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
@@ -942,7 +949,9 @@ def generate_lineage(self, database: str) -> Iterable[MetadataWorkUnit]:
                 )
                 if lineage_info:
                     yield from gen_lineage(
-                        dataset_urn, lineage_info, self.config.incremental_lineage
+                        dataset_urn,
+                        lineage_info,
+                        incremental_lineage=False,  # incremental lineage generation is taken care by auto_incremental_lineage
                     )
 
         for schema in self.db_views[database]:
@@ -956,7 +965,9 @@ def generate_lineage(self, database: str) -> Iterable[MetadataWorkUnit]:
                 )
                 if lineage_info:
                     yield from gen_lineage(
-                        dataset_urn, lineage_info, self.config.incremental_lineage
+                        dataset_urn,
+                        lineage_info,
+                        incremental_lineage=False,  # incremental lineage generation is taken care by auto_incremental_lineage
                     )
 
     def add_config_to_report(self):
diff --git a/metadata-ingestion/tests/unit/test_redshift_config.py b/metadata-ingestion/tests/unit/test_redshift_config.py
new file mode 100644
index 0000000000000..8a165e7f5f3fe
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_redshift_config.py
@@ -0,0 +1,6 @@
+from datahub.ingestion.source.redshift.config import RedshiftConfig
+
+
+def test_incremental_lineage_default_to_false():
+    config = RedshiftConfig(host_port="localhost:5439", database="test")
+    assert config.incremental_lineage is False

From 9cccd22c04bf357b574f4d9d7dae3aee633bf7d3 Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Wed, 25 Oct 2023 11:01:49 +0100
Subject: [PATCH 097/259] feat(auth): Add backwards compatible field resolver
 (#9096)

---
 .../com/datahub/authorization/EntityFieldType.java  | 13 +++++++++++++
 .../authorization/DefaultEntitySpecResolver.java    | 13 +++++++------
 .../DataPlatformInstanceFieldResolverProvider.java  | 10 +++++-----
 .../DomainFieldResolverProvider.java                |  5 +++--
 .../EntityFieldResolverProvider.java                |  6 ++++--
 .../EntityTypeFieldResolverProvider.java            |  7 +++++--
 .../EntityUrnFieldResolverProvider.java             |  7 +++++--
 .../GroupMembershipFieldResolverProvider.java       |  5 +++--
 .../OwnerFieldResolverProvider.java                 |  5 +++--
 ...taPlatformInstanceFieldResolverProviderTest.java |  2 +-
 .../GroupMembershipFieldResolverProviderTest.java   |  2 +-
 11 files changed, 50 insertions(+), 25 deletions(-)

diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
index 46763f29a7040..1258d958f2092 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java
@@ -4,6 +4,19 @@
  * List of entity field types to fetch for a given entity
  */
 public enum EntityFieldType {
+
+  /**
+   * Type of the entity (e.g. dataset, chart)
+   * @deprecated
+   */
+  @Deprecated
+  RESOURCE_URN,
+  /**
+   * Urn of the entity
+   * @deprecated
+   */
+  @Deprecated
+  RESOURCE_TYPE,
   /**
    * Type of the entity (e.g. dataset, chart)
    */
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
index 4ad14ed59c9c0..65b0329a9c4f2 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java
@@ -1,15 +1,16 @@
 package com.datahub.authorization;
 
-import com.datahub.authorization.fieldresolverprovider.DataPlatformInstanceFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
 import com.datahub.authentication.Authentication;
+import com.datahub.authorization.fieldresolverprovider.DataPlatformInstanceFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.DomainFieldResolverProvider;
-import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.EntityFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider;
 import com.datahub.authorization.fieldresolverprovider.GroupMembershipFieldResolverProvider;
+import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.util.Pair;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -34,7 +35,7 @@ public ResolvedEntitySpec resolve(EntitySpec entitySpec) {
 
   private Map<EntityFieldType, FieldResolver> getFieldResolvers(EntitySpec entitySpec) {
     return _entityFieldResolverProviders.stream()
-        .collect(Collectors.toMap(EntityFieldResolverProvider::getFieldType,
-            hydrator -> hydrator.getFieldResolver(entitySpec)));
+        .flatMap(resolver -> resolver.getFieldTypes().stream().map(fieldType -> Pair.of(fieldType, resolver)))
+        .collect(Collectors.toMap(Pair::getKey, pair -> pair.getValue().getFieldResolver(entitySpec)));
   }
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
index 27cb8fcee8138..cbb237654e969 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProvider.java
@@ -1,8 +1,5 @@
 package com.datahub.authorization.fieldresolverprovider;
 
-import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME;
-import static com.linkedin.metadata.Constants.DATA_PLATFORM_INSTANCE_ENTITY_NAME;
-
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
@@ -14,10 +11,13 @@
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.client.EntityClient;
 import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import static com.linkedin.metadata.Constants.*;
+
 /**
  * Provides field resolver for domain given resourceSpec
  */
@@ -29,8 +29,8 @@ public class DataPlatformInstanceFieldResolverProvider implements EntityFieldRes
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.DATA_PLATFORM_INSTANCE;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.DATA_PLATFORM_INSTANCE);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
index 25c2165f02b94..15d821b75c0bd 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java
@@ -14,6 +14,7 @@
 
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -37,8 +38,8 @@ public class DomainFieldResolverProvider implements EntityFieldResolverProvider
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.DOMAIN;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.DOMAIN);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
index a76db0ecb5102..227d403a9cd1d 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityFieldResolverProvider.java
@@ -3,6 +3,7 @@
 import com.datahub.authorization.FieldResolver;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
+import java.util.List;
 
 
 /**
@@ -11,9 +12,10 @@
 public interface EntityFieldResolverProvider {
 
   /**
-   * Field that this hydrator is hydrating
+   * List of fields that this hydrator is hydrating.
+   * @return
    */
-  EntityFieldType getFieldType();
+  List<EntityFieldType> getFieldTypes();
 
   /**
    * Return resolver for fetching the field values given the entity
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
index 187f696904947..addac84c68b18 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java
@@ -3,16 +3,19 @@
 import com.datahub.authorization.FieldResolver;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
+import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList;
 import java.util.Collections;
+import java.util.List;
 
 
 /**
  * Provides field resolver for entity type given entitySpec
  */
 public class EntityTypeFieldResolverProvider implements EntityFieldResolverProvider {
+
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.TYPE;
+  public List<EntityFieldType> getFieldTypes() {
+    return ImmutableList.of(EntityFieldType.TYPE, EntityFieldType.RESOURCE_TYPE);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
index 2f5c4a7c6c961..32960de687839 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java
@@ -3,16 +3,19 @@
 import com.datahub.authorization.FieldResolver;
 import com.datahub.authorization.EntityFieldType;
 import com.datahub.authorization.EntitySpec;
+import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList;
 import java.util.Collections;
+import java.util.List;
 
 
 /**
  * Provides field resolver for entity urn given entitySpec
  */
 public class EntityUrnFieldResolverProvider implements EntityFieldResolverProvider {
+
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.URN;
+  public List<EntityFieldType> getFieldTypes() {
+    return ImmutableList.of(EntityFieldType.URN, EntityFieldType.RESOURCE_URN);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
index 8db029632d7e2..b1202d9f4bbd3 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProvider.java
@@ -13,6 +13,7 @@
 import com.linkedin.identity.NativeGroupMembership;
 import com.linkedin.metadata.Constants;
 import com.linkedin.identity.GroupMembership;
+import java.util.Collections;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -35,8 +36,8 @@ public class GroupMembershipFieldResolverProvider implements EntityFieldResolver
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.GROUP_MEMBERSHIP;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.GROUP_MEMBERSHIP);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
index bdd652d1d3871..3c27f9e6ce8d7 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java
@@ -12,6 +12,7 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
 import java.util.Collections;
+import java.util.List;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -28,8 +29,8 @@ public class OwnerFieldResolverProvider implements EntityFieldResolverProvider {
   private final Authentication _systemAuthentication;
 
   @Override
-  public EntityFieldType getFieldType() {
-    return EntityFieldType.OWNER;
+  public List<EntityFieldType> getFieldTypes() {
+    return Collections.singletonList(EntityFieldType.OWNER);
   }
 
   @Override
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
index b2343bbb01509..5c7d87f1c05a9 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/DataPlatformInstanceFieldResolverProviderTest.java
@@ -56,7 +56,7 @@ public void setup() {
 
   @Test
   public void shouldReturnDataPlatformInstanceType() {
-    assertEquals(EntityFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldType());
+    assertEquals(EntityFieldType.DATA_PLATFORM_INSTANCE, dataPlatformInstanceFieldResolverProvider.getFieldTypes().get(0));
   }
 
   @Test
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
index 54675045b4413..af547f14cd3fc 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/GroupMembershipFieldResolverProviderTest.java
@@ -53,7 +53,7 @@ public void setup() {
 
   @Test
   public void shouldReturnGroupsMembershipType() {
-    assertEquals(EntityFieldType.GROUP_MEMBERSHIP, groupMembershipFieldResolverProvider.getFieldType());
+    assertEquals(EntityFieldType.GROUP_MEMBERSHIP, groupMembershipFieldResolverProvider.getFieldTypes().get(0));
   }
 
   @Test

From dd5d997390d489c777aac25dbbd3f47c4bab8340 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 25 Oct 2023 10:54:55 -0400
Subject: [PATCH 098/259] build(gradle): Support IntelliJ 2023.2.3 (#9034)

---
 metadata-models/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle
index 53e7765152aef..bd8052283e168 100644
--- a/metadata-models/build.gradle
+++ b/metadata-models/build.gradle
@@ -23,6 +23,7 @@ dependencies {
       }
     }
   api project(':li-utils')
+  api project(path: ':li-utils', configuration: "dataTemplate")
   dataModel project(':li-utils')
 
   compileOnly externalDependency.lombok

From 8a80e858a7b6bf67105e082475ada57a27c37c67 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 25 Oct 2023 13:06:12 -0400
Subject: [PATCH 099/259] build(ingest): Bump avro pin: security vulnerability
 (#9042)

---
 metadata-ingestion/scripts/avro_codegen.py    |   3 +-
 metadata-ingestion/scripts/modeldocgen.py     |   4 +-
 metadata-ingestion/setup.py                   |   8 +-
 .../ingestion/extractor/schema_util.py        | 109 ++++++++++++------
 .../src/datahub/ingestion/source/kafka.py     |  19 ++-
 .../src/datahub/utilities/mapping.py          |   4 +-
 .../unit/data_lake/test_schema_inference.py   |   6 +-
 7 files changed, 99 insertions(+), 54 deletions(-)

diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py
index a9b9b4b20f5ac..021ebd4a31eb3 100644
--- a/metadata-ingestion/scripts/avro_codegen.py
+++ b/metadata-ingestion/scripts/avro_codegen.py
@@ -152,7 +152,8 @@ def add_name(self, name_attr, space_attr, new_schema):
     return encoded
 
 
-autogen_header = """# flake8: noqa
+autogen_header = """# mypy: ignore-errors
+# flake8: noqa
 
 # This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
 # Do not modify manually!
diff --git a/metadata-ingestion/scripts/modeldocgen.py b/metadata-ingestion/scripts/modeldocgen.py
index ffa80515dbafd..81b26145e620c 100644
--- a/metadata-ingestion/scripts/modeldocgen.py
+++ b/metadata-ingestion/scripts/modeldocgen.py
@@ -351,8 +351,8 @@ def strip_types(field_path: str) -> str:
             field_objects = []
             for f in entity_fields:
                 field = avro.schema.Field(
-                    type=f["type"],
-                    name=f["name"],
+                    f["type"],
+                    f["name"],
                     has_default=False,
                 )
                 field_objects.append(field)
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 72b0e776a0da5..0b8661b0df5f5 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -32,7 +32,7 @@
     "expandvars>=0.6.5",
     "avro-gen3==0.7.11",
     # "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3",
-    "avro>=1.10.2,<1.11",
+    "avro>=1.11.3,<1.12",
     "python-dateutil>=2.8.0",
     "tabulate",
     "progressbar2",
@@ -355,7 +355,11 @@
     | {"psycopg2-binary", "pymysql>=1.0.2"},
     "pulsar": {"requests"},
     "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib,
-    "redshift": sql_common | redshift_common | usage_common | {"redshift-connector"} | sqlglot_lib,
+    "redshift": sql_common
+    | redshift_common
+    | usage_common
+    | {"redshift-connector"}
+    | sqlglot_lib,
     "redshift-legacy": sql_common | redshift_common | sqlglot_lib,
     "redshift-usage-legacy": sql_common | redshift_common | sqlglot_lib | usage_common,
     "s3": {*s3_base, *data_lake_profiling},
diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
index 4acf99a50e50e..df0b732833fbe 100644
--- a/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
+++ b/metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
@@ -1,6 +1,18 @@
 import json
 import logging
-from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Type,
+    Union,
+    cast,
+    overload,
+)
 
 import avro.schema
 
@@ -54,6 +66,8 @@
     avro.schema.PrimitiveSchema,
 ]
 
+SchemaOrField = Union[avro.schema.Schema, avro.schema.Field]
+
 FieldStack = List[avro.schema.Field]
 
 # The latest avro code contains this type definition in a compatibility module,
@@ -124,16 +138,22 @@ def __init__(
         self._meta_mapping_processor = meta_mapping_processor
         self._schema_tags_field = schema_tags_field
         self._tag_prefix = tag_prefix
+
         # Map of avro schema type to the conversion handler
-        self._avro_type_to_mce_converter_map: Dict[
-            avro.schema.Schema,
-            Callable[[ExtendedAvroNestedSchemas], Generator[SchemaField, None, None]],
+        # TODO: Clean up this type... perhaps refactor
+        self._avro_type_to_mce_converter_map: Mapping[
+            Union[
+                Type[avro.schema.Schema],
+                Type[avro.schema.Field],
+                Type[avro.schema.LogicalSchema],
+            ],
+            Callable[[SchemaOrField], Iterable[SchemaField]],
         ] = {
             avro.schema.RecordSchema: self._gen_from_non_field_nested_schemas,
             avro.schema.UnionSchema: self._gen_from_non_field_nested_schemas,
             avro.schema.ArraySchema: self._gen_from_non_field_nested_schemas,
             avro.schema.MapSchema: self._gen_from_non_field_nested_schemas,
-            avro.schema.Field: self._gen_nested_schema_from_field,
+            avro.schema.Field: self._gen_nested_schema_from_field,  # type: ignore
             avro.schema.PrimitiveSchema: self._gen_non_nested_to_mce_fields,
             avro.schema.FixedSchema: self._gen_non_nested_to_mce_fields,
             avro.schema.EnumSchema: self._gen_non_nested_to_mce_fields,
@@ -142,20 +162,22 @@ def __init__(
 
     @staticmethod
     def _get_type_name(
-        avro_schema: avro.schema.Schema, logical_if_present: bool = False
+        avro_schema: SchemaOrField, logical_if_present: bool = False
     ) -> str:
         logical_type_name: Optional[str] = None
         if logical_if_present:
-            logical_type_name = getattr(
-                avro_schema, "logical_type", None
-            ) or avro_schema.props.get("logicalType")
+            logical_type_name = cast(
+                Optional[str],
+                getattr(avro_schema, "logical_type", None)
+                or avro_schema.props.get("logicalType"),
+            )
         return logical_type_name or str(
             getattr(avro_schema.type, "type", avro_schema.type)
         )
 
     @staticmethod
     def _get_column_type(
-        avro_schema: avro.schema.Schema, logical_type: Optional[str]
+        avro_schema: SchemaOrField, logical_type: Optional[str]
     ) -> SchemaFieldDataType:
         type_name: str = AvroToMceSchemaConverter._get_type_name(avro_schema)
         TypeClass: Optional[Type] = AvroToMceSchemaConverter.field_type_mapping.get(
@@ -186,7 +208,7 @@ def _get_column_type(
             )
         return dt
 
-    def _is_nullable(self, schema: avro.schema.Schema) -> bool:
+    def _is_nullable(self, schema: SchemaOrField) -> bool:
         if isinstance(schema, avro.schema.Field):
             return self._is_nullable(schema.type)
         if isinstance(schema, avro.schema.UnionSchema):
@@ -208,7 +230,7 @@ def _strip_namespace(name_or_fullname: str) -> str:
         return name_or_fullname.rsplit(".", maxsplit=1)[-1]
 
     @staticmethod
-    def _get_simple_native_type(schema: ExtendedAvroNestedSchemas) -> str:
+    def _get_simple_native_type(schema: SchemaOrField) -> str:
         if isinstance(schema, (avro.schema.RecordSchema, avro.schema.Field)):
             # For Records, fields, always return the name.
             return AvroToMceSchemaConverter._strip_namespace(schema.name)
@@ -226,7 +248,7 @@ def _get_simple_native_type(schema: ExtendedAvroNestedSchemas) -> str:
         return schema.type
 
     @staticmethod
-    def _get_type_annotation(schema: ExtendedAvroNestedSchemas) -> str:
+    def _get_type_annotation(schema: SchemaOrField) -> str:
         simple_native_type = AvroToMceSchemaConverter._get_simple_native_type(schema)
         if simple_native_type.startswith("__struct_"):
             simple_native_type = "struct"
@@ -237,10 +259,24 @@ def _get_type_annotation(schema: ExtendedAvroNestedSchemas) -> str:
         else:
             return f"[type={simple_native_type}]"
 
+    @staticmethod
+    @overload
+    def _get_underlying_type_if_option_as_union(
+        schema: SchemaOrField, default: SchemaOrField
+    ) -> SchemaOrField:
+        ...
+
+    @staticmethod
+    @overload
+    def _get_underlying_type_if_option_as_union(
+        schema: SchemaOrField, default: Optional[SchemaOrField] = None
+    ) -> Optional[SchemaOrField]:
+        ...
+
     @staticmethod
     def _get_underlying_type_if_option_as_union(
-        schema: AvroNestedSchemas, default: Optional[AvroNestedSchemas] = None
-    ) -> AvroNestedSchemas:
+        schema: SchemaOrField, default: Optional[SchemaOrField] = None
+    ) -> Optional[SchemaOrField]:
         if isinstance(schema, avro.schema.UnionSchema) and len(schema.schemas) == 2:
             (first, second) = schema.schemas
             if first.type == AVRO_TYPE_NULL:
@@ -258,8 +294,8 @@ class SchemaFieldEmissionContextManager:
 
         def __init__(
             self,
-            schema: avro.schema.Schema,
-            actual_schema: avro.schema.Schema,
+            schema: SchemaOrField,
+            actual_schema: SchemaOrField,
             converter: "AvroToMceSchemaConverter",
             description: Optional[str] = None,
             default_value: Optional[str] = None,
@@ -275,7 +311,7 @@ def __enter__(self):
             self._converter._prefix_name_stack.append(type_annotation)
             return self
 
-        def emit(self) -> Generator[SchemaField, None, None]:
+        def emit(self) -> Iterable[SchemaField]:
             if (
                 not isinstance(
                     self._actual_schema,
@@ -307,7 +343,7 @@ def emit(self) -> Generator[SchemaField, None, None]:
 
                 description = self._description
                 if not description and actual_schema.props.get("doc"):
-                    description = actual_schema.props.get("doc")
+                    description = cast(Optional[str], actual_schema.props.get("doc"))
 
                 if self._default_value is not None:
                     description = f"{description if description else ''}\nField default value: {self._default_value}"
@@ -320,12 +356,12 @@ def emit(self) -> Generator[SchemaField, None, None]:
                     native_data_type = native_data_type[
                         slice(len(type_prefix), len(native_data_type) - 1)
                     ]
-                native_data_type = actual_schema.props.get(
-                    "native_data_type", native_data_type
+                native_data_type = cast(
+                    str, actual_schema.props.get("native_data_type", native_data_type)
                 )
 
                 field_path = self._converter._get_cur_field_path()
-                merged_props = {}
+                merged_props: Dict[str, Any] = {}
                 merged_props.update(self._schema.other_props)
                 merged_props.update(schema.other_props)
 
@@ -363,12 +399,13 @@ def emit(self) -> Generator[SchemaField, None, None]:
 
                 meta_terms_aspect = meta_aspects.get(Constants.ADD_TERM_OPERATION)
 
-                logical_type_name: Optional[str] = (
+                logical_type_name: Optional[str] = cast(
+                    Optional[str],
                     # logicalType nested inside type
                     getattr(actual_schema, "logical_type", None)
                     or actual_schema.props.get("logicalType")
                     # bare logicalType
-                    or self._actual_schema.props.get("logicalType")
+                    or self._actual_schema.props.get("logicalType"),
                 )
 
                 field = SchemaField(
@@ -392,14 +429,12 @@ def emit(self) -> Generator[SchemaField, None, None]:
         def __exit__(self, exc_type, exc_val, exc_tb):
             self._converter._prefix_name_stack.pop()
 
-    def _get_sub_schemas(
-        self, schema: ExtendedAvroNestedSchemas
-    ) -> Generator[avro.schema.Schema, None, None]:
+    def _get_sub_schemas(self, schema: SchemaOrField) -> Iterable[SchemaOrField]:
         """Responsible for generation for appropriate sub-schemas for every nested AVRO type."""
 
         def gen_items_from_list_tuple_or_scalar(
             val: Any,
-        ) -> Generator[avro.schema.Schema, None, None]:
+        ) -> Iterable[avro.schema.Schema]:
             if isinstance(val, (list, tuple)):
                 for i in val:
                     yield i
@@ -433,7 +468,7 @@ def gen_items_from_list_tuple_or_scalar(
     def _gen_nested_schema_from_field(
         self,
         field: avro.schema.Field,
-    ) -> Generator[SchemaField, None, None]:
+    ) -> Iterable[SchemaField]:
         """Handles generation of MCE SchemaFields for an AVRO Field type."""
         # NOTE: Here we only manage the field stack and trigger MCE Field generation from this field's type.
         # The actual emitting of a field happens when
@@ -447,7 +482,7 @@ def _gen_nested_schema_from_field(
 
     def _gen_from_last_field(
         self, schema_to_recurse: Optional[AvroNestedSchemas] = None
-    ) -> Generator[SchemaField, None, None]:
+    ) -> Iterable[SchemaField]:
         """Emits the field most-recent field, optionally triggering sub-schema generation under the field."""
         last_field_schema = self._fields_stack[-1]
         # Generate the custom-description for the field.
@@ -467,8 +502,8 @@ def _gen_from_last_field(
                     yield from self._to_mce_fields(sub_schema)
 
     def _gen_from_non_field_nested_schemas(
-        self, schema: AvroNestedSchemas
-    ) -> Generator[SchemaField, None, None]:
+        self, schema: SchemaOrField
+    ) -> Iterable[SchemaField]:
         """Handles generation of MCE SchemaFields for all standard AVRO nested types."""
         # Handle recursive record definitions
         recurse: bool = True
@@ -511,8 +546,8 @@ def _gen_from_non_field_nested_schemas(
                         yield from self._to_mce_fields(sub_schema)
 
     def _gen_non_nested_to_mce_fields(
-        self, schema: AvroNonNestedSchemas
-    ) -> Generator[SchemaField, None, None]:
+        self, schema: SchemaOrField
+    ) -> Iterable[SchemaField]:
         """Handles generation of MCE SchemaFields for non-nested AVRO types."""
         with AvroToMceSchemaConverter.SchemaFieldEmissionContextManager(
             schema,
@@ -521,9 +556,7 @@ def _gen_non_nested_to_mce_fields(
         ) as non_nested_emitter:
             yield from non_nested_emitter.emit()
 
-    def _to_mce_fields(
-        self, avro_schema: avro.schema.Schema
-    ) -> Generator[SchemaField, None, None]:
+    def _to_mce_fields(self, avro_schema: SchemaOrField) -> Iterable[SchemaField]:
         # Invoke the relevant conversion handler for the schema element type.
         schema_type = (
             type(avro_schema)
@@ -541,7 +574,7 @@ def to_mce_fields(
         meta_mapping_processor: Optional[OperationProcessor] = None,
         schema_tags_field: Optional[str] = None,
         tag_prefix: Optional[str] = None,
-    ) -> Generator[SchemaField, None, None]:
+    ) -> Iterable[SchemaField]:
         """
         Converts a key or value type AVRO schema string to appropriate MCE SchemaFields.
         :param avro_schema_string: String representation of the AVRO schema.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
index d5039360da567..23770ff3cf812 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
@@ -3,7 +3,7 @@
 import logging
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Any, Dict, Iterable, List, Optional, Type
+from typing import Any, Dict, Iterable, List, Optional, Type, cast
 
 import avro.schema
 import confluent_kafka
@@ -316,13 +316,20 @@ def _extract_record(
             avro_schema = avro.schema.parse(
                 schema_metadata.platformSchema.documentSchema
             )
-            description = avro_schema.doc
+            description = getattr(avro_schema, "doc", None)
             # set the tags
             all_tags: List[str] = []
-            for tag in avro_schema.other_props.get(
-                self.source_config.schema_tags_field, []
-            ):
-                all_tags.append(self.source_config.tag_prefix + tag)
+            try:
+                schema_tags = cast(
+                    Iterable[str],
+                    avro_schema.other_props.get(
+                        self.source_config.schema_tags_field, []
+                    ),
+                )
+                for tag in schema_tags:
+                    all_tags.append(self.source_config.tag_prefix + tag)
+            except TypeError:
+                pass
 
             if self.source_config.enable_meta_mapping:
                 meta_aspects = self.meta_processor.process(avro_schema.other_props)
diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py
index eb2d975ee607f..f91c01d901ac1 100644
--- a/metadata-ingestion/src/datahub/utilities/mapping.py
+++ b/metadata-ingestion/src/datahub/utilities/mapping.py
@@ -4,7 +4,7 @@
 import re
 import time
 from functools import reduce
-from typing import Any, Dict, List, Match, Optional, Union, cast
+from typing import Any, Dict, List, Mapping, Match, Optional, Union, cast
 
 from datahub.emitter import mce_builder
 from datahub.emitter.mce_builder import OwnerType
@@ -111,7 +111,7 @@ def __init__(
         self.owner_source_type = owner_source_type
         self.match_nested_props = match_nested_props
 
-    def process(self, raw_props: Dict[str, Any]) -> Dict[str, Any]:
+    def process(self, raw_props: Mapping[str, Any]) -> Dict[str, Any]:
         # Defining the following local variables -
         # operations_map - the final resulting map when operations are processed.
         # Against each operation the values to be applied are stored.
diff --git a/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py b/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py
index cbd5be9e7d832..4a69deb572fbd 100644
--- a/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py
+++ b/metadata-ingestion/tests/unit/data_lake/test_schema_inference.py
@@ -1,14 +1,14 @@
 import tempfile
 from typing import List, Type
 
-import avro.schema
 import pandas as pd
 import ujson
 from avro import schema as avro_schema
 from avro.datafile import DataFileWriter
 from avro.io import DatumWriter
 
-from datahub.ingestion.source.schema_inference import avro, csv_tsv, json, parquet
+from datahub.ingestion.source.schema_inference import csv_tsv, json, parquet
+from datahub.ingestion.source.schema_inference.avro import AvroInferrer
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BooleanTypeClass,
     NumberTypeClass,
@@ -123,7 +123,7 @@ def test_infer_schema_avro():
 
         file.seek(0)
 
-        fields = avro.AvroInferrer().infer_schema(file)
+        fields = AvroInferrer().infer_schema(file)
         fields.sort(key=lambda x: x.fieldPath)
 
         assert_field_paths_match(fields, expected_field_paths_avro)

From b9508e6dd50c5d0eaf8eddb21c5bdf55bec1646a Mon Sep 17 00:00:00 2001
From: siddiquebagwan-gslab <mohdsiddique.bagwan@gslab.com>
Date: Wed, 25 Oct 2023 23:48:15 +0530
Subject: [PATCH 100/259] fix(ingestion/redshift): fix schema field data type
 mappings

---
 .../src/datahub/ingestion/source/redshift/redshift.py  | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
index 26237a6ce12e0..c7d01021773b1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -218,6 +218,9 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
     ] = {
         "BYTES": BytesType,
         "BOOL": BooleanType,
+        "BOOLEAN": BooleanType,
+        "DOUBLE": NumberType,
+        "DOUBLE PRECISION": NumberType,
         "DECIMAL": NumberType,
         "NUMERIC": NumberType,
         "BIGNUMERIC": NumberType,
@@ -244,6 +247,13 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
         "CHARACTER": StringType,
         "CHAR": StringType,
         "TIMESTAMP WITHOUT TIME ZONE": TimeType,
+        "REAL": NumberType,
+        "VARCHAR": StringType,
+        "TIMESTAMPTZ": TimeType,
+        "GEOMETRY": NullType,
+        "HLLSKETCH": NullType,
+        "TIMETZ": TimeType,
+        "VARBYTE": StringType,
     }
 
     def get_platform_instance_id(self) -> str:

From 1c77bca0c68878ca5cb86f741ca77ce0aa497272 Mon Sep 17 00:00:00 2001
From: Younghoon YUN <dyhn78@gmail.com>
Date: Thu, 26 Oct 2023 05:01:47 +0900
Subject: [PATCH 101/259] fix(datahub-protobuf): add check if nested field is
 reserved (#9058)

---
 .../src/main/java/datahub/protobuf/model/ProtobufField.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java
index 42884241d9f7c..d890c373f1299 100644
--- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java
+++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java
@@ -259,7 +259,9 @@ private FieldDescriptorProto getNestedTypeFields(List<Integer> pathList, Descrip
             messageType = messageType.getNestedType(value);
         }
 
-        if (pathList.get(pathSize - 2) == DescriptorProto.FIELD_FIELD_NUMBER) {
+        if (pathList.get(pathSize - 2) == DescriptorProto.FIELD_FIELD_NUMBER
+                && pathList.get(pathSize - 1) != DescriptorProto.RESERVED_RANGE_FIELD_NUMBER
+                && pathList.get(pathSize - 1) != DescriptorProto.RESERVED_NAME_FIELD_NUMBER) {
             return messageType.getField(pathList.get(pathSize - 1));
         } else {
             return null;

From 32f5dcb1544e5a47efbb48d39b215d3bdc33535b Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Wed, 25 Oct 2023 13:16:49 -0700
Subject: [PATCH 102/259] fix(ingest): better handling around sink errors
 (#9003)

---
 .../src/datahub/ingestion/run/pipeline.py     | 10 +++++-
 .../datahub/ingestion/sink/datahub_kafka.py   | 33 ++++++++-----------
 2 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
index 07b55e0e25a89..f2735c24ca19d 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py
@@ -390,7 +390,15 @@ def run(self) -> None:
                         record_envelopes = self.extractor.get_records(wu)
                         for record_envelope in self.transform(record_envelopes):
                             if not self.dry_run:
-                                self.sink.write_record_async(record_envelope, callback)
+                                try:
+                                    self.sink.write_record_async(
+                                        record_envelope, callback
+                                    )
+                                except Exception as e:
+                                    # In case the sink's error handling is bad, we still want to report the error.
+                                    self.sink.report.report_failure(
+                                        f"Failed to write record: {e}"
+                                    )
 
                     except RuntimeError:
                         raise
diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py
index 39054c256a7fd..38ddadaafc862 100644
--- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py
@@ -9,7 +9,6 @@
     MetadataChangeEvent,
     MetadataChangeProposal,
 )
-from datahub.metadata.schema_classes import MetadataChangeProposalClass
 
 
 class KafkaSinkConfig(KafkaEmitterConfig):
@@ -58,27 +57,21 @@ def write_record_async(
         ],
         write_callback: WriteCallback,
     ) -> None:
-        record = record_envelope.record
-        if isinstance(record, MetadataChangeEvent):
-            self.emitter.emit_mce_async(
+        callback = _KafkaCallback(
+            self.report, record_envelope, write_callback
+        ).kafka_callback
+        try:
+            record = record_envelope.record
+            self.emitter.emit(
                 record,
-                callback=_KafkaCallback(
-                    self.report, record_envelope, write_callback
-                ).kafka_callback,
-            )
-        elif isinstance(
-            record, (MetadataChangeProposalWrapper, MetadataChangeProposalClass)
-        ):
-            self.emitter.emit_mcp_async(
-                record,
-                callback=_KafkaCallback(
-                    self.report, record_envelope, write_callback
-                ).kafka_callback,
-            )
-        else:
-            raise ValueError(
-                f"The datahub-kafka sink only supports MetadataChangeEvent/MetadataChangeProposal[Wrapper] classes, not {type(record)}"
+                callback=callback,
             )
+        except Exception as err:
+            # In case we throw an exception while trying to emit the record,
+            # catch it and report the failure. This might happen if the schema
+            # registry is down or otherwise misconfigured, in which case we'd
+            # fail when serializing the record.
+            callback(err, f"Failed to write record: {err}")
 
     def close(self) -> None:
         self.emitter.flush()

From 6c932e8afeb4ac71c9b6b31e9fde3876c9e947cf Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 25 Oct 2023 16:17:09 -0400
Subject: [PATCH 103/259] feat(ingest/bigquery): Attempt to support raw dataset
 pattern (#9109)

---
 docs/how/updating-datahub.md                  |  8 +--
 .../source/bigquery_v2/bigquery_config.py     | 18 ++++++-
 .../tests/unit/test_bigquery_source.py        | 53 +++++++++++++++++++
 3 files changed, 74 insertions(+), 5 deletions(-)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 8813afee65be9..4d1535f28fa0a 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -53,10 +53,10 @@ into
 for example, using `datahub put` command. Policies can be also removed and re-created via UI.
 - #9077 - The BigQuery ingestion source by default sets `match_fully_qualified_names: true`.
 This means that any `dataset_pattern` or `schema_pattern` specified will be matched on the fully
-qualified dataset name, i.e. `<project_name>.<dataset_name>`. If this is not the case, please
-update your pattern (e.g. prepend your old dataset pattern with `.*\.` which matches the project part), 
-or set `match_fully_qualified_names: false` in your recipe. However, note that
-setting this to `false` is deprecated and this flag will be removed entirely in a future release.
+qualified dataset name, i.e. `<project_name>.<dataset_name>`. We attempt to support the old
+pattern format by prepending `.*\\.` to dataset patterns lacking a period, so in most cases this
+should not cause any issues. However, if you have a complex dataset pattern, we recommend you
+manually convert it to the fully qualified format to avoid any potential issues.
 
 ### Potential Downtime
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index a6a740385cf5c..6203192769750 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -299,7 +299,7 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
                 "use project_id_pattern whenever possible. project_id will be deprecated, please use project_id_pattern only if possible."
             )
 
-        dataset_pattern = values.get("dataset_pattern")
+        dataset_pattern: Optional[AllowDenyPattern] = values.get("dataset_pattern")
         schema_pattern = values.get("schema_pattern")
         if (
             dataset_pattern == AllowDenyPattern.allow_all()
@@ -329,6 +329,22 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
                 "Please update `dataset_pattern` to match against fully qualified schema name `<project_id>.<dataset_name>` and set config `match_fully_qualified_names : True`."
                 "The config option `match_fully_qualified_names` is deprecated and will be removed in a future release."
             )
+        elif match_fully_qualified_names and dataset_pattern is not None:
+            adjusted = False
+            for lst in [dataset_pattern.allow, dataset_pattern.deny]:
+                for i, pattern in enumerate(lst):
+                    if "." not in pattern:
+                        if pattern.startswith("^"):
+                            lst[i] = r"^.*\." + pattern[1:]
+                        else:
+                            lst[i] = r".*\." + pattern
+                        adjusted = True
+            if adjusted:
+                logger.warning(
+                    "`dataset_pattern` was adjusted to match against fully qualified schema names,"
+                    " of the form `<project_id>.<dataset_name>`."
+                )
+
         return values
 
     def get_table_pattern(self, pattern: List[str]) -> str:
diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py
index 5a11a933c8595..4cfa5c48d2377 100644
--- a/metadata-ingestion/tests/unit/test_bigquery_source.py
+++ b/metadata-ingestion/tests/unit/test_bigquery_source.py
@@ -53,6 +53,59 @@ def test_bigquery_uri_on_behalf():
     assert config.get_sql_alchemy_url() == "bigquery://test-project-on-behalf"
 
 
+def test_bigquery_dataset_pattern():
+    config = BigQueryV2Config.parse_obj(
+        {
+            "dataset_pattern": {
+                "allow": [
+                    "test-dataset",
+                    "test-project.test-dataset",
+                    ".*test-dataset",
+                ],
+                "deny": [
+                    "^test-dataset-2$",
+                    "project\\.second_dataset",
+                ],
+            },
+        }
+    )
+    assert config.dataset_pattern.allow == [
+        r".*\.test-dataset",
+        r"test-project.test-dataset",
+        r".*test-dataset",
+    ]
+    assert config.dataset_pattern.deny == [
+        r"^.*\.test-dataset-2$",
+        r"project\.second_dataset",
+    ]
+
+    config = BigQueryV2Config.parse_obj(
+        {
+            "dataset_pattern": {
+                "allow": [
+                    "test-dataset",
+                    "test-project.test-dataset",
+                    ".*test-dataset",
+                ],
+                "deny": [
+                    "^test-dataset-2$",
+                    "project\\.second_dataset",
+                ],
+            },
+            "match_fully_qualified_names": False,
+        }
+    )
+    assert config.dataset_pattern.allow == [
+        r"test-dataset",
+        r"test-project.test-dataset",
+        r".*test-dataset",
+    ]
+    assert config.dataset_pattern.deny == [
+        r"^test-dataset-2$",
+        r"project\.second_dataset",
+    ]
+
+
 def test_bigquery_uri_with_credential():
     expected_credential_json = {
         "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",

From 2ebf33eb13d14c17bc6cb0eaee3a97dba33ea338 Mon Sep 17 00:00:00 2001
From: Zachary McNellis <zacharymcnellis@gmail.com>
Date: Wed, 25 Oct 2023 16:25:41 -0400
Subject: [PATCH 104/259] docs(observability): Column Assertion user guide
 (#9106)

Co-authored-by: John Joyce <john@acryl.io>
---
 docs-website/sidebars.js                      |   1 +
 .../observe/column-assertions.md              | 358 ++++++++++++++++++
 2 files changed, 359 insertions(+)
 create mode 100644 docs/managed-datahub/observe/column-assertions.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index b2b3df4dfb33c..31d69aec46d8b 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -446,6 +446,7 @@ module.exports = {
             "docs/managed-datahub/observe/freshness-assertions",
             "docs/managed-datahub/observe/volume-assertions",
             "docs/managed-datahub/observe/custom-sql-assertions",
+            "docs/managed-datahub/observe/column-assertions",
           ],
         },
         {
diff --git a/docs/managed-datahub/observe/column-assertions.md b/docs/managed-datahub/observe/column-assertions.md
new file mode 100644
index 0000000000000..99a764f771676
--- /dev/null
+++ b/docs/managed-datahub/observe/column-assertions.md
@@ -0,0 +1,358 @@
+---
+description: This page provides an overview of working with DataHub Column Assertions
+---
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+
+# Column Assertions
+
+<FeatureAvailability saasOnly />
+
+> ⚠️ The **Column Assertions** feature is currently in private beta, part of the **Acryl Observe** module, and may only 
+> be available to a limited set of design partners.
+>
+> If you are interested in trying it and providing feedback, please reach out to your Acryl Customer Success
+> representative.
+
+## Introduction
+
+Can you remember a time when an important warehouse table column changed dramatically, with little or no notice? Perhaps the number of null values suddenly spiked, or a new value was added to a fixed set of possible values. If the answer is yes, how did you initially find out? We'll take a guess - someone looking at an internal reporting dashboard or worse, a user using your your product, sounded an alarm when a number looked a bit out of the ordinary.
+
+There are many reasons why important columns in your Snowflake, Redshift, or BigQuery tables may change - application code bugs, new feature rollouts, etc. Oftentimes, these changes break important assumptions made about the data used in building key downstream data products like reporting dashboards or data-driven product features.
+
+What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data issues before anyone else? With Acryl DataHub Column  Assertions, you can.
+
+With Acryl DataHub, you can define **Column Value** assertions to ensure each value in a column matches specific constraints, and **Column Metric** assertions to ensure that computed metrics from columns align with your expectations. As soon as things go wrong, your team will be the first to know, before the data issue becomes a larger data incident. 
+
+In this guide, we'll cover the basics of Column Assertions - what they are, how to configure them, and more - so that you and your team can start building trust in your most important data assets.
+
+Let's dive in!
+
+## Support
+
+Column Assertions are currently supported for:
+
+1. Snowflake
+2. Redshift
+3. BigQuery
+
+Note that an Ingestion Source _must_ be configured with the data platform of your choice in
+Acryl DataHub's **Ingestion** tab.
+
+> Note that Column Assertions are not yet supported if you are connecting to your warehouse
+> using the DataHub CLI or a Remote Ingestion Executor.
+
+## What is a Column Assertion?
+
+A **Column Assertion** is a highly configurable Data Quality rule used to monitor specific columns of a Data Warehouse table for unexpected changes. 
+
+Column Assertions are defined to validate a specific column, and can be used to 
+
+1. Validate that the values of the column match some constraints (regex, allowed values, max, min, etc) across rows OR
+2. Validate that specific column aggregation metrics match some expectations across rows.
+
+Column Assertions can be particularly useful for documenting and enforcing column-level "contracts", i.e. formal specifications about the expected contents of a particular column that can be used for coordinating among producers and consumers of the data.  
+
+### Anatomy of Column Assertion
+
+Column Assertions can be divided into two main types: **Column Value** and **Column Metric** Assertions.
+
+A **Column Value Assertion** is used to monitor the value of a specific column in a table, and ensure that every row
+adheres to a specific condition. In comparison, a **Column Metric Assertion** is used to compute a metric for that column,
+and ensure that the value of that metric adheres to a specific condition.
+
+At the most basic level, both types consist of a few important parts:
+
+1. An **Evaluation Schedule**
+2. A **Column Selection**
+3. A **Evaluation Criteria**
+4. A **Row Evaluation Type**
+
+In this section, we'll give an overview of each.
+
+#### 1. Evaluation Schedule
+
+The **Evaluation Schedule**: This defines how often to evaluate the Column Assertion against the given warehouse table.
+This should usually be configured to match the expected change frequency of the table, although it can also be less
+frequently depending on your requirements. You can also specify specific days of the week, hours in the day, or even
+minutes in an hour.
+
+#### 2. Column Selection
+
+The **Column Selection**: This defines the column that should be monitored by the Column Assertion. You can choose from
+any of the columns from the table listed in the dropdown. Note that columns of struct / object type are not currently supported.
+
+#### 3. Evaluation Criteria
+
+The **Evaluation Criteria**: This defines the condition that must be satisfied in order for the Column
+Assertion to pass.
+
+For **Column Value Assertions**, you will be able to choose from a set of operators that can be applied to the column
+value. The options presented will vary based on the data type of the selected column. For example, if you've selected a numeric column, you
+can verify that the column value is greater than a particular value. For string types, you can check that the column value
+matches a particular regex pattern. Additionally, you are able to control the behavior of the check in the presence of NULL values. If the
+**Allow Nulls** option is _disabled_, then any null values encountered will be reported as a failure when evaluating the
+assertion. If **Allow Nulls** is enabled, then nulls will be ignored; the condition will be evaluated for rows where the column value is non-null.  
+
+For **Column Metric Assertions**, you will be able to choose from a list of common column metrics - MAX, MIN, MEAN, NULL COUNT, etc - and then compare these metric values to an expected value. The list of metrics will vary based on the type of the selected column. For example
+if you've selected a numeric column, you can choose to compute the MEAN value of the column, and then assert that it is greater than a
+specific number. For string types, you can choose to compute the MAX LENGTH of the string across all column values, and then assert that it
+is less than a specific number.
+
+#### 4. Row Selection Set
+
+The **Row Selection Set**: This defines which rows in the table the Column Assertion will be evaluated across. You can choose
+from the following options:
+
+- **All Table Rows**: Evaluate the Column Assertion across all rows in the table. This is the default option. Note that
+this may not be desirable for large tables.
+
+- **Only Rows That Have Changed**: Evaluate the Column Assertion only against rows that have changed since the last
+evaluation of the assertion. If you choose this option, you will need to specify a **High Watermark Column** to help determine which rows
+have changed. A **High Watermark Column** is a column that contains a constantly incrementing value - a date, a time, or
+another always-increasing number - that can be used to find the "new rows" that were added since previous evaluation. When selected, a query will be issued to the table to find only the rows that have changed since the previous assertion evaluation. 
+
+## Creating a Column Assertion
+
+### Prerequisites
+
+1. **Permissions**: To create or delete Column Assertions for a specific entity on DataHub, you'll need to be granted the
+   `Edit Assertions` and `Edit Monitors` privileges for the entity. This is granted to Entity owners by default.
+
+2. **Data Platform Connection**: In order to create a Column Assertion, you'll need to have an **Ingestion Source** 
+   configured to your Data Platform: Snowflake, BigQuery, or Redshift under the **Ingestion** tab.
+
+Once these are in place, you're ready to create your Column Assertions!
+
+### Steps
+
+1. Navigate to the Table that you want to monitor
+2. Click the **Validations** tab
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/profile-validation-tab.png"/>
+</p>
+
+3. Click **+ Create Assertion**
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-choose-type.png"/>
+</p>
+
+4. Choose **Column**
+
+5. Configure the evaluation **schedule**. This is the frequency at which the assertion will be evaluated to produce a
+   pass or fail result, and the times when the column values will be checked.
+
+6. Configure the **column assertion type**. You can choose from **Column Value** or **Column Metric**.
+   **Column Value** assertions are used to monitor the value of a specific column in a table, and ensure that every row
+   adheres to a specific condition. **Column Metric** assertions are used to compute a metric for that column, and then compare the value of that metric to your expectations.
+
+<p align="center">
+  <img width="45%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-assertion-type.png"/>
+</p>
+
+7. Configure the **column selection**. This defines the column that should be monitored by the Column Assertion. 
+   You can choose from any of the columns from the table listed in the dropdown.
+
+<p align="center">
+  <img width="45%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-field-selection.png"/>
+</p>
+
+8. Configure the **evaluation criteria**. This step varies based on the type of assertion you chose in the previous step.
+
+   - **Column Value Assertions**: You will be able to choose from a set of operators that can be applied to the column
+     value. The options presented will vary based on the data type of the selected column. For example with numeric types, you
+     can check that the column value is greater than a specific value. For string types, you can check that the column value
+     matches a particular regex pattern. You will also be able to control the behavior of null values in the column. If the
+     **Allow Nulls** option is _disabled_, any null values encountered will be reported as a failure when evaluating the
+     assertion.
+
+   - **Column Metric Assertions**: You will be able to choose from a list of common metrics and then specify the operator
+     and value to compare against. The list of metrics will vary based on the data type of the selected column. For example
+     with numeric types, you can choose to compute the average value of the column, and then assert that it is greater than a
+     specific number. For string types, you can choose to compute the max length of all column values, and then assert that it
+     is less than a specific number.
+
+9. Configure the **row evaluation type**. This defines which rows in the table the Column Assertion should evaluate. You can choose
+   from the following options:
+
+   - **All Table Rows**: Evaluate the Column Assertion against all rows in the table. This is the default option. Note that
+     this may not be desirable for large tables.
+
+   - **Only Rows That Have Changed**: Evaluate the Column Assertion only against rows that have changed since the last
+     evaluation. If you choose this option, you will need to specify a **High Watermark Column** to help determine which rows
+     have changed. A **High Watermark Column** is a column that contains a constantly-incrementing value - a date, a time, or
+     another always-increasing number. When selected, a query will be issued to the table find only the rows which have changed since the last assertion run. 
+
+<p align="center">
+  <img width="60%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/assertion-builder-column-row-evaluation-type.png"/>
+</p>
+
+10. (Optional) Click **Advanced** to further customize the Column Assertion. The options listed here will vary based on the
+    type of assertion you chose in the previous step.
+
+    - **Invalid Values Threshold**: For **Column Value** assertions, you can configure the number of invalid values
+      (i.e. rows) that are allowed to fail before the assertion is marked as failing. This is useful if you want to allow a limited number
+      of invalid values in the column. By default this is 0, meaning the assertion will fail if any rows have an invalid column value.
+
+    - **Source**: For **Column Metric** assertions, you can choose the mechanism that will be used to obtain the column
+      metric. **Query** will issue a query to the dataset to compute the metric. **DataHub Dataset Profile** will use the
+      DataHub Dataset Profile metadata to compute the metric. Note that this option requires that dataset profiling
+      statistics are up-to-date as of the assertion run time.
+
+    - **Additional Filters**: You can choose to add additional filters to the query that will be used to evaluate the
+      assertion. This is useful if you want to limit the assertion to a subset of rows in the table. Note this option will not
+      be available if you choose **DataHub Dataset Profile** as the **source**.
+
+11. Click **Next**
+12. Configure actions that should be taken when the Column Assertion passes or fails
+
+<p align="left">
+  <img width="45%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-builder-actions.png"/>
+</p>
+
+- **Raise incident**: Automatically raise a new DataHub `Column` Incident for the Table whenever the Column Assertion is failing. This
+  may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when
+  an incident is created due to an Assertion failure.
+- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Column Assertion. Note that
+  any other incidents will not be impacted.
+
+10. Click **Save**.
+
+And that's it! DataHub will now begin to monitor your Column Assertion for the table.
+
+To view the time of the next Column Assertion evaluation, simply click **Column** and then click on your
+new Assertion:
+
+<p align="center">
+  <img width="40%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/assertion-next-evaluation-time.png"/>
+</p>
+
+Once your assertion has run, you will begin to see Success or Failure status for the Table
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/column/profile-passing-column-assertions-expanded.png"/>
+</p>
+
+## Stopping a Column Assertion
+
+In order to temporarily stop the evaluation of a Column Assertion:
+
+1. Navigate to the **Validations** tab of the table with the assertion
+2. Click **Column** to open the Column Assertions list
+3. Click the three-dot menu on the right side of the assertion you want to disable
+4. Click **Stop**
+
+<p align="left">
+  <img width="25%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/manage-assertion-menu.png"/>
+</p>
+
+To resume the Column Assertion, simply click **Turn On**.
+
+<p align="center">
+  <img width="90%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/observe/freshness/stopped-assertion.png"/>
+</p>
+
+## Creating Column Assertions via API
+
+Under the hood, Acryl DataHub implements Column Assertion Monitoring using two "entity" concepts:
+
+- **Assertion**: The specific expectation for the column metric. e.g. "The value of an integer column is greater than 10 for all rows in the table." This is the "what".
+
+- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific
+  mechanisms. This is the "how".
+
+Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the
+`Edit Assertions` and `Edit Monitors` privileges for it.
+
+#### GraphQL
+
+In order to create a Column Assertion that is being monitored on a specific **Evaluation Schedule**, you'll need to use 2
+GraphQL mutation queries to create a Column Assertion entity and create an Assertion Monitor entity responsible for evaluating it.
+
+Start by creating the Column Assertion entity using the `createFieldAssertion` query and hang on to the 'urn' field of the Assertion entity
+you get back. Then continue by creating a Monitor entity using the `createAssertionMonitor`.
+
+##### Examples
+
+To create a Column Assertion Entity that checks that the value of an integer column is greater than 10:
+
+```json
+mutation createFieldAssertion {
+  createFieldAssertion(
+  input: {
+    entityUrn: "<urn of the table to be monitored>",
+    type: FIELD_VALUES,
+    fieldValuesAssertion: {
+      field: {
+        path: "<name of the column to be monitored>",
+        type: "NUMBER",
+        nativeType: "NUMBER(38,0)"
+      },
+      operator: GREATER_THAN,
+      parameters: {
+        value: {
+          type: NUMBER,
+          value: "10"
+        }
+      },
+      failThreshold: {
+        type: COUNT,
+        value: 0
+      },
+      excludeNulls: true
+    }
+  }
+  ) {
+  urn
+}
+}
+```
+
+To create an Assertion Monitor Entity that evaluates the column assertion every 8 hours using all rows in the table:
+
+```json
+mutation createAssertionMonitor {
+  createAssertionMonitor(
+    input: {
+      entityUrn: "<urn of entity being monitored>",
+      assertionUrn: "<urn of assertion created in first query>",
+      schedule: {
+        cron: "0 */8 * * *",
+        timezone: "America/Los_Angeles"
+      },
+      parameters: {
+        type: DATASET_FIELD,
+        datasetFieldParameters: {
+          sourceType: ALL_ROWS_QUERY
+        }
+      }
+    }
+  ) {
+    urn
+  }
+}
+```
+
+This entity defines _when_ to run the check (Using CRON format - every 8th hour) and _how_ to run the check (using a query against all rows of the table).
+
+After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone.
+
+You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`.
+
+### Tips
+
+:::info
+**Authorization**
+
+Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows:
+
+```
+Authorization: Bearer <personal-access-token>
+```
+
+**Exploring GraphQL API**
+
+Also, remember that you can play with an interactive version of the Acryl GraphQL API at `https://your-account-id.acryl.io/api/graphiql`
+:::

From f402090c1ebec9601e5fef6e45879d3a0a015dbd Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Thu, 26 Oct 2023 21:44:32 +0530
Subject: [PATCH 105/259] feat(ingest): support view lineage for all sqlalchemy
 sources (#9039)

---
 metadata-ingestion/setup.py                   |  52 +-
 .../src/datahub/configuration/common.py       |   2 +-
 .../datahub/emitter/sql_parsing_builder.py    |   5 +-
 .../api/incremental_lineage_helper.py         |  13 +-
 .../src/datahub/ingestion/api/source.py       |   1 +
 .../ingestion/source/dbt/dbt_common.py        |   5 +
 .../source/snowflake/snowflake_lineage_v2.py  |  14 +-
 .../src/datahub/ingestion/source/sql/hive.py  |  83 ++-
 .../datahub/ingestion/source/sql/postgres.py  |  20 +-
 .../ingestion/source/sql/sql_common.py        | 126 +++-
 .../ingestion/source/sql/sql_config.py        |  19 +-
 .../datahub/ingestion/source/sql/teradata.py  |  54 +-
 .../source/sql/two_tier_sql_source.py         |   6 +-
 .../datahub/ingestion/source/sql/vertica.py   |   2 +-
 .../source/state/stateful_ingestion_base.py   |   3 +-
 .../ingestion/source_config/sql/snowflake.py  |  12 +-
 .../src/datahub/utilities/sqlglot_lineage.py  |  21 +-
 .../hive/hive_mces_all_db_golden.json         | 581 +++++++++++++++---
 .../integration/hive/hive_mces_golden.json    | 530 ++++++++++++++--
 .../tests/integration/hive/hive_setup.sql     |  22 +-
 .../mysql/mysql_mces_no_db_golden.json        | 272 ++++++--
 .../postgres_all_db_mces_with_db_golden.json  | 324 ++++++++--
 ..._db_to_file_with_db_estimate_row_count.yml |   2 +-
 .../postgres_mces_with_db_golden.json         | 264 +++++++-
 ...res_to_file_with_db_estimate_row_count.yml |   2 +-
 .../snowflake/test_snowflake_failures.py      |   3 +-
 .../trino/trino_hive_mces_golden.json         | 211 +++++--
 .../test_incremental_lineage_helper.py        |  21 +
 28 files changed, 2193 insertions(+), 477 deletions(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 0b8661b0df5f5..7f7826abe2095 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -101,22 +101,36 @@
     "grpcio-tools>=1.44.0,<2",
 }
 
-sql_common = {
-    # Required for all SQL sources.
-    # This is temporary lower bound that we're open to loosening/tightening as requirements show up
-    "sqlalchemy>=1.4.39, <2",
-    # Required for SQL profiling.
-    "great-expectations>=0.15.12, <=0.15.50",
-    # scipy version restricted to reduce backtracking, used by great-expectations,
-    "scipy>=1.7.2",
-    # GE added handling for higher version of jinja2
-    # https://github.com/great-expectations/great_expectations/pull/5382/files
-    # datahub does not depend on traitlets directly but great expectations does.
-    # https://github.com/ipython/traitlets/issues/741
-    "traitlets<5.2.2",
-    "greenlet",
+usage_common = {
+    "sqlparse",
+}
+
+sqlglot_lib = {
+    # Using an Acryl fork of sqlglot.
+    # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
+    "acryl-sqlglot==18.5.2.dev45",
 }
 
+sql_common = (
+    {
+        # Required for all SQL sources.
+        # This is temporary lower bound that we're open to loosening/tightening as requirements show up
+        "sqlalchemy>=1.4.39, <2",
+        # Required for SQL profiling.
+        "great-expectations>=0.15.12, <=0.15.50",
+        # scipy version restricted to reduce backtracking, used by great-expectations,
+        "scipy>=1.7.2",
+        # GE added handling for higher version of jinja2
+        # https://github.com/great-expectations/great_expectations/pull/5382/files
+        # datahub does not depend on traitlets directly but great expectations does.
+        # https://github.com/ipython/traitlets/issues/741
+        "traitlets<5.2.2",
+        "greenlet",
+    }
+    | usage_common
+    | sqlglot_lib
+)
+
 sqllineage_lib = {
     "sqllineage==1.3.8",
     # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage.
@@ -125,12 +139,6 @@
     "sqlparse==0.4.4",
 }
 
-sqlglot_lib = {
-    # Using an Acryl fork of sqlglot.
-    # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
-    "acryl-sqlglot==18.5.2.dev45",
-}
-
 aws_common = {
     # AWS Python SDK
     "boto3",
@@ -243,10 +251,6 @@
 
 powerbi_report_server = {"requests", "requests_ntlm"}
 
-usage_common = {
-    "sqlparse",
-}
-
 databricks = {
     # 0.1.11 appears to have authentication issues with azure databricks
     "databricks-sdk>=0.9.0",
diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py
index c909b89eb0c2d..73ac4baac48c0 100644
--- a/metadata-ingestion/src/datahub/configuration/common.py
+++ b/metadata-ingestion/src/datahub/configuration/common.py
@@ -283,7 +283,7 @@ class VersionedConfig(ConfigModel):
 
 class LineageConfig(ConfigModel):
     incremental_lineage: bool = Field(
-        default=True,
+        default=False,
         description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
     )
 
diff --git a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
index dedcfa0385f75..cedaa4fbbd7f6 100644
--- a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
@@ -106,6 +106,7 @@ def process_sql_parsing_result(
         user: Optional[UserUrn] = None,
         custom_operation_type: Optional[str] = None,
         include_urns: Optional[Set[DatasetUrn]] = None,
+        include_column_lineage: bool = True,
     ) -> Iterable[MetadataWorkUnit]:
         """Process a single query and yield any generated workunits.
 
@@ -130,7 +131,9 @@ def process_sql_parsing_result(
                 _merge_lineage_data(
                     downstream_urn=downstream_urn,
                     upstream_urns=result.in_tables,
-                    column_lineage=result.column_lineage,
+                    column_lineage=result.column_lineage
+                    if include_column_lineage
+                    else None,
                     upstream_edges=self._lineage_map[downstream_urn],
                     query_timestamp=query_timestamp,
                     is_view_ddl=is_view_ddl,
diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
index 9478c5cf7efa2..945b201ca5758 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
@@ -130,10 +130,13 @@ def auto_incremental_lineage(
                 if len(wu.metadata.proposedSnapshot.aspects) > 0:
                     yield wu
 
-            yield _lineage_wu_via_read_modify_write(
-                graph, urn, lineage_aspect, wu.metadata.systemMetadata
-            ) if lineage_aspect.fineGrainedLineages else _convert_upstream_lineage_to_patch(
-                urn, lineage_aspect, wu.metadata.systemMetadata
-            )
+            if lineage_aspect.fineGrainedLineages:
+                yield _lineage_wu_via_read_modify_write(
+                    graph, urn, lineage_aspect, wu.metadata.systemMetadata
+                )
+            elif lineage_aspect.upstreams:
+                yield _convert_upstream_lineage_to_patch(
+                    urn, lineage_aspect, wu.metadata.systemMetadata
+                )
         else:
             yield wu
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py
index b86844b1c4c83..8940642f7008a 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source.py
@@ -215,6 +215,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
             )
         ):
             auto_lowercase_dataset_urns = auto_lowercase_urns
+
         return [
             auto_lowercase_dataset_urns,
             auto_status_aspect,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 48d2118a9b091..c4de24bf192f1 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -280,6 +280,11 @@ class DBTCommonConfig(
         default=False,
         description="When enabled, dbt test warnings will be treated as failures.",
     )
+    # override fault value to True.
+    incremental_lineage: bool = Field(
+        default=True,
+        description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
+    )
 
     @validator("target_platform")
     def validate_target_platform_value(cls, target_platform: str) -> str:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
index 0a15c352fc842..9649054dbe6cb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
@@ -136,7 +136,6 @@ def get_workunits(
             return
 
         self._populate_external_lineage_map(discovered_tables)
-
         if self.config.include_view_lineage:
             if len(discovered_views) > 0:
                 yield from self.get_view_upstream_workunits(
@@ -200,14 +199,15 @@ def _gen_workunit_from_sql_parsing_result(
         self,
         dataset_identifier: str,
         result: SqlParsingResult,
-    ) -> MetadataWorkUnit:
+    ) -> Iterable[MetadataWorkUnit]:
         upstreams, fine_upstreams = self.get_upstreams_from_sql_parsing_result(
             self.dataset_urn_builder(dataset_identifier), result
         )
-        self.report.num_views_with_upstreams += 1
-        return self._create_upstream_lineage_workunit(
-            dataset_identifier, upstreams, fine_upstreams
-        )
+        if upstreams:
+            self.report.num_views_with_upstreams += 1
+            yield self._create_upstream_lineage_workunit(
+                dataset_identifier, upstreams, fine_upstreams
+            )
 
     def _gen_workunits_from_query_result(
         self,
@@ -251,7 +251,7 @@ def get_view_upstream_workunits(
                     )
                     if result:
                         views_processed.add(view_identifier)
-                        yield self._gen_workunit_from_sql_parsing_result(
+                        yield from self._gen_workunit_from_sql_parsing_result(
                             view_identifier, result
                         )
                 self.report.view_lineage_parse_secs = timer.elapsed_seconds()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
index 63b21bc82eddd..d081acb6c1eff 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
@@ -1,15 +1,18 @@
 import json
 import logging
 import re
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 from pydantic.class_validators import validator
 from pydantic.fields import Field
 
 # This import verifies that the dependencies are available.
 from pyhive import hive  # noqa: F401
-from pyhive.sqlalchemy_hive import HiveDate, HiveDecimal, HiveTimestamp
+from pyhive.sqlalchemy_hive import HiveDate, HiveDecimal, HiveDialect, HiveTimestamp
+from sqlalchemy.engine.reflection import Inspector
 
+from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.decorators import (
     SourceCapability,
     SupportStatus,
@@ -18,8 +21,10 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.extractor import schema_util
-from datahub.ingestion.source.sql.sql_common import register_custom_type
+from datahub.ingestion.source.sql.sql_common import SqlWorkUnit, register_custom_type
+from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
 from datahub.ingestion.source.sql.two_tier_sql_source import (
     TwoTierSQLAlchemyConfig,
     TwoTierSQLAlchemySource,
@@ -31,6 +36,7 @@
     SchemaField,
     TimeTypeClass,
 )
+from datahub.metadata.schema_classes import ViewPropertiesClass
 from datahub.utilities import config_clean
 from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column
 
@@ -90,19 +96,34 @@ def dbapi_get_columns_patched(self, connection, table_name, schema=None, **kw):
     logger.warning(f"Failed to patch method due to {e}")
 
 
+@reflection.cache  # type: ignore
+def get_view_names_patched(self, connection, schema=None, **kw):
+    query = "SHOW VIEWS"
+    if schema:
+        query += " IN " + self.identifier_preparer.quote_identifier(schema)
+    return [row[0] for row in connection.execute(query)]
+
+
+@reflection.cache  # type: ignore
+def get_view_definition_patched(self, connection, view_name, schema=None, **kw):
+    full_table = self.identifier_preparer.quote_identifier(view_name)
+    if schema:
+        full_table = "{}.{}".format(
+            self.identifier_preparer.quote_identifier(schema),
+            self.identifier_preparer.quote_identifier(view_name),
+        )
+    row = connection.execute("SHOW CREATE TABLE {}".format(full_table)).fetchone()
+    return row[0]
+
+
+HiveDialect.get_view_names = get_view_names_patched
+HiveDialect.get_view_definition = get_view_definition_patched
+
+
 class HiveConfig(TwoTierSQLAlchemyConfig):
     # defaults
     scheme = Field(default="hive", hidden_from_docs=True)
 
-    # Hive SQLAlchemy connector returns views as tables.
-    # See https://github.com/dropbox/PyHive/blob/b21c507a24ed2f2b0cf15b0b6abb1c43f31d3ee0/pyhive/sqlalchemy_hive.py#L270-L273.
-    # Disabling views helps us prevent this duplication.
-    include_views = Field(
-        default=False,
-        hidden_from_docs=True,
-        description="Hive SQLAlchemy connector returns views as tables. See https://github.com/dropbox/PyHive/blob/b21c507a24ed2f2b0cf15b0b6abb1c43f31d3ee0/pyhive/sqlalchemy_hive.py#L270-L273. Disabling views helps us prevent this duplication.",
-    )
-
     @validator("host_port")
     def clean_host_port(cls, v):
         return config_clean.remove_protocol(v)
@@ -174,3 +195,41 @@ def get_schema_fields_for_column(
             return new_fields
 
         return fields
+
+    # Hive SQLAlchemy connector returns views as tables in get_table_names.
+    # See https://github.com/dropbox/PyHive/blob/b21c507a24ed2f2b0cf15b0b6abb1c43f31d3ee0/pyhive/sqlalchemy_hive.py#L270-L273.
+    # This override makes sure that we ingest view definitions for views
+    def _process_view(
+        self,
+        dataset_name: str,
+        inspector: Inspector,
+        schema: str,
+        view: str,
+        sql_config: SQLCommonConfig,
+    ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]:
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            self.platform,
+            dataset_name,
+            self.config.platform_instance,
+            self.config.env,
+        )
+
+        try:
+            view_definition = inspector.get_view_definition(view, schema)
+            if view_definition is None:
+                view_definition = ""
+            else:
+                # Some dialects return a TextClause instead of a raw string,
+                # so we need to convert them to a string.
+                view_definition = str(view_definition)
+        except NotImplementedError:
+            view_definition = ""
+
+        if view_definition:
+            view_properties_aspect = ViewPropertiesClass(
+                materialized=False, viewLanguage="SQL", viewLogic=view_definition
+            )
+            yield MetadataChangeProposalWrapper(
+                entityUrn=dataset_urn,
+                aspect=view_properties_aspect,
+            ).as_workunit()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
index a6a9d8e2c8597..4f133c6459a0f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
@@ -103,10 +103,6 @@ class BasePostgresConfig(BasicSQLAlchemyConfig):
 
 
 class PostgresConfig(BasePostgresConfig):
-    include_view_lineage = Field(
-        default=False, description="Include table lineage for views"
-    )
-
     database_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
         description=(
@@ -183,9 +179,10 @@ def get_inspectors(self) -> Iterable[Inspector]:
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
         yield from super().get_workunits_internal()
 
-        for inspector in self.get_inspectors():
-            if self.config.include_view_lineage:
-                yield from self._get_view_lineage_workunits(inspector)
+        if self.views_failed_parsing:
+            for inspector in self.get_inspectors():
+                if self.config.include_view_lineage:
+                    yield from self._get_view_lineage_workunits(inspector)
 
     def _get_view_lineage_elements(
         self, inspector: Inspector
@@ -245,11 +242,14 @@ def _get_view_lineage_workunits(
             dependent_view, dependent_schema = key
 
             # Construct a lineage object.
+            view_identifier = self.get_identifier(
+                schema=dependent_schema, entity=dependent_view, inspector=inspector
+            )
+            if view_identifier not in self.views_failed_parsing:
+                return
             urn = mce_builder.make_dataset_urn_with_platform_instance(
                 platform=self.platform,
-                name=self.get_identifier(
-                    schema=dependent_schema, entity=dependent_view, inspector=inspector
-                ),
+                name=view_identifier,
                 platform_instance=self.config.platform_instance,
                 env=self.config.env,
             )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index fad9b9e8018a5..51909eaf4ed55 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -2,12 +2,14 @@
 import logging
 import traceback
 from dataclasses import dataclass, field
+from functools import partial
 from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
     Iterable,
     List,
+    MutableMapping,
     Optional,
     Set,
     Tuple,
@@ -29,7 +31,9 @@
     make_tag_urn,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
 from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import (
@@ -86,9 +90,16 @@
     ViewPropertiesClass,
 )
 from datahub.telemetry import telemetry
+from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.registries.domain_registry import DomainRegistry
 from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport
+from datahub.utilities.sqlglot_lineage import (
+    SchemaResolver,
+    SqlParsingResult,
+    sqlglot_lineage,
+    view_definition_lineage_helper,
+)
 
 if TYPE_CHECKING:
     from datahub.ingestion.source.ge_data_profiler import (
@@ -110,6 +121,11 @@ class SQLSourceReport(StaleEntityRemovalSourceReport):
 
     query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None
 
+    num_view_definitions_parsed: int = 0
+    num_view_definitions_failed_parsing: int = 0
+    num_view_definitions_failed_column_parsing: int = 0
+    view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList)
+
     def report_entity_scanned(self, name: str, ent_type: str = "table") -> None:
         """
         Entity could be a view or a table
@@ -319,6 +335,18 @@ def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str)
                 cached_domains=[k for k in self.config.domain], graph=self.ctx.graph
             )
 
+        self.views_failed_parsing: Set[str] = set()
+        self.schema_resolver: SchemaResolver = SchemaResolver(
+            platform=self.platform,
+            platform_instance=self.config.platform_instance,
+            env=self.config.env,
+        )
+        self._view_definition_cache: MutableMapping[str, str]
+        if self.config.use_file_backed_cache:
+            self._view_definition_cache = FileBackedDict[str]()
+        else:
+            self._view_definition_cache = {}
+
     def warn(self, log: logging.Logger, key: str, reason: str) -> None:
         self.report.report_warning(key, reason[:100])
         log.warning(f"{key} => {reason}")
@@ -455,6 +483,11 @@ def get_schema_level_workunits(
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
+            partial(
+                auto_incremental_lineage,
+                self.ctx.graph,
+                self.config.incremental_lineage,
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,
@@ -512,6 +545,35 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit
                     profile_requests, profiler, platform=self.platform
                 )
 
+        if self.config.include_view_lineage:
+            yield from self.get_view_lineage()
+
+    def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
+        builder = SqlParsingBuilder(
+            generate_lineage=True,
+            generate_usage_statistics=False,
+            generate_operations=False,
+        )
+        for dataset_name in self._view_definition_cache.keys():
+            view_definition = self._view_definition_cache[dataset_name]
+            result = self._run_sql_parser(
+                dataset_name,
+                view_definition,
+                self.schema_resolver,
+            )
+            if result and result.out_tables:
+                # This does not yield any workunits but we use
+                # yield here to execute this method
+                yield from builder.process_sql_parsing_result(
+                    result=result,
+                    query=view_definition,
+                    is_view_ddl=True,
+                    include_column_lineage=self.config.include_view_column_lineage,
+                )
+            else:
+                self.views_failed_parsing.add(dataset_name)
+        yield from builder.gen_workunits()
+
     def get_identifier(
         self, *, schema: str, entity: str, inspector: Inspector, **kwargs: Any
     ) -> str:
@@ -658,6 +720,8 @@ def _process_table(
             schema_fields,
         )
         dataset_snapshot.aspects.append(schema_metadata)
+        if self.config.include_view_lineage:
+            self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata)
         db_name = self.get_db_name(inspector)
 
         yield from self.add_table_to_schema_container(
@@ -862,6 +926,12 @@ def _process_view(
         view: str,
         sql_config: SQLCommonConfig,
     ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]:
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            self.platform,
+            dataset_name,
+            self.config.platform_instance,
+            self.config.env,
+        )
         try:
             columns = inspector.get_columns(view, schema)
         except KeyError:
@@ -877,6 +947,8 @@ def _process_view(
                 columns,
                 canonical_schema=schema_fields,
             )
+            if self.config.include_view_lineage:
+                self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata)
         description, properties, _ = self.get_table_properties(inspector, schema, view)
         try:
             view_definition = inspector.get_view_definition(view, schema)
@@ -890,12 +962,9 @@ def _process_view(
             view_definition = ""
         properties["view_definition"] = view_definition
         properties["is_view"] = "True"
-        dataset_urn = make_dataset_urn_with_platform_instance(
-            self.platform,
-            dataset_name,
-            self.config.platform_instance,
-            self.config.env,
-        )
+        if view_definition and self.config.include_view_lineage:
+            self._view_definition_cache[dataset_name] = view_definition
+
         dataset_snapshot = DatasetSnapshot(
             urn=dataset_urn,
             aspects=[StatusClass(removed=False)],
@@ -942,6 +1011,51 @@ def _process_view(
                 domain_registry=self.domain_registry,
             )
 
+    def _run_sql_parser(
+        self, view_identifier: str, query: str, schema_resolver: SchemaResolver
+    ) -> Optional[SqlParsingResult]:
+        try:
+            database, schema = self.get_db_schema(view_identifier)
+        except ValueError:
+            logger.warning(f"Invalid view identifier: {view_identifier}")
+            return None
+        raw_lineage = sqlglot_lineage(
+            query,
+            schema_resolver=schema_resolver,
+            default_db=database,
+            default_schema=schema,
+        )
+        view_urn = make_dataset_urn_with_platform_instance(
+            self.platform,
+            view_identifier,
+            self.config.platform_instance,
+            self.config.env,
+        )
+
+        if raw_lineage.debug_info.table_error:
+            logger.debug(
+                f"Failed to parse lineage for view {view_identifier}: "
+                f"{raw_lineage.debug_info.table_error}"
+            )
+            self.report.num_view_definitions_failed_parsing += 1
+            self.report.view_definitions_parsing_failures.append(
+                f"Table-level sql parsing error for view {view_identifier}: {raw_lineage.debug_info.table_error}"
+            )
+            return None
+
+        elif raw_lineage.debug_info.column_error:
+            self.report.num_view_definitions_failed_column_parsing += 1
+            self.report.view_definitions_parsing_failures.append(
+                f"Column-level sql parsing error for view {view_identifier}: {raw_lineage.debug_info.column_error}"
+            )
+        else:
+            self.report.num_view_definitions_parsed += 1
+        return view_definition_lineage_helper(raw_lineage, view_urn)
+
+    def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
+        database, schema, _view = dataset_identifier.split(".")
+        return database, schema
+
     def get_profiler_instance(self, inspector: Inspector) -> "DatahubGEProfiler":
         from datahub.ingestion.source.ge_data_profiler import DatahubGEProfiler
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 57aae32b361cf..095b8e6443171 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -6,7 +6,7 @@
 from pydantic import Field
 from sqlalchemy.engine import URL
 
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel, LineageConfig
 from datahub.configuration.source_common import (
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
@@ -28,6 +28,7 @@ class SQLCommonConfig(
     StatefulIngestionConfigBase,
     DatasetSourceConfigMixin,
     LowerCaseDatasetUrnConfigMixin,
+    LineageConfig,
 ):
     options: dict = pydantic.Field(
         default_factory=dict,
@@ -70,6 +71,22 @@ class SQLCommonConfig(
         description="If the source supports it, include table lineage to the underlying storage location.",
     )
 
+    include_view_lineage: bool = Field(
+        default=True,
+        description="Populates view->view and table->view lineage using DataHub's sql parser.",
+    )
+
+    include_view_column_lineage: bool = Field(
+        default=True,
+        description="Populates column-level lineage for  view->view and table->view lineage using DataHub's sql parser."
+        " Requires `include_view_lineage` to be enabled.",
+    )
+
+    use_file_backed_cache: bool = Field(
+        default=True,
+        description="Whether to use a file backed cache for the view definitions.",
+    )
+
     profiling: GEProfilingConfig = GEProfilingConfig()
     # Custom Stateful Ingestion settings
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index e628e4dbd3446..899a7b6697c0a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -1,7 +1,7 @@
 import logging
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Iterable, MutableMapping, Optional, Union
+from typing import Iterable, Optional, Union
 
 # This import verifies that the dependencies are available.
 import teradatasqlalchemy  # noqa: F401
@@ -33,14 +33,11 @@
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
-from datahub.metadata._schema_classes import SchemaMetadataClass, ViewPropertiesClass
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BytesTypeClass,
     TimeTypeClass,
 )
-from datahub.utilities.file_backed_collections import FileBackedDict
 from datahub.utilities.sqlglot_lineage import SchemaResolver, sqlglot_lineage
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -87,11 +84,6 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         "This requires to have the table lineage feature enabled.",
     )
 
-    include_view_lineage = Field(
-        default=True,
-        description="Whether to include view lineage in the ingestion. "
-        "This requires to have the view lineage feature enabled.",
-    )
     usage: BaseUsageConfig = Field(
         description="The usage config to use when generating usage statistics",
         default=BaseUsageConfig(),
@@ -107,11 +99,6 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         description="Generate usage statistic.",
     )
 
-    use_file_backed_cache: bool = Field(
-        default=True,
-        description="Whether to use a file backed cache for the view definitions.",
-    )
-
 
 @platform_name("Teradata")
 @config_class(TeradataConfig)
@@ -143,8 +130,6 @@ class TeradataSource(TwoTierSQLAlchemySource):
      and "timestamp" < TIMESTAMP '{end_time}'
      """
 
-    _view_definition_cache: MutableMapping[str, str]
-
     def __init__(self, config: TeradataConfig, ctx: PipelineContext):
         super().__init__(config, ctx, "teradata")
 
@@ -167,34 +152,11 @@ def __init__(self, config: TeradataConfig, ctx: PipelineContext):
             env=self.config.env,
         )
 
-        if self.config.use_file_backed_cache:
-            self._view_definition_cache = FileBackedDict[str]()
-        else:
-            self._view_definition_cache = {}
-
     @classmethod
     def create(cls, config_dict, ctx):
         config = TeradataConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
-    def get_view_lineage(self) -> Iterable[MetadataWorkUnit]:
-        for key in self._view_definition_cache.keys():
-            view_definition = self._view_definition_cache[key]
-            dataset_urn = DatasetUrn.create_from_string(key)
-
-            db_name: Optional[str] = None
-            # We need to get the default db from the dataset urn otherwise the builder generates the wrong urns
-            if "." in dataset_urn.get_dataset_name():
-                db_name = dataset_urn.get_dataset_name().split(".", 1)[0]
-
-            self.report.num_view_ddl_parsed += 1
-            if self.report.num_view_ddl_parsed % 1000 == 0:
-                logger.info(f"Parsed {self.report.num_queries_parsed} view ddl")
-
-            yield from self.gen_lineage_from_query(
-                query=view_definition, default_database=db_name, is_view_ddl=True
-            )
-
     def get_audit_log_mcps(self) -> Iterable[MetadataWorkUnit]:
         engine = self.get_metadata_engine()
         for entry in engine.execute(
@@ -252,19 +214,7 @@ def get_metadata_engine(self) -> Engine:
 
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
         # Add all schemas to the schema resolver
-        for wu in super().get_workunits_internal():
-            urn = wu.get_urn()
-            schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass)
-            if schema_metadata:
-                self.schema_resolver.add_schema_metadata(urn, schema_metadata)
-            view_properties = wu.get_aspect_of_type(ViewPropertiesClass)
-            if view_properties and self.config.include_view_lineage:
-                self._view_definition_cache[urn] = view_properties.viewLogic
-            yield wu
-
-        if self.config.include_view_lineage:
-            self.report.report_ingestion_stage_start("view lineage extraction")
-            yield from self.get_view_lineage()
+        yield from super().get_workunits_internal()
 
         if self.config.include_table_lineage or self.config.include_usage_statistics:
             self.report.report_ingestion_stage_start("audit log extraction")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
index 7a49551dc1235..efb1d3ffe119f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py
@@ -1,6 +1,6 @@
 import typing
 import urllib.parse
-from typing import Any, Dict, Iterable, Optional
+from typing import Any, Dict, Iterable, Optional, Tuple
 
 from pydantic.fields import Field
 from sqlalchemy import create_engine, inspect
@@ -71,6 +71,10 @@ def __init__(self, config, ctx, platform):
         super().__init__(config, ctx, platform)
         self.config: TwoTierSQLAlchemyConfig = config
 
+    def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
+        schema, _view = dataset_identifier.split(".", 1)
+        return None, schema
+
     def get_database_container_key(self, db_name: str, schema: str) -> ContainerKey:
         # Because our overridden get_allowed_schemas method returns db_name as the schema name,
         # the db_name and schema here will be the same. Hence, we just ignore the schema parameter.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
index a417cae2b1ab0..b89db755853bc 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py
@@ -86,7 +86,7 @@ class VerticaConfig(BasicSQLAlchemyConfig):
         default=True, description="Whether Models should be ingested."
     )
 
-    include_view_lineage: Optional[bool] = pydantic.Field(
+    include_view_lineage: bool = pydantic.Field(
         default=True,
         description="If the source supports it, include view lineage to the underlying storage location.",
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
index be97e9380f1f5..7fb2cf9813cab 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
@@ -11,7 +11,6 @@
     ConfigModel,
     ConfigurationError,
     DynamicTypedConfig,
-    LineageConfig,
 )
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
@@ -100,7 +99,7 @@ class StatefulIngestionConfigBase(GenericModel, Generic[CustomConfig]):
     )
 
 
-class StatefulLineageConfigMixin(LineageConfig):
+class StatefulLineageConfigMixin:
     enable_stateful_lineage_ingestion: bool = Field(
         default=True,
         description="Enable stateful lineage ingestion."
diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
index 0d72fc52da0ca..c3e8c175f1de5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
@@ -166,13 +166,17 @@ def _check_oauth_config(oauth_config: Optional[OAuthConfiguration]) -> None:
                 "but should be set when using use_certificate false for oauth_config"
             )
 
-    @pydantic.validator("include_view_lineage")
-    def validate_include_view_lineage(cls, v, values):
-        if not values.get("include_table_lineage") and v:
+    @pydantic.root_validator()
+    def validate_include_view_lineage(cls, values):
+        if (
+            "include_table_lineage" in values
+            and not values.get("include_table_lineage")
+            and values.get("include_view_lineage")
+        ):
             raise ValueError(
                 "include_table_lineage must be True for include_view_lineage to be set."
             )
-        return v
+        return values
 
     def get_sql_alchemy_url(
         self,
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 526d90b2a1bfa..1d74b20569814 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -623,9 +623,9 @@ def _schema_aware_fuzzy_column_resolve(
         statement = sqlglot.optimizer.annotate_types.annotate_types(
             statement, schema=sqlglot_db_schema
         )
-    except sqlglot.errors.OptimizeError as e:
+    except (sqlglot.errors.OptimizeError, sqlglot.errors.ParseError) as e:
         # This is not a fatal error, so we can continue.
-        logger.debug("sqlglot failed to annotate types: %s", e)
+        logger.debug("sqlglot failed to annotate or parse types: %s", e)
 
     try:
         assert isinstance(statement, _SupportedColumnLineageTypesTuple)
@@ -1156,3 +1156,20 @@ def create_lineage_sql_parsed_result(
     finally:
         if needs_close:
             schema_resolver.close()
+
+
+def view_definition_lineage_helper(
+    result: SqlParsingResult, view_urn: str
+) -> SqlParsingResult:
+    if result.query_type is QueryType.SELECT:
+        # Some platforms (e.g. postgres) store only <select statement> from view definition
+        # `create view V as <select statement>` . For such view definitions, `result.out_tables` and
+        # `result.column_lineage[].downstream` are empty in `sqlglot_lineage` response, whereas upstream
+        # details and downstream column details are extracted correctly.
+        # Here, we inject view V's urn in `result.out_tables` and `result.column_lineage[].downstream`
+        # to get complete lineage result.
+        result.out_tables = [view_urn]
+        if result.column_lineage:
+            for col_result in result.column_lineage:
+                col_result.downstream.table = view_urn
+    return result
diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
index f3b6d2b8138cc..6774d4c7055b9 100644
--- a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
+++ b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -93,7 +98,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -111,7 +117,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore",
@@ -121,7 +127,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -187,7 +193,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,7 +211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -224,7 +232,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -239,7 +248,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -257,17 +267,19 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: COLUMN_STATS_ACCURATE": "{\\\"BASIC_STATS\\\":\\\"true\\\"}",
+                            "Table Parameters: another.comment": "This table has no partitions",
+                            "Table Parameters: comment": "This table has array of structs",
                             "Table Parameters: numFiles": "1",
                             "Table Parameters: numRows": "1",
                             "Table Parameters: rawDataSize": "32",
                             "Table Parameters: totalSize": "33",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721976",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -278,6 +290,7 @@
                             "Storage Desc Params: serialization.format": "1"
                         },
                         "name": "array_struct_test",
+                        "description": "This table has array of structs",
                         "tags": []
                     }
                 },
@@ -304,6 +317,7 @@
                             {
                                 "fieldPath": "property_id",
                                 "nullable": true,
+                                "description": "id of property",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -316,6 +330,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
                                 "nullable": true,
+                                "description": "service types and providers",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.ArrayType": {
@@ -368,7 +383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -385,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -405,7 +422,189 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Table Type:": "VIRTUAL_VIEW",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "null",
+                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.array_struct_test",
+                            "View Expanded Text:": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "array_struct_test_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.array_struct_test_view",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "record"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<struct<type:string,provider:array<int>>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<struct<type:string,provider:array<int>>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -420,7 +619,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -438,7 +638,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test",
@@ -448,7 +648,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -518,7 +718,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -535,7 +736,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -555,7 +757,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -570,7 +773,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -588,7 +792,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test",
@@ -598,7 +802,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -717,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -734,7 +939,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -754,7 +960,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -769,7 +976,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -787,16 +995,17 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:22 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:08 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: numFiles": "1",
+                            "Table Parameters: numPartitions": "1",
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "5812",
-                            "Table Parameters: transient_lastDdlTime": "1688578704",
+                            "Table Parameters: transient_lastDdlTime": "1697721968",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -853,6 +1062,18 @@
                                 "nativeDataType": "string",
                                 "recursive": false,
                                 "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "baz",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false
                             }
                         ]
                     }
@@ -862,7 +1083,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -879,7 +1101,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +1122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +1138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -932,7 +1157,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test",
@@ -942,7 +1167,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -1039,7 +1264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1056,7 +1282,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1076,7 +1303,188 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized",
+                            "Table Type:": "MATERIALIZED_VIEW",
+                            "Table Parameters: numFiles": "0",
+                            "Table Parameters: totalSize": "0",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.struct_test",
+                            "View Expanded Text:": "null",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "struct_test_view_materialized",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.struct_test_view_materialized",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
+                                    }
+                                },
+                                "nativeDataType": "struct<type:string,provider:array<int>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"struct<type:string,provider:array<int>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1091,7 +1499,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1109,7 +1518,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test",
@@ -1119,10 +1528,10 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
-                            "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
-                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
-                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
                             "Compressed:": "No",
                             "Num Buckets:": "-1",
                             "Bucket Columns:": "[]",
@@ -1285,7 +1694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1302,7 +1712,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1322,7 +1733,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW `db1.array_struct_test_view` AS select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1342,7 +1772,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1357,7 +1788,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1372,7 +1804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1389,7 +1822,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1404,7 +1838,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1419,7 +1854,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1437,7 +1873,7 @@
                         "customProperties": {
                             "Database:": "db2",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:24 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:10 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db2.db/pokes",
@@ -1446,7 +1882,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "5812",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721971",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -1454,10 +1890,7 @@
                             "Num Buckets:": "-1",
                             "Bucket Columns:": "[]",
                             "Sort Columns:": "[]",
-                            "Storage Desc Params: serialization.format": "1",
-                            "Table:": "db2.pokes",
-                            "Constraint Name:": "pk_1173723383_1683022998392_0",
-                            "Column Names:": "foo"
+                            "Storage Desc Params: serialization.format": "1"
                         },
                         "name": "pokes",
                         "tags": []
@@ -1515,7 +1948,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1552,7 +1987,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1572,7 +2008,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1587,7 +2024,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1602,7 +2040,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1619,7 +2058,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1634,7 +2074,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json
index 08f281f398909..e93924049f626 100644
--- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json
+++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -93,7 +98,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -111,7 +117,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore",
@@ -121,7 +127,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -187,7 +193,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,7 +211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -224,7 +232,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -239,7 +248,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -257,17 +267,19 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: COLUMN_STATS_ACCURATE": "{\\\"BASIC_STATS\\\":\\\"true\\\"}",
+                            "Table Parameters: another.comment": "This table has no partitions",
+                            "Table Parameters: comment": "This table has array of structs",
                             "Table Parameters: numFiles": "1",
                             "Table Parameters: numRows": "1",
                             "Table Parameters: rawDataSize": "32",
                             "Table Parameters: totalSize": "33",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721976",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -278,6 +290,7 @@
                             "Storage Desc Params: serialization.format": "1"
                         },
                         "name": "array_struct_test",
+                        "description": "This table has array of structs",
                         "tags": []
                     }
                 },
@@ -304,6 +317,7 @@
                             {
                                 "fieldPath": "property_id",
                                 "nullable": true,
+                                "description": "id of property",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.NumberType": {}
@@ -316,6 +330,7 @@
                             {
                                 "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
                                 "nullable": true,
+                                "description": "service types and providers",
                                 "type": {
                                     "type": {
                                         "com.linkedin.pegasus2avro.schema.ArrayType": {
@@ -368,7 +383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -385,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -405,7 +422,189 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Table Type:": "VIRTUAL_VIEW",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "null",
+                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.array_struct_test",
+                            "View Expanded Text:": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "array_struct_test_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.array_struct_test_view",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "record"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<struct<type:string,provider:array<int>>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<struct<type:string,provider:array<int>>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -420,7 +619,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -438,7 +638,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test",
@@ -448,7 +648,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -518,7 +718,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -535,7 +736,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -555,7 +757,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -570,7 +773,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -588,7 +792,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test",
@@ -598,7 +802,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -717,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -734,7 +939,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -754,7 +960,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -769,7 +976,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -787,16 +995,17 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:22 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:08 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes",
                             "Table Type:": "MANAGED_TABLE",
                             "Table Parameters: numFiles": "1",
+                            "Table Parameters: numPartitions": "1",
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "5812",
-                            "Table Parameters: transient_lastDdlTime": "1688578704",
+                            "Table Parameters: transient_lastDdlTime": "1697721968",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -853,6 +1062,18 @@
                                 "nativeDataType": "string",
                                 "recursive": false,
                                 "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "baz",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false
                             }
                         ]
                     }
@@ -862,7 +1083,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -879,7 +1101,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +1122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +1138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -932,7 +1157,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test",
@@ -942,7 +1167,7 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578706",
+                            "Table Parameters: transient_lastDdlTime": "1697721972",
                             "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
                             "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
                             "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
@@ -1039,7 +1264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1056,7 +1282,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1076,7 +1303,188 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:ded36d15fcfbbb939830549697122661"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "Database:": "db1",
+                            "Owner:": "root",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
+                            "LastAccessTime:": "UNKNOWN",
+                            "Retention:": "0",
+                            "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized",
+                            "Table Type:": "MATERIALIZED_VIEW",
+                            "Table Parameters: numFiles": "0",
+                            "Table Parameters: totalSize": "0",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+                            "Compressed:": "No",
+                            "Num Buckets:": "-1",
+                            "Bucket Columns:": "[]",
+                            "Sort Columns:": "[]",
+                            "View Original Text:": "select * from db1.struct_test",
+                            "View Expanded Text:": "null",
+                            "View Rewrite Enabled:": "No"
+                        },
+                        "name": "struct_test_view_materialized",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "db1.struct_test_view_materialized",
+                        "platform": "urn:li:dataPlatform:hive",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "property_id",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "int",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
+                                    }
+                                },
+                                "nativeDataType": "struct<type:string,provider:array<int>>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"struct<type:string,provider:array<int>>\"}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
+                            },
+                            {
+                                "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.ArrayType": {
+                                            "nestedType": [
+                                                "int"
+                                            ]
+                                        }
+                                    }
+                                },
+                                "nativeDataType": "array<int>",
+                                "recursive": false,
+                                "isPartOfKey": false,
+                                "jsonProps": "{\"native_data_type\": \"array<int>\"}"
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test_view_materialized,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:ded36d15fcfbbb939830549697122661",
+                    "urn": "urn:li:container:ded36d15fcfbbb939830549697122661"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1091,7 +1499,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1109,7 +1518,7 @@
                         "customProperties": {
                             "Database:": "db1",
                             "Owner:": "root",
-                            "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023",
+                            "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023",
                             "LastAccessTime:": "UNKNOWN",
                             "Retention:": "0",
                             "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test",
@@ -1119,10 +1528,10 @@
                             "Table Parameters: numRows": "0",
                             "Table Parameters: rawDataSize": "0",
                             "Table Parameters: totalSize": "0",
-                            "Table Parameters: transient_lastDdlTime": "1688578710",
-                            "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
-                            "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat",
-                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "Table Parameters: transient_lastDdlTime": "1697721978",
+                            "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+                            "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+                            "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
                             "Compressed:": "No",
                             "Num Buckets:": "-1",
                             "Bucket Columns:": "[]",
@@ -1285,7 +1694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1302,7 +1712,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1322,7 +1733,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "hive-test"
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "CREATE VIEW `db1.array_struct_test_view` AS select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/hive/hive_setup.sql b/metadata-ingestion/tests/integration/hive/hive_setup.sql
index 8fb8498894bc0..323a78e24d10b 100644
--- a/metadata-ingestion/tests/integration/hive/hive_setup.sql
+++ b/metadata-ingestion/tests/integration/hive/hive_setup.sql
@@ -1,10 +1,10 @@
 CREATE DATABASE IF NOT EXISTS db1;
 CREATE DATABASE IF NOT EXISTS db2;
 -- Setup a "pokes" example table.
-CREATE TABLE IF NOT EXISTS db1.pokes (foo INT, bar STRING);
-LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE db1.pokes;
+CREATE TABLE IF NOT EXISTS db1.pokes (foo INT, bar STRING) PARTITIONED BY (baz STRING);
+LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE db1.pokes PARTITION (baz='dummy');
 
-CREATE TABLE IF NOT EXISTS db2.pokes (foo INT, bar STRING, CONSTRAINT pk_1173723383_1683022998392_0 primary key(foo) DISABLE NOVALIDATE NORELY);
+CREATE TABLE IF NOT EXISTS db2.pokes (foo INT, bar STRING);
 LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE db2.pokes;
 
 -- Setup a table with a special character.
@@ -23,12 +23,12 @@ CREATE TABLE IF NOT EXISTS db1.struct_test
 
 CREATE TABLE IF NOT EXISTS db1.array_struct_test
 (
- property_id INT,
+ property_id INT COMMENT 'id of property',
  service array<STRUCT<
                 type: STRING
                ,provider: ARRAY<INT>
-               >>
-);
+               >> COMMENT 'service types and providers'
+)  TBLPROPERTIES ('comment' = 'This table has array of structs', 'another.comment' = 'This table has no partitions');;
 
 WITH
 test_data as (
@@ -39,6 +39,9 @@ test_data as (
 INSERT INTO TABLE db1.array_struct_test
 select * from test_data;
 
+CREATE MATERIALIZED VIEW db1.struct_test_view_materialized as select * from db1.struct_test;
+CREATE VIEW db1.array_struct_test_view as select * from db1.array_struct_test;
+
 CREATE TABLE IF NOT EXISTS db1.nested_struct_test
 (
  property_id INT,
@@ -50,9 +53,6 @@ CREATE TABLE IF NOT EXISTS db1.nested_struct_test
 
 CREATE TABLE db1.union_test(
     foo UNIONTYPE<int, double, array<string>, struct<a:int,b:string>, struct<c:int,d:double>>
-);
+) STORED AS ORC ;
 
-CREATE TABLE db1.map_test(
-    KeyValue String, 
-    RecordId map<int,string>
-); 
\ No newline at end of file
+CREATE TABLE db1.map_test(KeyValue String, RecordId map<int,string>); 
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
index 4aaefb48d33e1..38b03ce238d1c 100644
--- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
+++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -93,7 +98,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -213,7 +219,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -230,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -250,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -265,7 +274,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -361,7 +371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -378,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -398,7 +410,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +567,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -969,7 +983,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -989,7 +1004,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1004,7 +1020,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1019,7 +1036,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1036,7 +1054,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1053,7 +1072,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1068,7 +1088,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1083,7 +1104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1215,7 +1237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1232,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1249,7 +1273,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1269,7 +1294,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1284,7 +1310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1445,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1435,7 +1463,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1452,7 +1481,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1472,7 +1502,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1487,7 +1518,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1586,7 +1618,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1603,7 +1636,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1620,7 +1654,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1637,7 +1672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1657,7 +1693,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1677,7 +1714,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1692,7 +1730,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1707,7 +1746,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1724,7 +1764,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1739,7 +1780,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1754,7 +1796,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1874,7 +1917,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1891,7 +1935,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1911,7 +1956,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1926,7 +1972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2022,7 +2069,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2039,7 +2087,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2059,7 +2108,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2182,7 +2232,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2233,7 +2284,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2253,7 +2305,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2268,7 +2321,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2283,7 +2337,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2300,7 +2355,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2315,7 +2371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2330,7 +2387,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2390,7 +2448,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2407,7 +2466,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2427,7 +2487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2442,7 +2503,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2502,7 +2564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2519,7 +2582,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2539,7 +2603,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2568,7 +2633,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2597,7 +2663,79 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "mysql-test"
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),path)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "mysql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json
index 535ce964c6058..b9b2a3b2141a8 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json
+++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -99,7 +104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -181,7 +191,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -201,7 +212,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -216,7 +228,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -231,7 +244,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -248,7 +262,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -263,7 +278,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -284,7 +300,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -299,7 +316,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -314,7 +332,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -331,7 +350,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -346,7 +366,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -366,7 +387,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -381,7 +403,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -537,7 +560,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +578,186 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+                            "is_view": "True"
+                        },
+                        "name": "metadata_aspect_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "postgrestest.public.metadata_aspect_view",
+                        "platform": "urn:li:dataPlatform:postgres",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "urn",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=500)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "aspect",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=200)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -634,31 +837,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
-                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
-                },
-                {
-                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
-                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -675,29 +855,39 @@
                         "actor": "urn:li:corpuser:unknown"
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
-                    "type": "TRANSFORMED"
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml b/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml
index b390d9246677e..2bfa39a65363b 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml
+++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml
@@ -25,7 +25,7 @@ source:
       include_field_distinct_value_frequencies: false
       include_field_histogram: false
       catch_exceptions: true
-    include_views: false
+    include_views: true
 sink:
   type: file
   config:
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json
index bf36a39a8c103..f6fa0a0ed032e 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json
+++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -99,7 +104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -181,7 +191,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -196,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -352,7 +364,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +382,186 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "view_definition": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+                            "is_view": "True"
+                        },
+                        "name": "metadata_aspect_view",
+                        "tags": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "postgrestest.public.metadata_aspect_view",
+                        "platform": "urn:li:dataPlatform:postgres",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
+                                "tableSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "urn",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=500)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "aspect",
+                                "nullable": true,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR(length=200)",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
+                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                },
+                {
+                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
+                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1646575200000,
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -391,31 +583,57 @@
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
+            "upstreams": [
                 {
-                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
-                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
+                    ],
+                    "confidenceScore": 1.0
                 },
                 {
-                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
-                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1646575200000,
-        "runId": "postgres-test"
+        "runId": "postgres-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml b/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml
index a489877d52a23..4a2cc543f2d01 100644
--- a/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml
+++ b/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml
@@ -13,7 +13,7 @@ source:
       profile_table_row_count_estimate_only: true
       turn_off_expensive_profiling_metrics: true
       catch_exceptions: true
-    include_views: false
+    include_views: true
 sink:
   type: file
   config:
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
index cd53b8f7db4f6..4b0dd2b1045a3 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py
@@ -287,8 +287,9 @@ def test_snowflake_unexpected_snowflake_view_lineage_error_causes_pipeline_warni
             SnowflakeV2Config,
             cast(PipelineConfig, snowflake_pipeline_config1).source.config,
         )
+        config.include_table_lineage = True
         config.include_view_lineage = True
-        config.incremental_lineage = False
+
         pipeline = Pipeline(snowflake_pipeline_config1)
         pipeline.run()
         pipeline.raise_from_status()  # pipeline should not fail
diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json
index 19961e48b4a33..c43223c68a6b6 100644
--- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json
+++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -99,7 +104,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +120,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -146,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -181,7 +191,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -196,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -219,7 +231,7 @@
                             "numrows": "1",
                             "rawdatasize": "32",
                             "totalsize": "33",
-                            "transient_lastddltime": "1688422059"
+                            "transient_lastddltime": "1698223433"
                         },
                         "name": "array_struct_test",
                         "description": "This table has array of structs",
@@ -315,7 +327,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -332,7 +345,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -356,7 +370,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -371,7 +386,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -392,7 +408,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422063"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "map_test",
                         "tags": []
@@ -454,7 +470,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -471,7 +488,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -495,7 +513,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -510,7 +529,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -531,7 +551,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422062"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "nested_struct_test",
                         "tags": []
@@ -642,7 +662,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -659,7 +680,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -683,7 +705,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -698,7 +721,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -714,7 +738,7 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "transient_lastddltime": "1688421792"
+                            "transient_lastddltime": "1698223429"
                         },
                         "name": "pokes",
                         "tags": []
@@ -784,7 +808,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -801,7 +826,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -825,7 +851,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -840,7 +867,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -861,7 +889,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688421808"
+                            "transient_lastddltime": "1698223431"
                         },
                         "name": "struct_test",
                         "tags": []
@@ -950,7 +978,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -967,7 +996,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -991,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1006,7 +1037,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1024,7 +1056,7 @@
                         "customProperties": {
                             "numfiles": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422062"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "struct_test_view_materialized",
                         "tags": []
@@ -1113,7 +1145,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1130,7 +1163,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1154,7 +1188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1169,7 +1204,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1190,7 +1226,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688421807"
+                            "transient_lastddltime": "1698223431"
                         },
                         "name": "_test_table_underscore",
                         "tags": []
@@ -1248,7 +1284,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1265,7 +1302,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1289,7 +1327,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1304,7 +1343,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1325,7 +1365,7 @@
                             "numrows": "0",
                             "rawdatasize": "0",
                             "totalsize": "0",
-                            "transient_lastddltime": "1688422062"
+                            "transient_lastddltime": "1698223435"
                         },
                         "name": "union_test",
                         "tags": []
@@ -1467,7 +1507,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1484,7 +1525,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1550,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1523,7 +1566,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1539,7 +1583,7 @@
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                         "customProperties": {
-                            "transient_lastddltime": "1688422062",
+                            "transient_lastddltime": "1698223435",
                             "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"",
                             "is_view": "True"
                         },
@@ -1634,7 +1678,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1651,7 +1696,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1668,7 +1714,57 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD),property_id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),property_id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD),service)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),service)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1632398400000,
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1692,7 +1788,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1632398400000,
-        "runId": "trino-hive-test"
+        "runId": "trino-hive-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
index 54a22d860285c..e8485106c6a81 100644
--- a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
+++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py
@@ -104,6 +104,27 @@ def test_incremental_table_lineage(tmp_path, pytestconfig):
     )
 
 
+def test_incremental_table_lineage_empty_upstreams(tmp_path, pytestconfig):
+
+    urn = make_dataset_urn(platform, "dataset1")
+    aspect = make_lineage_aspect(
+        "dataset1",
+        upstreams=[],
+    )
+
+    processed_wus = auto_incremental_lineage(
+        graph=None,
+        incremental_lineage=True,
+        stream=[
+            MetadataChangeProposalWrapper(
+                entityUrn=urn, aspect=aspect, systemMetadata=system_metadata
+            ).as_workunit()
+        ],
+    )
+
+    assert [wu.metadata for wu in processed_wus] == []
+
+
 @pytest.mark.parametrize(
     "gms_aspect,current_aspect,output_aspect",
     [

From a96a512166564cf9c40af4b83e7138dcb48c914d Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Thu, 26 Oct 2023 18:46:10 +0200
Subject: [PATCH 106/259] fix(ingest/bigquery): Fixing lineage filter query
 (#9114)

---
 .../ingestion/source/bigquery_v2/bigquery_config.py    |  1 +
 .../datahub/ingestion/source/bigquery_v2/lineage.py    |  8 ++++++--
 .../src/datahub/ingestion/source/bigquery_v2/usage.py  | 10 ++++++----
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index 6203192769750..f762d451849ab 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -309,6 +309,7 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
                 "dataset_pattern is not set but schema_pattern is set, using schema_pattern as dataset_pattern. schema_pattern will be deprecated, please use dataset_pattern instead."
             )
             values["dataset_pattern"] = schema_pattern
+            dataset_pattern = schema_pattern
         elif (
             dataset_pattern != AllowDenyPattern.allow_all()
             and schema_pattern != AllowDenyPattern.allow_all()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
index aa462435b8105..e9acf5ea86044 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
@@ -20,6 +20,7 @@
 from google.cloud.datacatalog import lineage_v1
 from google.cloud.logging_v2.client import Client as GCPLoggingClient
 
+from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
@@ -683,8 +684,11 @@ def _create_lineage_map(
                 self.report.num_skipped_lineage_entries_missing_data[e.project_id] += 1
                 continue
 
-            if not self.config.dataset_pattern.allowed(
-                destination_table.table_identifier.dataset
+            if not is_schema_allowed(
+                self.config.dataset_pattern,
+                destination_table.table_identifier.dataset,
+                destination_table.table_identifier.project_id,
+                self.config.match_fully_qualified_names,
             ) or not self.config.table_pattern.allowed(
                 destination_table.table_identifier.get_table_name()
             ):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
index 7fc38991e5928..65b559550ffc5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
@@ -21,6 +21,7 @@
 
 import humanfriendly
 
+from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.configuration.time_window_config import (
     BaseTimeWindowConfig,
     get_time_bucket,
@@ -335,10 +336,11 @@ def get_time_window(self) -> Tuple[datetime, datetime]:
     def _is_table_allowed(self, table_ref: Optional[BigQueryTableRef]) -> bool:
         return (
             table_ref is not None
-            and self.config.dataset_pattern.allowed(
-                f"{table_ref.table_identifier.project_id}.{table_ref.table_identifier.dataset}"
-                if self.config.match_fully_qualified_names
-                else table_ref.table_identifier.dataset
+            and is_schema_allowed(
+                self.config.dataset_pattern,
+                table_ref.table_identifier.dataset,
+                table_ref.table_identifier.project_id,
+                self.config.match_fully_qualified_names,
             )
             and self.config.table_pattern.allowed(str(table_ref.table_identifier))
         )

From 852267972c8efc1ceb5a0cbd71594d7ea2529d49 Mon Sep 17 00:00:00 2001
From: "nicholas.fwang" <nicholas.fwang@kakaoent.com>
Date: Fri, 27 Oct 2023 01:57:43 +0900
Subject: [PATCH 107/259] refactor(ingestion/mongodb): Add platform_instance to
 mongodb (#8663)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../src/datahub/ingestion/source/mongodb.py      | 16 +++++++++++++---
 .../integration/mongodb/mongodb_mces_golden.json | 16 ++++++++--------
 .../tests/integration/mongodb/test_mongodb.py    |  1 +
 3 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
index f02b6845e40b5..890c5c64bd5e6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
@@ -11,7 +11,11 @@
 from pymongo.mongo_client import MongoClient
 
 from datahub.configuration.common import AllowDenyPattern
-from datahub.configuration.source_common import EnvConfigMixin
+from datahub.configuration.source_common import (
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+)
+from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -55,7 +59,7 @@
 DENY_DATABASE_LIST = set(["admin", "config", "local"])
 
 
-class MongoDBConfig(EnvConfigMixin):
+class MongoDBConfig(PlatformInstanceConfigMixin, EnvConfigMixin):
     # See the MongoDB authentication docs for details and examples.
     # https://pymongo.readthedocs.io/en/stable/examples/authentication.html
     connect_uri: str = Field(
@@ -199,6 +203,7 @@ def construct_schema_pymongo(
 @platform_name("MongoDB")
 @config_class(MongoDBConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @dataclass
 class MongoDBSource(Source):
@@ -320,7 +325,12 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     self.report.report_dropped(dataset_name)
                     continue
 
-                dataset_urn = f"urn:li:dataset:(urn:li:dataPlatform:{platform},{dataset_name},{self.config.env})"
+                dataset_urn = make_dataset_urn_with_platform_instance(
+                    platform=platform,
+                    name=dataset_name,
+                    env=self.config.env,
+                    platform_instance=self.config.platform_instance,
+                )
 
                 dataset_snapshot = DatasetSnapshot(
                     urn=dataset_urn,
diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
index 1f662cfe514e2..e16101b137ac9 100644
--- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
+++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
@@ -2,7 +2,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.emptyCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -41,7 +41,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.firstCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -345,7 +345,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.largeCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -3988,7 +3988,7 @@
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.secondCollection,PROD)",
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
             "aspects": [
                 {
                     "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@@ -4135,7 +4135,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.emptyCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4150,7 +4150,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.firstCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4165,7 +4165,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.largeCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4180,7 +4180,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.secondCollection,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
diff --git a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py
index 5228c21223e24..56fb471d4c9f1 100644
--- a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py
+++ b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py
@@ -25,6 +25,7 @@ def test_mongodb_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time
                         "username": "mongoadmin",
                         "password": "examplepass",
                         "maxDocumentSize": 25000,
+                        "platform_instance": "instance",
                     },
                 },
                 "sink": {

From ce6f833be444497972f17fd8bfe170f00af4bca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mert=20Tun=C3=A7?= <mert.tunc@udemy.com>
Date: Fri, 27 Oct 2023 01:06:37 +0300
Subject: [PATCH 108/259] fix(kafka-setup): Don't set truststore pass for PEM
 files (#8656)

Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
---
 docker/kafka-setup/kafka-setup.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh
index 629e9bc9484ee..b5024e49e59f1 100755
--- a/docker/kafka-setup/kafka-setup.sh
+++ b/docker/kafka-setup/kafka-setup.sh
@@ -36,7 +36,9 @@ if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SSL" ]]; then
     fi
     if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION ]]; then
         echo "ssl.truststore.location=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH
-        echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH
+        if [[ $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE != "PEM" ]]; then
+          echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH
+        fi
         if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE ]]; then
             echo "ssl.truststore.type=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH
         fi

From 12f6fe0f906dd21fbc3985cfd13ceac4fc9ac8f0 Mon Sep 17 00:00:00 2001
From: Tony Ouyang <tony@terratrue.com>
Date: Thu, 26 Oct 2023 15:07:36 -0700
Subject: [PATCH 109/259] fix(ingest): Fix roll back failure when
 REST_API_AUTHORIZATION_ENABLED is set to true (#9092)

---
 metadata-service/war/src/main/resources/boot/policies.json | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json
index 410596cc30cbe..18cb48bfcf1f0 100644
--- a/metadata-service/war/src/main/resources/boot/policies.json
+++ b/metadata-service/war/src/main/resources/boot/policies.json
@@ -64,7 +64,8 @@
         "GET_TIMELINE_PRIVILEGE",
         "PRODUCE_PLATFORM_EVENT_PRIVILEGE",
         "MANAGE_DATA_PRODUCTS",
-        "MANAGE_GLOBAL_OWNERSHIP_TYPES"
+        "MANAGE_GLOBAL_OWNERSHIP_TYPES",
+        "DELETE_ENTITY"
       ],
       "displayName":"Root User - Edit and View All Resources",
       "description":"Grants full edit and view privileges for all resources to root 'datahub' root user.",
@@ -263,7 +264,8 @@
         "GET_ENTITY_PRIVILEGE",
         "GET_TIMELINE_PRIVILEGE",
         "PRODUCE_PLATFORM_EVENT_PRIVILEGE",
-        "MANAGE_DATA_PRODUCTS"
+        "MANAGE_DATA_PRODUCTS",
+        "DELETE_ENTITY"
       ],
       "displayName":"Admins - Metadata Policy",
       "description":"Admins have all metadata privileges.",

From 1ac831f07aa2bdab555acf50431f6466bb291f61 Mon Sep 17 00:00:00 2001
From: Raj Tekal <varadaraj_tekal@optum.com>
Date: Thu, 26 Oct 2023 19:33:09 -0400
Subject: [PATCH 110/259] (fix): Avoid
 java.util.ConcurrentModificationException (#9090)

Co-authored-by: Pedro Silva <pedro@acryl.io>
---
 .../authorization/DataHubAuthorizer.java      | 105 +++++++++++-------
 1 file changed, 64 insertions(+), 41 deletions(-)

diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
index 4553139e3ca54..e30fb93109915 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
@@ -19,6 +19,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -53,6 +55,7 @@ public enum AuthorizationMode {
   // Maps privilege name to the associated set of policies for fast access.
   // Not concurrent data structure because writes are always against the entire thing.
   private final Map<String, List<DataHubPolicyInfo>> _policyCache = new HashMap<>(); // Shared Policy Cache.
+  private final ReadWriteLock _lockPolicyCache = new ReentrantReadWriteLock();
 
   private final ScheduledExecutorService _refreshExecutorService = Executors.newScheduledThreadPool(1);
   private final PolicyRefreshRunnable _policyRefreshRunnable;
@@ -71,7 +74,7 @@ public DataHubAuthorizer(
     _systemAuthentication = Objects.requireNonNull(systemAuthentication);
     _mode = Objects.requireNonNull(mode);
     _policyEngine = new PolicyEngine(systemAuthentication, Objects.requireNonNull(entityClient));
-    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache);
+    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache, _lockPolicyCache);
     _refreshExecutorService.scheduleAtFixedRate(_policyRefreshRunnable, delayIntervalSeconds, refreshIntervalSeconds, TimeUnit.SECONDS);
   }
 
@@ -90,31 +93,41 @@ public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request
 
     Optional<ResolvedEntitySpec> resolvedResourceSpec = request.getResourceSpec().map(_entitySpecResolver::resolve);
 
-    // 1. Fetch the policies relevant to the requested privilege.
-    final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>());
-
-    // 2. Evaluate each policy.
-    for (DataHubPolicyInfo policy : policiesToEvaluate) {
-      if (isRequestGranted(policy, request, resolvedResourceSpec)) {
-        // Short circuit if policy has granted privileges to this actor.
-        return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW,
-            String.format("Granted by policy with type: %s", policy.getType()));
+    _lockPolicyCache.readLock().lock();
+    try {
+      // 1. Fetch the policies relevant to the requested privilege.
+      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>());
+
+      // 2. Evaluate each policy.
+      for (DataHubPolicyInfo policy : policiesToEvaluate) {
+        if (isRequestGranted(policy, request, resolvedResourceSpec)) {
+          // Short circuit if policy has granted privileges to this actor.
+          return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW,
+              String.format("Granted by policy with type: %s", policy.getType()));
+        }
       }
+      return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
+    } finally {
+      _lockPolicyCache.readLock().unlock();
     }
-    return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
   }
 
   public List<String> getGrantedPrivileges(final String actor, final Optional<EntitySpec> resourceSpec) {
 
-    // 1. Fetch all policies
-    final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>());
+    _lockPolicyCache.readLock().lock();
+    try {
+      // 1. Fetch all policies
+      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>());
 
-    Urn actorUrn = UrnUtils.getUrn(actor);
-    final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
+      Urn actorUrn = UrnUtils.getUrn(actor);
+      final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
 
-    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
+      Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-    return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
+       return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
+    } finally {
+      _lockPolicyCache.readLock().unlock();
+    }
   }
 
   /**
@@ -124,36 +137,42 @@ public List<String> getGrantedPrivileges(final String actor, final Optional<Enti
   public AuthorizedActors authorizedActors(
       final String privilege,
       final Optional<EntitySpec> resourceSpec) {
-    // Step 1: Find policies granting the privilege.
-    final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>());
-
-    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
     final List<Urn> authorizedUsers = new ArrayList<>();
     final List<Urn> authorizedGroups = new ArrayList<>();
     boolean allUsers = false;
     boolean allGroups = false;
 
-    // Step 2: For each policy, determine whether the resource is a match.
-    for (DataHubPolicyInfo policy : policiesToEvaluate) {
-      if (!PoliciesConfig.ACTIVE_POLICY_STATE.equals(policy.getState())) {
-        // Policy is not active, skip.
-        continue;
-      }
+    _lockPolicyCache.readLock().lock();
+    try {
+      // Step 1: Find policies granting the privilege.
+      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>());
 
-      final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec);
+      Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-      // Step 3: For each matching policy, add actors that are authorized.
-      authorizedUsers.addAll(matchingActors.getUsers());
-      authorizedGroups.addAll(matchingActors.getGroups());
-      if (matchingActors.allUsers()) {
-        allUsers = true;
-      }
-      if (matchingActors.allGroups()) {
-        allGroups = true;
+
+      // Step 2: For each policy, determine whether the resource is a match.
+      for (DataHubPolicyInfo policy : policiesToEvaluate) {
+        if (!PoliciesConfig.ACTIVE_POLICY_STATE.equals(policy.getState())) {
+          // Policy is not active, skip.
+          continue;
+        }
+
+        final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec);
+
+        // Step 3: For each matching policy, add actors that are authorized.
+        authorizedUsers.addAll(matchingActors.getUsers());
+        authorizedGroups.addAll(matchingActors.getGroups());
+        if (matchingActors.allUsers()) {
+          allUsers = true;
+        }
+        if (matchingActors.allGroups()) {
+          allGroups = true;
+        }
       }
+    } finally {
+      _lockPolicyCache.readLock().unlock();
     }
-
     // Step 4: Return all authorized users and groups.
     return new AuthorizedActors(privilege, authorizedUsers, authorizedGroups, allUsers, allGroups);
   }
@@ -228,6 +247,7 @@ static class PolicyRefreshRunnable implements Runnable {
     private final Authentication _systemAuthentication;
     private final PolicyFetcher _policyFetcher;
     private final Map<String, List<DataHubPolicyInfo>> _policyCache;
+    private final ReadWriteLock _lockPolicyCache;
 
     @Override
     public void run() {
@@ -253,10 +273,13 @@ public void run() {
                 "Failed to retrieve policy urns! Skipping updating policy cache until next refresh. start: {}, count: {}", start, count, e);
             return;
           }
-          synchronized (_policyCache) {
-            _policyCache.clear();
-            _policyCache.putAll(newCache);
-          }
+        }
+        _lockPolicyCache.writeLock().lock();
+        try {
+          _policyCache.clear();
+          _policyCache.putAll(newCache);
+        } finally {
+          _lockPolicyCache.writeLock().unlock();
         }
         log.debug(String.format("Successfully fetched %s policies.", total));
       } catch (Exception e) {

From cf617d77f383a19bd6a9bce00bb2cfbd6a226e55 Mon Sep 17 00:00:00 2001
From: David Sanchez <david.sanchez@cabify.com>
Date: Fri, 27 Oct 2023 07:01:30 +0200
Subject: [PATCH 111/259] Fix(ingest/bigquery): fix extracting comments from
 complex types (#8950)

Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
---
 .../datahub/ingestion/source/bigquery_v2/bigquery.py   | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
index 692d8c4f81bb6..6959a48313010 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -1050,8 +1050,14 @@ def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]:
                     for idx, field in enumerate(schema_fields):
                         # Remove all the [version=2.0].[type=struct]. tags to get the field path
                         if (
-                            re.sub(r"\[.*?\]\.", "", field.fieldPath, 0, re.MULTILINE)
-                            == col.field_path
+                            re.sub(
+                                r"\[.*?\]\.",
+                                "",
+                                field.fieldPath.lower(),
+                                0,
+                                re.MULTILINE,
+                            )
+                            == col.field_path.lower()
                         ):
                             field.description = col.comment
                             schema_fields[idx] = field

From cc7511501b051b9a9f66dbcc4dc4ab16ce2668e5 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Fri, 27 Oct 2023 19:42:53 +0900
Subject: [PATCH 112/259] docs: add versions 0.12.0 (#9125)

---
 docs-website/versions.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs-website/versions.json b/docs-website/versions.json
index a5493c26a4c65..a66607b67ddd5 100644
--- a/docs-website/versions.json
+++ b/docs-website/versions.json
@@ -1,4 +1,5 @@
 [
+  "0.12.0",
   "0.11.0",
   "0.10.5"
 ]

From 07a5e4c81b9e7b46faf9dbc830c9bac9648e5161 Mon Sep 17 00:00:00 2001
From: Chris Collins <chriscollins3456@gmail.com>
Date: Fri, 27 Oct 2023 07:52:24 -0400
Subject: [PATCH 113/259] fix(ui) Fix filtering logic for everwhere generating
 OR filters (#9116)

---
 .../src/app/search/useGetSearchQueryInputs.ts | 12 ++----
 .../utils/__tests__/generateOrFilters.test.ts | 38 ++++++++++++++-----
 .../src/app/search/utils/generateOrFilters.ts | 16 +++++---
 3 files changed, 43 insertions(+), 23 deletions(-)

diff --git a/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts b/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
index 05419e5abed35..9a3af8fb8d56c 100644
--- a/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
+++ b/datahub-web-react/src/app/search/useGetSearchQueryInputs.ts
@@ -3,7 +3,7 @@ import { useLocation, useParams } from 'react-router';
 import { useMemo } from 'react';
 import { FacetFilterInput, EntityType } from '../../types.generated';
 import { useEntityRegistry } from '../useEntityRegistry';
-import { ENTITY_FILTER_NAME, FILTER_DELIMITER, UnionType } from './utils/constants';
+import { ENTITY_FILTER_NAME, UnionType } from './utils/constants';
 import { useUserContext } from '../context/useUserContext';
 import useFilters from './utils/useFilters';
 import { generateOrFilters } from './utils/generateOrFilters';
@@ -27,12 +27,6 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     const sortInput = useSortInput();
 
     const filters: Array<FacetFilterInput> = useFilters(params);
-    const nonNestedFilters = filters.filter(
-        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
-    const nestedFilters = filters.filter(
-        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
-    );
     const entityFilters: Array<EntityType> = useMemo(
         () =>
             filters
@@ -43,8 +37,8 @@ export default function useGetSearchQueryInputs(excludedFilterFields?: Array<str
     );
 
     const orFilters = useMemo(
-        () => generateOrFilters(unionType, nonNestedFilters, nestedFilters),
-        [nonNestedFilters, nestedFilters, unionType],
+        () => generateOrFilters(unionType, filters, excludedFilterFields),
+        [filters, excludedFilterFields, unionType],
     );
 
     return { entityFilters, query, unionType, filters, orFilters, viewUrn, page, activeType, sortInput };
diff --git a/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts b/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
index 505c50efb289f..fd5a5691b454e 100644
--- a/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
+++ b/datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts
@@ -1,7 +1,7 @@
 import {
     DOMAINS_FILTER_NAME,
     ENTITY_SUB_TYPE_FILTER_NAME,
-    ENTITY_TYPE_FILTER_NAME,
+    ENTITY_FILTER_NAME,
     TAGS_FILTER_NAME,
     UnionType,
 } from '../constants';
@@ -10,7 +10,7 @@ import { generateOrFilters } from '../generateOrFilters';
 describe('generateOrFilters', () => {
     it('should generate orFilters with UnionType.AND', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.AND, filters);
@@ -24,7 +24,7 @@ describe('generateOrFilters', () => {
 
     it('should generate orFilters with UnionType.OR', () => {
         const filters = [
-            { field: ENTITY_TYPE_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
         ];
         const orFilters = generateOrFilters(UnionType.OR, filters);
@@ -43,17 +43,23 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.AND, filters, nestedFilters);
+        // const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
+        const orFilters = generateOrFilters(UnionType.AND, filters);
 
         expect(orFilters).toMatchObject([
             {
-                and: [...filters, { field: '_entityType', values: ['CONTAINER'] }],
+                and: [
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+                    { field: '_entityType', values: ['CONTAINER'] },
+                ],
             },
             {
                 and: [
-                    ...filters,
+                    { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+                    { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
                     { field: '_entityType', values: ['DATASET'] },
                     { field: 'typeNames', values: ['table'] },
                 ],
@@ -65,9 +71,9 @@ describe('generateOrFilters', () => {
         const filters = [
             { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
             { field: DOMAINS_FILTER_NAME, values: ['urn:li:domains:domain1'] },
+            { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] },
         ];
-        const nestedFilters = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['CONTAINER', 'DATASET␞table'] }];
-        const orFilters = generateOrFilters(UnionType.OR, filters, nestedFilters);
+        const orFilters = generateOrFilters(UnionType.OR, filters);
 
         expect(orFilters).toMatchObject([
             {
@@ -87,4 +93,18 @@ describe('generateOrFilters', () => {
             },
         ]);
     });
+
+    it('should generate orFilters and exclude filters with a provided exclude field', () => {
+        const filters = [
+            { field: ENTITY_FILTER_NAME, values: ['DATASET', 'CONTAINER'] },
+            { field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] },
+        ];
+        const orFilters = generateOrFilters(UnionType.AND, filters, [ENTITY_FILTER_NAME]);
+
+        expect(orFilters).toMatchObject([
+            {
+                and: [{ field: TAGS_FILTER_NAME, values: ['urn:li:tag:tag1'] }],
+            },
+        ]);
+    });
 });
diff --git a/datahub-web-react/src/app/search/utils/generateOrFilters.ts b/datahub-web-react/src/app/search/utils/generateOrFilters.ts
index b665a2e0f0495..fa2939b3436f5 100644
--- a/datahub-web-react/src/app/search/utils/generateOrFilters.ts
+++ b/datahub-web-react/src/app/search/utils/generateOrFilters.ts
@@ -26,20 +26,26 @@ function generateInputWithNestedFilters(filters: FacetFilterInput[], nestedFilte
 export function generateOrFilters(
     unionType: UnionType,
     filters: FacetFilterInput[],
-    nestedFilters: FacetFilterInput[] = [],
+    excludedFilterFields: string[] = [],
 ): AndFilterInput[] {
-    if ((filters?.length || 0) === 0 && nestedFilters.length === 0) {
+    if ((filters?.length || 0) === 0) {
         return [];
     }
+    const nonNestedFilters = filters.filter(
+        (f) => !f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
+    const nestedFilters = filters.filter(
+        (f) => f.field.includes(FILTER_DELIMITER) && !excludedFilterFields?.includes(f.field),
+    );
 
     if (unionType === UnionType.OR) {
         const orFiltersWithNestedFilters = generateInputWithNestedFilters([], nestedFilters);
-        const orFilters = filters.map((filter) => ({
+        const orFilters = nonNestedFilters.map((filter) => ({
             and: [filter],
         }));
         return [...orFilters, ...orFiltersWithNestedFilters];
     }
-    const andFiltersWithNestedFilters = generateInputWithNestedFilters(filters, nestedFilters);
+    const andFiltersWithNestedFilters = generateInputWithNestedFilters(nonNestedFilters, nestedFilters);
 
     if (andFiltersWithNestedFilters.length) {
         return andFiltersWithNestedFilters;
@@ -47,7 +53,7 @@ export function generateOrFilters(
 
     return [
         {
-            and: filters,
+            and: nonNestedFilters,
         },
     ];
 }

From 379ffc8d9457bb86029383e857aaa41eae40f329 Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Fri, 27 Oct 2023 17:17:27 +0100
Subject: [PATCH 114/259] build(release): Update files for 0.12.0 release
 (#9130)

---
 .../src/app/ingest/source/builder/NameSourceStep.tsx            | 2 +-
 gradle/versioning/versioning.gradle                             | 2 +-
 .../tests/cypress/cypress/e2e/mutations/managed_ingestion.js    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
index f4c048bcaf0d2..3092364bb8bdd 100644
--- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
+++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -190,7 +190,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                             <Input
                                 data-testid="cli-version-input"
                                 className="cli-version-input"
-                                placeholder="(e.g. 0.10.5)"
+                                placeholder="(e.g. 0.12.0)"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
                             />
diff --git a/gradle/versioning/versioning.gradle b/gradle/versioning/versioning.gradle
index 1fac894d165a8..39a8a3faf8011 100644
--- a/gradle/versioning/versioning.gradle
+++ b/gradle/versioning/versioning.gradle
@@ -21,7 +21,7 @@ Produces the following variables and supports token replacement
 import org.apache.tools.ant.filters.ReplaceTokens
 
 def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def cliMajorVersion = "0.10.5" // base default cli major version
+def cliMajorVersion = "0.12.0" // base default cli major version
 def snapshotVersion = false
 if (project.hasProperty("releaseVersion")) {
   version = releaseVersion
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
index 24a24cc21138d..3d052695e818f 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
@@ -7,7 +7,7 @@ describe("run managed ingestion", () => {
     it("create run managed ingestion source", () => {
         let number = Math.floor(Math.random() * 100000);
         let testName = `cypress test source ${number}`
-        let cli_version = "0.10.5.4";
+        let cli_version = "0.12.0";
         cy.login();
         cy.goToIngestionPage();
         cy.clickOptionWithText("Create new source");

From 5166d90433123891bc8f9555d4c6660a2b5c1451 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Fri, 27 Oct 2023 22:49:51 +0530
Subject: [PATCH 115/259] fix(ingest/sql-server): update queries to use escaped
 procedure name (#9127)

---
 .../ingestion/source/sql/mssql/source.py      |   4 +-
 .../golden_mces_mssql_no_db_to_file.json      | 774 +++++++++++-------
 .../golden_mces_mssql_no_db_with_filter.json  | 471 +++++++----
 .../golden_mces_mssql_to_file.json            | 471 +++++++----
 ...golden_mces_mssql_with_lower_case_urn.json | 365 ++++++---
 .../integration/sql_server/setup/setup.sql    |   2 +-
 6 files changed, 1324 insertions(+), 763 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
index 685d4fb3074c9..710825c8ba55d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
@@ -530,7 +530,7 @@ def _get_procedure_inputs(
     def _get_procedure_code(
         conn: Connection, procedure: StoredProcedure
     ) -> Tuple[Optional[str], Optional[str]]:
-        query = f"EXEC [{procedure.db}].dbo.sp_helptext '{procedure.full_name}'"
+        query = f"EXEC [{procedure.db}].dbo.sp_helptext '{procedure.escape_full_name}'"
         try:
             code_data = conn.execute(query)
         except ProgrammingError:
@@ -567,7 +567,7 @@ def _get_procedure_properties(
                 create_date as date_created,
                 modify_date as date_modified
             FROM sys.procedures
-            WHERE object_id = object_id('{procedure.full_name}')
+            WHERE object_id = object_id('{procedure.escape_full_name}')
             """
         )
         properties = {}
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
index a495d04c4e398..2fe7a76fd01ae 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -80,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -91,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "1df94c0f-15fd-4b68-8ca3-6053a0332362",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-03-10 16:27:54.970000",
-                "date_modified": "2023-03-10 16:27:55.097000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -110,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -127,22 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                },
+                {
+                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
+                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1830,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1843,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-03-10 16:27:54.907000",
-                "date_modified": "2023-03-10 16:27:54.907000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1858,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1875,31 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                },
-                {
-                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
-                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,7 +2297,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2208,7 +2318,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2228,7 +2339,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2243,7 +2355,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2258,7 +2371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2275,7 +2389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2290,7 +2405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2311,7 +2427,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2326,7 +2443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2341,7 +2459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2358,7 +2477,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2373,7 +2493,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2393,7 +2514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2414,7 +2536,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2429,7 +2552,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2444,7 +2568,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2461,7 +2586,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2476,7 +2602,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2496,7 +2623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2517,7 +2645,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2532,7 +2661,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2547,7 +2677,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2564,7 +2695,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2579,7 +2711,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2599,7 +2732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2620,7 +2754,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2635,7 +2770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2650,7 +2786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2667,7 +2804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2682,7 +2820,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2702,7 +2841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2723,7 +2863,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2738,7 +2879,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2753,7 +2895,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2770,7 +2913,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2785,7 +2929,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2805,7 +2950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2826,7 +2972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2841,7 +2988,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2856,7 +3004,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2873,7 +3022,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2888,7 +3038,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2908,7 +3059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2929,7 +3081,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2944,7 +3097,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2959,7 +3113,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2976,7 +3131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2991,7 +3147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3011,7 +3168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3032,7 +3190,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3047,7 +3206,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3062,7 +3222,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3079,7 +3240,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3094,7 +3256,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3114,7 +3277,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3135,7 +3299,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3150,7 +3315,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3165,7 +3331,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3182,7 +3349,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3197,7 +3365,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3217,7 +3386,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3238,7 +3408,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3253,7 +3424,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3268,7 +3440,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3285,7 +3458,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3300,7 +3474,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3320,7 +3495,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3335,7 +3511,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3407,7 +3584,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3424,7 +3602,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3448,7 +3627,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3469,7 +3649,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3484,7 +3665,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3499,7 +3681,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3516,7 +3699,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3531,7 +3715,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3551,7 +3736,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3566,7 +3752,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3638,7 +3825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3655,7 +3843,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3679,7 +3868,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3694,7 +3884,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3790,7 +3981,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3807,7 +3999,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3831,7 +4024,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3852,7 +4046,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3867,7 +4062,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3882,7 +4078,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3899,7 +4096,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3914,7 +4112,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3934,7 +4133,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3955,7 +4155,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3970,7 +4171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3985,7 +4187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4002,7 +4205,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4017,7 +4221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4037,7 +4242,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4058,7 +4264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4073,7 +4280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4088,7 +4296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4105,7 +4314,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4120,27 +4330,34 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59",
+                    "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59"
+                }
+            ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4150,12 +4367,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4165,12 +4383,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -4180,27 +4399,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b",
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59",
-                    "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
index 8277ff8bf7e89..c1984828750eb 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -80,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -91,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "1df94c0f-15fd-4b68-8ca3-6053a0332362",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-03-10 16:27:54.970000",
-                "date_modified": "2023-03-10 16:27:55.097000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -110,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -127,22 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                },
+                {
+                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
+                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1830,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1843,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-03-10 16:27:54.907000",
-                "date_modified": "2023-03-10 16:27:54.907000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1858,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1875,31 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                },
-                {
-                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
-                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,27 +2297,34 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                }
+            ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2218,12 +2334,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2233,12 +2350,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2248,27 +2366,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
index f3714bba6364d..804a8d74d0d51 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -80,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -91,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "1df94c0f-15fd-4b68-8ca3-6053a0332362",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-03-10 16:27:54.970000",
-                "date_modified": "2023-03-10 16:27:55.097000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -110,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -127,22 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.SalesReason,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                },
+                {
+                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
+                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1830,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1843,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-03-10 16:27:54.907000",
-                "date_modified": "2023-03-10 16:27:54.907000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1858,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1875,31 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.SalesReason,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                },
-                {
-                    "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
-                    "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,27 +2297,34 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
+                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
+                }
+            ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2218,12 +2334,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2233,12 +2350,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2248,27 +2366,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
-                    "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
index d25d23daae2ea..9d1b288057a16 100644
--- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
+++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
@@ -16,7 +16,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +32,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +48,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +66,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -78,7 +82,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -95,7 +100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -106,11 +112,11 @@
     "aspect": {
         "json": {
             "customProperties": {
-                "job_id": "b6a0c1e2-f90a-4c86-a226-bf7ca59ad79f",
+                "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642",
                 "job_name": "Weekly Demo Data Backup",
                 "description": "No description available.",
-                "date_created": "2023-08-06 21:01:05.157000",
-                "date_modified": "2023-08-06 21:01:05.283000",
+                "date_created": "2023-10-27 10:11:55.540000",
+                "date_modified": "2023-10-27 10:11:55.667000",
                 "step_id": "1",
                 "step_name": "Set database to read only",
                 "subsystem": "TSQL",
@@ -125,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -142,7 +149,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -163,7 +171,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -178,7 +187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -193,7 +203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -245,7 +258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -281,7 +296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -296,7 +312,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -313,7 +330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -348,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -369,7 +389,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -384,7 +405,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -399,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -416,7 +439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -431,7 +455,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -451,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -472,7 +498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -502,7 +530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -519,7 +548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -534,7 +564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +585,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -590,7 +623,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -605,7 +639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -637,7 +673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -657,7 +694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -693,7 +732,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -708,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -725,7 +766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -740,7 +782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -760,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +825,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +841,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -811,7 +857,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -828,7 +875,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -843,7 +891,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -863,7 +912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -884,7 +934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -899,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -914,7 +966,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -931,7 +984,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -946,7 +1000,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1021,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -987,7 +1043,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1002,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1017,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1034,7 +1093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1109,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1069,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1090,7 +1152,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1105,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1120,7 +1184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1137,7 +1202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1152,7 +1218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1172,7 +1239,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1187,7 +1255,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1259,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1276,7 +1346,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1321,7 +1393,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1336,7 +1409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1351,7 +1425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1368,7 +1443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1383,7 +1459,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1403,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1491,7 +1570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1508,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1532,7 +1613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1547,7 +1629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1644,7 +1727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1661,7 +1745,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1685,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1700,7 +1786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1796,7 +1883,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1813,7 +1901,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1837,7 +1926,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1854,12 +1944,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -1867,14 +1958,14 @@
             "customProperties": {
                 "procedure_depends_on": "{}",
                 "depending_on_procedure": "{}",
-                "code": "CREATE PROCEDURE Foo.DBs @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
+                "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n    SELECT @ID AS ThatDB;\n",
                 "input parameters": "['@ID']",
                 "parameter @ID": "{'type': 'int'}",
-                "date_created": "2023-08-06 21:01:05.093000",
-                "date_modified": "2023-08-06 21:01:05.093000"
+                "date_created": "2023-10-27 10:11:55.460000",
+                "date_modified": "2023-10-27 10:11:55.460000"
             },
             "externalUrl": "",
-            "name": "demodata.Foo.DBs",
+            "name": "demodata.Foo.Proc.With.SpecialChar",
             "type": {
                 "string": "MSSQL_STORED_PROCEDURE"
             }
@@ -1882,12 +1973,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -1899,7 +1991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1920,7 +2013,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1935,7 +2029,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1950,7 +2045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1967,7 +2063,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1982,7 +2079,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2002,7 +2100,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2023,7 +2122,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2038,7 +2138,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2053,7 +2154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2070,7 +2172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2188,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2105,7 +2209,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,7 +2231,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2141,7 +2247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2156,7 +2263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2173,7 +2281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,7 +2297,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2208,7 +2318,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2223,7 +2334,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2238,7 +2350,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2253,12 +2366,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),DBs)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2268,7 +2382,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mssql-test"
+        "runId": "mssql-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
index c1347a7c8caca..a17d52f9a39b1 100644
--- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
+++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
@@ -45,7 +45,7 @@ CREATE TABLE Foo.SalesReason
    )
 ;
 GO
-CREATE PROCEDURE Foo.DBs @ID INT
+CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT
 AS
     SELECT @ID AS ThatDB;
 GO

From 8b1d2094aa768d1f795a2d240de888e65c26f6fc Mon Sep 17 00:00:00 2001
From: richenc <125420929+richenc@users.noreply.github.com>
Date: Fri, 27 Oct 2023 10:36:47 -0700
Subject: [PATCH 116/259] feat(airflow): retry callback, support
 ExternalTaskSensor subclasses (#8514)

Co-authored-by: Richie Chen <richie.chen@hulu.com>
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../client/airflow_generator.py               |  7 +++-
 .../datahub_plugin_v22.py                     | 36 ++++++++++++++++++-
 .../integration/goldens/v1_basic_iolets.json  |  7 +++-
 .../integration/goldens/v1_simple_dag.json    | 14 ++++++--
 .../integration/goldens/v2_basic_iolets.json  |  7 +++-
 .../v2_basic_iolets_no_dag_listener.json      |  7 +++-
 .../integration/goldens/v2_simple_dag.json    | 12 +++++--
 .../v2_simple_dag_no_dag_listener.json        | 14 ++++++--
 .../goldens/v2_snowflake_operator.json        |  7 +++-
 .../goldens/v2_sqlite_operator.json           | 27 +++++++++++---
 .../v2_sqlite_operator_no_dag_listener.json   | 35 +++++++++++++++---
 11 files changed, 151 insertions(+), 22 deletions(-)

diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
index 16585f70e820b..e1d53be7bae6b 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py
@@ -98,7 +98,7 @@ def _get_dependencies(
         # It is possible to tie an external sensor to DAG if external_task_id is omitted but currently we can't tie
         # jobflow to anothet jobflow.
         external_task_upstreams = []
-        if task.task_type == "ExternalTaskSensor":
+        if isinstance(task, ExternalTaskSensor):
             task = cast(ExternalTaskSensor, task)
             if hasattr(task, "external_task_id") and task.external_task_id is not None:
                 external_task_upstreams = [
@@ -155,6 +155,8 @@ def generate_dataflow(
             "_concurrency",
             # "_default_view",
             "catchup",
+            "description",
+            "doc_md",
             "fileloc",
             "is_paused_upon_creation",
             "start_date",
@@ -431,6 +433,9 @@ def run_datajob(
         job_property_bag["operator"] = str(ti.operator)
         job_property_bag["priority_weight"] = str(ti.priority_weight)
         job_property_bag["log_url"] = ti.log_url
+        job_property_bag["orchestrator"] = "airflow"
+        job_property_bag["dag_id"] = str(dag.dag_id)
+        job_property_bag["task_id"] = str(ti.task_id)
         dpi.properties.update(job_property_bag)
         dpi.url = ti.log_url
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
index 046fbb5efaa03..f9a2119f51e32 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py
@@ -23,6 +23,7 @@
 
 TASK_ON_FAILURE_CALLBACK = "on_failure_callback"
 TASK_ON_SUCCESS_CALLBACK = "on_success_callback"
+TASK_ON_RETRY_CALLBACK = "on_retry_callback"
 
 
 def get_task_inlets_advanced(task: BaseOperator, context: Any) -> Iterable[Any]:
@@ -259,6 +260,28 @@ def custom_on_success_callback(context):
     return custom_on_success_callback
 
 
+def _wrap_on_retry_callback(on_retry_callback):
+    def custom_on_retry_callback(context):
+        config = get_lineage_config()
+        if config.enabled:
+            context["_datahub_config"] = config
+            try:
+                datahub_task_status_callback(
+                    context, status=InstanceRunResult.UP_FOR_RETRY
+                )
+            except Exception as e:
+                if not config.graceful_exceptions:
+                    raise e
+                else:
+                    print(f"Exception: {traceback.format_exc()}")
+
+        # Call original policy
+        if on_retry_callback:
+            on_retry_callback(context)
+
+    return custom_on_retry_callback
+
+
 def task_policy(task: Union[BaseOperator, MappedOperator]) -> None:
     task.log.debug(f"Setting task policy for Dag: {task.dag_id} Task: {task.task_id}")
     # task.add_inlets(["auto"])
@@ -274,7 +297,14 @@ def task_policy(task: Union[BaseOperator, MappedOperator]) -> None:
         on_success_callback_prop: property = getattr(
             MappedOperator, TASK_ON_SUCCESS_CALLBACK
         )
-        if not on_failure_callback_prop.fset or not on_success_callback_prop.fset:
+        on_retry_callback_prop: property = getattr(
+            MappedOperator, TASK_ON_RETRY_CALLBACK
+        )
+        if (
+            not on_failure_callback_prop.fset
+            or not on_success_callback_prop.fset
+            or not on_retry_callback_prop.fset
+        ):
             task.log.debug(
                 "Using MappedOperator's partial_kwargs instead of callback properties"
             )
@@ -284,10 +314,14 @@ def task_policy(task: Union[BaseOperator, MappedOperator]) -> None:
             task.partial_kwargs[TASK_ON_SUCCESS_CALLBACK] = _wrap_on_success_callback(
                 task.on_success_callback
             )
+            task.partial_kwargs[TASK_ON_RETRY_CALLBACK] = _wrap_on_retry_callback(
+                task.on_retry_callback
+            )
             return
 
     task.on_failure_callback = _wrap_on_failure_callback(task.on_failure_callback)  # type: ignore
     task.on_success_callback = _wrap_on_success_callback(task.on_success_callback)  # type: ignore
+    task.on_retry_callback = _wrap_on_retry_callback(task.on_retry_callback)  # type: ignore
     # task.pre_execute = _wrap_pre_execution(task.pre_execute)
 
 
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
index 26aa2afaa831a..a4c17c73e9c7e 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -373,7 +375,10 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets",
+                "orchestrator": "airflow",
+                "dag_id": "basic_iolets",
+                "task_id": "run_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets",
             "name": "basic_iolets_run_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
index b2e3a1fe47da7..a0a95716a0993 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -302,7 +304,10 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "task_1"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag",
             "name": "simple_dag_task_1_manual_run_test",
@@ -433,6 +438,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -654,7 +661,10 @@
                 "state": "success",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "run_another_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag",
             "name": "simple_dag_run_another_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
index 2e733c2ad40a9..1974f1f085df0 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -224,7 +226,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "basic_iolets",
+                "task_id": "run_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
             "name": "basic_iolets_run_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
index 44b288efda954..d02951bc9e82d 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -224,7 +226,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "basic_iolets",
+                "task_id": "run_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1",
             "name": "basic_iolets_run_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
index 454c509279e11..9acc47ec1321e 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -189,7 +191,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "task_1"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_task_1_manual_run_test",
@@ -523,7 +528,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "run_another_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_run_another_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
index 73b5765e96b7d..03299c483f57f 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -189,7 +191,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "2",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "task_1"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_task_1_manual_run_test",
@@ -435,6 +440,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "'A simple DAG that runs a few fake data tasks.'",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -579,7 +586,10 @@
                 "state": "running",
                 "operator": "BashOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "simple_dag",
+                "task_id": "run_another_data_task"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1",
             "name": "simple_dag_run_another_data_task_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
index affc395d421da..11a0b17b45b95 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -234,7 +236,10 @@
                 "state": "running",
                 "operator": "SnowflakeOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "snowflake_operator",
+                "task_id": "transform_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1",
             "name": "snowflake_operator_transform_cost_table_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
index 81d0a71b651d9..19e4aac9fb95e 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -201,7 +203,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "5",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "create_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_create_cost_table_manual_run_test",
@@ -562,7 +567,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "4",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "populate_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
@@ -922,7 +930,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "3",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "transform_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
@@ -1364,7 +1375,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
@@ -1658,7 +1672,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_processed_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
index 96a0f02ccec17..b67464b385335 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
@@ -9,6 +9,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -201,7 +203,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "5",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "create_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_create_cost_table_manual_run_test",
@@ -460,6 +465,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -617,7 +624,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "4",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "populate_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=populate_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
@@ -805,6 +815,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -1032,7 +1044,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "3",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "transform_cost_table"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
@@ -1370,6 +1385,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -1529,7 +1546,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
@@ -1719,6 +1739,8 @@
             "customProperties": {
                 "_access_control": "None",
                 "catchup": "False",
+                "description": "None",
+                "doc_md": "None",
                 "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
@@ -1878,7 +1900,10 @@
                 "state": "running",
                 "operator": "SqliteOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1"
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
+                "orchestrator": "airflow",
+                "dag_id": "sqlite_operator",
+                "task_id": "cleanup_processed_costs"
             },
             "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1",
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",

From 649f6d031789252fb9ac97d932fd71396f4875f2 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Sat, 28 Oct 2023 04:02:43 +0900
Subject: [PATCH 117/259] docs: fix saasonly flags for some pages (#9124)

---
 docs-website/sidebars.js                 | 29 +++++++++++++++++++-----
 docs/managed-datahub/chrome-extension.md |  2 --
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 31d69aec46d8b..39eaea57444ed 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -442,11 +442,29 @@ module.exports = {
         },
         "docs/act-on-metadata/impact-analysis",
         {
-          Observability: [
-            "docs/managed-datahub/observe/freshness-assertions",
-            "docs/managed-datahub/observe/volume-assertions",
-            "docs/managed-datahub/observe/custom-sql-assertions",
-            "docs/managed-datahub/observe/column-assertions",
+          label: "Observability",
+          type: "category",
+          items: [
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/freshness-assertions",
+              className: "saasOnly",
+            },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/volume-assertions",
+              className: "saasOnly",
+            },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/custom-sql-assertions",
+              className: "saasOnly",
+            },
+            {
+              type: "doc",
+              id: "docs/managed-datahub/observe/column-assertions",
+              className: "saasOnly",
+            },
           ],
         },
         {
@@ -606,7 +624,6 @@ module.exports = {
         {
           type: "doc",
           id: "docs/managed-datahub/chrome-extension",
-          className: "saasOnly",
         },
         {
           "Managed DataHub Release History": [
diff --git a/docs/managed-datahub/chrome-extension.md b/docs/managed-datahub/chrome-extension.md
index 0aa0860d03b67..a4560bc8cc09b 100644
--- a/docs/managed-datahub/chrome-extension.md
+++ b/docs/managed-datahub/chrome-extension.md
@@ -1,10 +1,8 @@
 ---
 description: Learn how to upload and use the Acryl DataHub Chrome extension (beta) locally before it's available on the Chrome store.
 ---
-import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
 # Acryl DataHub Chrome Extension
-<FeatureAvailability saasOnly />
 
 ## Installing the Extension
 

From e02b9096bd68c14944e640dbd3a235651ecebbaf Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Sat, 28 Oct 2023 00:33:43 +0530
Subject: [PATCH 118/259] fix(ingest/snowflake): missing view downstream cll if
 platform instance is set (#8966)

---
 .../source/snowflake/snowflake_lineage_v2.py  |   75 +-
 .../tests/integration/snowflake/common.py     |    2 +-
 .../snowflake/snowflake_golden.json           |  706 ++++--
 .../snowflake_privatelink_golden.json         | 2075 ++++++++++++-----
 .../integration/snowflake/test_snowflake.py   |    5 +-
 5 files changed, 2055 insertions(+), 808 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
index 9649054dbe6cb..4219533dc217c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py
@@ -20,12 +20,12 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.sql_parsing_builder import SqlParsingBuilder
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage
 from datahub.ingestion.source.snowflake.constants import (
     LINEAGE_PERMISSION_ERROR,
     SnowflakeEdition,
-    SnowflakeObjectDomain,
 )
 from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config
 from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery
@@ -53,7 +53,6 @@
     sqlglot_lineage,
 )
 from datahub.utilities.time import ts_millis_to_datetime
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -195,20 +194,6 @@ def get_table_upstream_workunits(
                 f"Upstream lineage detected for {self.report.num_tables_with_upstreams} tables.",
             )
 
-    def _gen_workunit_from_sql_parsing_result(
-        self,
-        dataset_identifier: str,
-        result: SqlParsingResult,
-    ) -> Iterable[MetadataWorkUnit]:
-        upstreams, fine_upstreams = self.get_upstreams_from_sql_parsing_result(
-            self.dataset_urn_builder(dataset_identifier), result
-        )
-        if upstreams:
-            self.report.num_views_with_upstreams += 1
-            yield self._create_upstream_lineage_workunit(
-                dataset_identifier, upstreams, fine_upstreams
-            )
-
     def _gen_workunits_from_query_result(
         self,
         discovered_assets: Collection[str],
@@ -242,18 +227,31 @@ def get_view_upstream_workunits(
         schema_resolver: SchemaResolver,
         view_definitions: MutableMapping[str, str],
     ) -> Iterable[MetadataWorkUnit]:
-        views_processed = set()
+        views_failed_parsing = set()
         if self.config.include_view_column_lineage:
             with PerfTimer() as timer:
+                builder = SqlParsingBuilder(
+                    generate_lineage=True,
+                    generate_usage_statistics=False,
+                    generate_operations=False,
+                )
                 for view_identifier, view_definition in view_definitions.items():
                     result = self._run_sql_parser(
                         view_identifier, view_definition, schema_resolver
                     )
-                    if result:
-                        views_processed.add(view_identifier)
-                        yield from self._gen_workunit_from_sql_parsing_result(
-                            view_identifier, result
+                    if result and result.out_tables:
+                        self.report.num_views_with_upstreams += 1
+                        # This does not yield any workunits but we use
+                        # yield here to execute this method
+                        yield from builder.process_sql_parsing_result(
+                            result=result,
+                            query=view_definition,
+                            is_view_ddl=True,
                         )
+                    else:
+                        views_failed_parsing.add(view_identifier)
+
+                yield from builder.gen_workunits()
                 self.report.view_lineage_parse_secs = timer.elapsed_seconds()
 
         with PerfTimer() as timer:
@@ -261,7 +259,7 @@ def get_view_upstream_workunits(
 
             if results:
                 yield from self._gen_workunits_from_query_result(
-                    set(discovered_views) - views_processed,
+                    views_failed_parsing,
                     results,
                     upstream_for_view=True,
                 )
@@ -349,39 +347,6 @@ def get_upstreams_from_query_result_row(
 
         return upstreams, fine_upstreams
 
-    def get_upstreams_from_sql_parsing_result(
-        self, downstream_table_urn: str, result: SqlParsingResult
-    ) -> Tuple[List[UpstreamClass], List[FineGrainedLineage]]:
-        # Note: This ignores the out_tables section of the sql parsing result.
-        upstreams = [
-            UpstreamClass(dataset=upstream_table_urn, type=DatasetLineageTypeClass.VIEW)
-            for upstream_table_urn in set(result.in_tables)
-        ]
-
-        # Maps downstream_col -> [upstream_col]
-        fine_lineage: Dict[str, Set[SnowflakeColumnId]] = defaultdict(set)
-        for column_lineage in result.column_lineage or []:
-            out_column = column_lineage.downstream.column
-            for upstream_column_info in column_lineage.upstreams:
-                upstream_table_name = DatasetUrn.create_from_string(
-                    upstream_column_info.table
-                ).get_dataset_name()
-                fine_lineage[out_column].add(
-                    SnowflakeColumnId(
-                        columnName=upstream_column_info.column,
-                        objectName=upstream_table_name,
-                        objectDomain=SnowflakeObjectDomain.VIEW.value,
-                    )
-                )
-        fine_upstreams = [
-            self.build_finegrained_lineage(
-                downstream_table_urn, downstream_col, upstream_cols
-            )
-            for downstream_col, upstream_cols in fine_lineage.items()
-        ]
-
-        return upstreams, list(filter(None, fine_upstreams))
-
     def _populate_external_lineage_map(self, discovered_tables: List[str]) -> None:
         with PerfTimer() as timer:
             self.report.num_external_table_edges_scanned = 0
diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py
index 81e307a78ae9e..ff448eca01071 100644
--- a/metadata-ingestion/tests/integration/snowflake/common.py
+++ b/metadata-ingestion/tests/integration/snowflake/common.py
@@ -94,7 +94,7 @@ def default_query_results(  # noqa: C901
                 "name": "VIEW_{}".format(view_idx),
                 "created_on": datetime(2021, 6, 8, 0, 0, 0, 0),
                 "comment": "Comment for View",
-                "text": None,
+                "text": f"create view view_{view_idx} as select * from table_{view_idx}",
             }
             for view_idx in range(1, num_views + 1)
         ]
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
index a424b258e68ff..c7273fee5a2e5 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
@@ -24,7 +24,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -39,7 +40,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -54,7 +56,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -71,7 +74,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -90,7 +94,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -105,7 +110,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -121,7 +127,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -150,7 +157,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -165,7 +173,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -180,7 +189,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -197,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -216,7 +227,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -231,7 +243,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -251,7 +264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -266,7 +280,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -462,7 +477,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -488,7 +504,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -503,7 +520,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -520,7 +538,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -544,7 +563,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -559,7 +579,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -755,7 +776,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -781,7 +803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -796,7 +819,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -813,7 +837,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -837,7 +862,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -852,7 +878,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1048,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1074,7 +1102,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1089,7 +1118,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1106,7 +1136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1130,7 +1161,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1145,7 +1177,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1341,7 +1374,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1367,7 +1401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1382,7 +1417,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1399,7 +1435,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1423,7 +1460,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1438,7 +1476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1634,7 +1673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1660,7 +1700,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1675,7 +1716,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1692,7 +1734,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1716,7 +1759,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1731,7 +1775,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1927,7 +1972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1953,7 +1999,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1968,7 +2015,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1985,7 +2033,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2009,7 +2058,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2024,7 +2074,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2220,7 +2271,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2246,7 +2298,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2261,7 +2314,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2278,7 +2332,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2302,7 +2357,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2317,7 +2373,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2513,7 +2570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2539,7 +2597,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2554,7 +2613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2571,7 +2631,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2595,7 +2656,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2610,7 +2672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2806,7 +2869,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2832,7 +2896,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2847,7 +2912,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2864,7 +2930,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2888,7 +2955,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2903,7 +2971,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3099,7 +3168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3125,7 +3195,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3140,7 +3211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3157,7 +3229,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3181,7 +3254,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3197,7 +3271,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3212,7 +3287,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3382,7 +3458,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3408,7 +3485,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3423,7 +3501,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3440,7 +3519,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_1 as select * from table_1",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3464,7 +3562,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3480,7 +3579,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3496,7 +3596,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3512,7 +3613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3527,7 +3629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3690,7 +3793,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3716,7 +3820,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "snowflake-2023_08_04-09_52_28"
+        "runId": "snowflake-2023_08_04-09_52_28",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3731,7 +3836,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3748,7 +3854,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3773,7 +3880,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3797,7 +3905,26 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_2 as select * from table_2",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3819,7 +3946,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3841,7 +3969,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3863,7 +3992,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3885,7 +4015,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3907,7 +4038,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3929,7 +4061,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3951,7 +4084,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3973,7 +4107,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3995,7 +4130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4017,7 +4153,145 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "upstreamLineage",
+    "aspect": {
+        "json": {
+            "upstreams": [
+                {
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4034,14 +4308,127 @@
                         "actor": "urn:li:corpuser:unknown"
                     },
                     "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
     },
     "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "lastObserved": 1615443388097,
+        "runId": "snowflake-2023_10_06-17_59_03",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4204,7 +4591,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4340,7 +4728,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4476,7 +4865,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4612,7 +5002,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4748,7 +5139,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4884,7 +5276,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5020,7 +5413,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5156,7 +5550,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5292,7 +5687,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5428,7 +5824,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5456,7 +5853,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5484,7 +5882,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5512,7 +5911,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5540,7 +5940,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5568,7 +5969,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5596,7 +5998,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5624,7 +6027,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5652,7 +6056,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5680,7 +6085,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5708,7 +6114,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5736,7 +6143,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5764,7 +6172,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5786,7 +6195,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5808,7 +6218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5830,7 +6241,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5852,7 +6264,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5874,7 +6287,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5896,7 +6310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5918,7 +6333,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5940,7 +6356,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5962,7 +6379,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5984,7 +6402,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -5999,7 +6418,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6014,7 +6434,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6029,7 +6450,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6044,7 +6466,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -6059,7 +6482,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00"
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
index 5057dacd5b0c8..5e55860483d24 100644
--- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
+++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
@@ -1,13 +1,14 @@
 [
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
     "aspectName": "containerProperties",
     "aspect": {
         "json": {
             "customProperties": {
                 "platform": "snowflake",
+                "instance": "instance1",
                 "env": "PROD",
                 "database": "test_db"
             },
@@ -29,13 +30,14 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "removed": false
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -46,12 +48,12 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:snowflake"
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -61,15 +63,13 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Database"
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -80,12 +80,14 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "path": []
+            "typeNames": [
+                "Database"
+            ]
         }
     },
     "systemMetadata": {
@@ -96,26 +98,17 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "containerProperties",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "customProperties": {
-                "platform": "snowflake",
-                "env": "PROD",
-                "database": "test_db",
-                "schema": "test_schema"
-            },
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/",
-            "name": "TEST_SCHEMA",
-            "description": "comment for TEST_DB.TEST_SCHEMA",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            }
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -126,12 +119,13 @@
 },
 {
     "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityUrn": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "removed": false
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -141,13 +135,24 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "dataPlatformInstance",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:snowflake"
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
+            "name": "TABLE_3",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -157,14 +162,14 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "Schema"
+                "Table"
             ]
         }
     },
@@ -175,32 +180,24 @@
     }
 },
 {
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "container",
-    "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -212,13 +209,28 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "containerProperties",
     "aspect": {
         "json": {
-            "removed": false
+            "customProperties": {
+                "platform": "snowflake",
+                "instance": "instance1",
+                "env": "PROD",
+                "database": "test_db",
+                "schema": "test_schema"
+            },
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/",
+            "name": "TEST_SCHEMA",
+            "description": "comment for TEST_DB.TEST_SCHEMA",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            }
         }
     },
     "systemMetadata": {
@@ -229,12 +241,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_1",
+            "schemaName": "test_db.test_schema.table_3",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -392,24 +404,13 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
-            "name": "TABLE_1",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -420,12 +421,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -435,14 +436,14 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
         "json": {
             "typeNames": [
-                "Table"
+                "Schema"
             ]
         }
     },
@@ -453,20 +454,53 @@
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 }
             ]
         }
@@ -479,7 +513,85 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_1 as select * from table_1",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/",
+            "name": "VIEW_1",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1",
+            "description": "Comment for View",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -495,12 +607,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_2",
+            "schemaName": "test_db.test_schema.table_1",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -659,23 +771,14 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
-            "name": "TABLE_2",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "typeNames": [
+                "View"
+            ]
         }
     },
     "systemMetadata": {
@@ -686,12 +789,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -702,39 +805,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
+            "name": "TABLE_1",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -745,12 +832,25 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -761,12 +861,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_3",
+            "schemaName": "test_db.test_schema.view_1",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -925,23 +1025,43 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
-            "name": "TABLE_3",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -952,12 +1072,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -968,14 +1088,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -986,19 +1105,131 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "upstreamLineage",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
+            "upstreams": [
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "auditStamp": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
@@ -1011,7 +1242,40 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1027,12 +1291,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_4",
+            "schemaName": "test_db.test_schema.table_2",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -1191,15 +1455,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
-            "name": "TABLE_4",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
+            "name": "TABLE_10",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
             "description": "Comment for Table",
             "created": {
                 "time": 1623110400000
@@ -1218,66 +1482,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1293,7 +1498,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
@@ -1457,7 +1662,39 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
@@ -1484,12 +1721,41 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
+            "name": "TABLE_2",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "typeNames": [
+                "Table"
+            ]
         }
     },
     "systemMetadata": {
@@ -1500,7 +1766,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -1518,19 +1784,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -1543,12 +1813,25 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "removed": false
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -1559,12 +1842,76 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "schemaMetadata",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_6",
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "test_db.test_schema.table_10",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -1723,93 +2070,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
-            "name": "TABLE_6",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1825,12 +2086,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_7",
+            "schemaName": "test_db.test_schema.table_6",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -1989,15 +2250,48 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
-            "name": "TABLE_7",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
+            "name": "TABLE_6",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
             "description": "Comment for Table",
             "created": {
                 "time": 1623110400000
@@ -2016,12 +2310,29 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2032,7 +2343,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2050,19 +2361,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -2075,7 +2390,40 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2091,12 +2439,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_8",
+            "schemaName": "test_db.test_schema.table_4",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -2255,15 +2603,15 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
-            "name": "TABLE_8",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
+            "name": "TABLE_7",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
             "description": "Comment for Table",
             "created": {
                 "time": 1623110400000
@@ -2282,12 +2630,39 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
+            "name": "TABLE_4",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -2298,7 +2673,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "subTypes",
     "aspect": {
@@ -2316,19 +2691,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
                 {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
                 },
                 {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -2341,12 +2720,43 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "status",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "removed": false
+            "typeNames": [
+                "Table"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
+            ]
         }
     },
     "systemMetadata": {
@@ -2357,12 +2767,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_9",
+            "schemaName": "test_db.test_schema.table_7",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -2521,93 +2931,7 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
-    "aspect": {
-        "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
-            "name": "TABLE_9",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "Table"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1654621200000,
-        "runId": "snowflake-2022_06_07-17_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -2623,12 +2947,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
     "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "schemaName": "test_db.test_schema.table_10",
+            "schemaName": "test_db.test_schema.table_8",
             "platform": "urn:li:dataPlatform:snowflake",
             "version": 0,
             "created": {
@@ -2787,23 +3111,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "customProperties": {},
-            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
-            "name": "TABLE_10",
-            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
-            "description": "Comment for Table",
-            "created": {
-                "time": 1623110400000
-            },
-            "lastModified": {
-                "time": 1623110400000
-            },
-            "tags": []
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2814,12 +3128,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -2830,14 +3144,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "subTypes",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "typeNames": [
-                "Table"
-            ]
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
+            "name": "TABLE_8",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -2848,21 +3171,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
+    "aspectName": "container",
     "aspect": {
         "json": {
-            "path": [
-                {
-                    "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585",
-                    "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585"
-                },
-                {
-                    "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c",
-                    "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c"
-                }
-            ]
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
         }
     },
     "systemMetadata": {
@@ -2873,21 +3187,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2898,20 +3204,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -2923,19 +3222,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -2948,21 +3251,12 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "status",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -2973,21 +3267,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
         }
     },
     "systemMetadata": {
@@ -2998,21 +3284,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
-            ]
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
+            "name": "TABLE_9",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
+            "description": "Comment for Table",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
@@ -3023,20 +3311,13 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "subTypes",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "typeNames": [
+                "Table"
             ]
         }
     },
@@ -3048,19 +3329,23 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
-            "upstreams": [
+            "path": [
                 {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
                 }
             ]
         }
@@ -3073,20 +3358,300 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
     "changeType": "UPSERT",
-    "aspectName": "upstreamLineage",
+    "aspectName": "schemaMetadata",
     "aspect": {
         "json": {
-            "upstreams": [
-                {
-                    "auditStamp": {
-                        "time": 0,
-                        "actor": "urn:li:corpuser:unknown"
-                    },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
-                }
+            "schemaName": "test_db.test_schema.table_9",
+            "platform": "urn:li:dataPlatform:snowflake",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "col_1",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "NUMBER(38,0)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_2",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_3",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_4",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_5",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_6",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_7",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_8",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_9",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_10",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "create view view_2 as select * from table_2",
+            "viewLanguage": "SQL"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:snowflake",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/",
+            "name": "VIEW_2",
+            "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2",
+            "description": "Comment for View",
+            "created": {
+                "time": 1623110400000
+            },
+            "lastModified": {
+                "time": 1623110400000
+            },
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)"
+                },
+                {
+                    "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab",
+                    "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab"
+                },
+                {
+                    "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f",
+                    "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+                }
             ]
         }
     },
@@ -3098,7 +3663,187 @@
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "test_db.test_schema.view_2",
+            "platform": "urn:li:dataPlatform:snowflake",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.MySqlDDL": {
+                    "tableSchema": ""
+                }
+            },
+            "fields": [
+                {
+                    "fieldPath": "col_1",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "NUMBER(38,0)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_2",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_3",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_4",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_5",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_6",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_7",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_8",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_9",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "col_10",
+                    "nullable": false,
+                    "description": "Comment for column",
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "VARCHAR(255)",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "snowflake-2022_06_07-17_00_00",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
     "changeType": "UPSERT",
     "aspectName": "upstreamLineage",
     "aspect": {
@@ -3109,8 +3854,120 @@
                         "time": 0,
                         "actor": "urn:li:corpuser:unknown"
                     },
-                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
-                    "type": "TRANSFORMED"
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)",
+                    "type": "VIEW"
+                }
+            ],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_1)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_1)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_10)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_10)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_2)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_2)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_3)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_3)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_4)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_4)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_5)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_5)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_6)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_6)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_7)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_7)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_8)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_8)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_9)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_9)"
+                    ],
+                    "confidenceScore": 1.0
                 }
             ]
         }
diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
index 3dafe85ef950a..4c00e48ede9fb 100644
--- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
+++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py
@@ -211,11 +211,12 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_
                         include_technical_schema=True,
                         include_table_lineage=True,
                         include_column_lineage=False,
-                        include_views=False,
-                        include_view_lineage=False,
+                        include_views=True,
+                        include_view_lineage=True,
                         include_usage_stats=False,
                         incremental_lineage=False,
                         include_operational_stats=False,
+                        platform_instance="instance1",
                         start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace(
                             tzinfo=timezone.utc
                         ),

From 4d2c009d400406b3cc41767864b07e9933dfe841 Mon Sep 17 00:00:00 2001
From: kushagra-apptware <81357546+kushagra-apptware@users.noreply.github.com>
Date: Sat, 28 Oct 2023 01:32:11 +0530
Subject: [PATCH 119/259] feat: Add flag to hide/display the autocomplete query
 for search bar (#9104)

Co-authored-by: John Joyce <john@acryl.io>
---
 datahub-web-react/src/app/home/HomePageHeader.tsx | 1 +
 datahub-web-react/src/app/search/SearchBar.tsx    | 6 ++++--
 datahub-web-react/src/app/search/SearchHeader.tsx | 1 +
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
index 5919d2dbf5b7e..e5c01252a865b 100644
--- a/datahub-web-react/src/app/home/HomePageHeader.tsx
+++ b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -275,6 +275,7 @@ export const HomePageHeader = () => {
                         viewsEnabled={viewsEnabled}
                         combineSiblings
                         showQuickFilters
+                        showViewAllResults
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>
diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
index b4699994bc460..5f797e68fe0e8 100644
--- a/datahub-web-react/src/app/search/SearchBar.tsx
+++ b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -119,6 +119,7 @@ interface Props {
     setIsSearchBarFocused?: (isSearchBarFocused: boolean) => void;
     onFocus?: () => void;
     onBlur?: () => void;
+    showViewAllResults?: boolean;
 }
 
 const defaultProps = {
@@ -146,6 +147,7 @@ export const SearchBar = ({
     setIsSearchBarFocused,
     onFocus,
     onBlur,
+    showViewAllResults = false,
 }: Props) => {
     const history = useHistory();
     const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
@@ -203,7 +205,7 @@ export const SearchBar = ({
     const { quickFilters, selectedQuickFilter, setSelectedQuickFilter } = useQuickFiltersContext();
 
     const autoCompleteQueryOptions = useMemo(() => {
-        if (effectiveQuery === '') return [];
+        if (effectiveQuery === '' || !showViewAllResults) return [];
 
         return [
             {
@@ -212,7 +214,7 @@ export const SearchBar = ({
                 type: EXACT_AUTOCOMPLETE_OPTION_TYPE,
             },
         ];
-    }, [effectiveQuery]);
+    }, [effectiveQuery, showViewAllResults]);
 
     const autoCompleteEntityOptions = useMemo(() => {
         return suggestions.map((suggestion: AutoCompleteResultForEntity) => {
diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx
index 74bc562e275d1..91f9753a3d601 100644
--- a/datahub-web-react/src/app/search/SearchHeader.tsx
+++ b/datahub-web-react/src/app/search/SearchHeader.tsx
@@ -107,6 +107,7 @@ export const SearchHeader = ({
                     combineSiblings
                     fixAutoComplete
                     showQuickFilters
+                    showViewAllResults
                 />
             </LogoSearchContainer>
             <NavGroup>

From aceff13ebb2d6758a5e42b592f4b5eb7d5af29e3 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Sat, 28 Oct 2023 04:55:57 +0530
Subject: [PATCH 120/259] docs(timeline): correct markdown heading level
 (#9126)

---
 docs/dev-guides/timeline.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/dev-guides/timeline.md b/docs/dev-guides/timeline.md
index 829aef1d3eefa..6a8e158d40ebf 100644
--- a/docs/dev-guides/timeline.md
+++ b/docs/dev-guides/timeline.md
@@ -228,7 +228,7 @@ http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3Adat
 	REMOVE GLOSSARY_TERM dataset:hive:testTimelineDataset (urn:li:glossaryTerm:SavingsAccount): The GlossaryTerm 'SavingsAccount' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed.
 ```
 
-# Explore the API
+## Explore the API
 
 The API is browse-able via the UI through through the dropdown.
 Here are a few screenshots showing how to navigate to it. You can try out the API and send example requests.
@@ -243,7 +243,7 @@ Here are a few screenshots showing how to navigate to it. You can try out the AP
 </p>
 
 
-# Future Work
+## Future Work
 
 - Supporting versions as start and end parameters as part of the call to the timeline API
 - Supporting entities beyond Datasets

From 9ae0e93d82eac2040af2c3d23d52878e57e19df1 Mon Sep 17 00:00:00 2001
From: Ellie O'Neil <110510035+eboneil@users.noreply.github.com>
Date: Fri, 27 Oct 2023 20:18:31 -0700
Subject: [PATCH 121/259] docs(graphql): Correct mutation -> query for
 searchAcrossLineage examples (#9134)

---
 docs/api/tutorials/lineage.md                            | 8 ++------
 metadata-ingestion/examples/library/read_lineage_rest.py | 2 +-
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/docs/api/tutorials/lineage.md b/docs/api/tutorials/lineage.md
index 4baad09099d07..13ec716b7870b 100644
--- a/docs/api/tutorials/lineage.md
+++ b/docs/api/tutorials/lineage.md
@@ -113,12 +113,10 @@ Expected Response:
 
 You can now see the lineage between `fct_users_deleted` and `logging_events`.
 
-
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/apis/tutorials/lineage-added.png"/>
 </p>
 
-
 ## Add Column-level Lineage
 
 <Tabs>
@@ -135,12 +133,10 @@ You can now see the lineage between `fct_users_deleted` and `logging_events`.
 
 You can now see the column-level lineage between datasets. Note that you have to enable `Show Columns` to be able to see the column-level lineage.
 
-
 <p align="center">
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/apis/tutorials/column-level-lineage-added.png"/>
 </p>
 
-
 ## Read Lineage
 
 <Tabs>
@@ -180,7 +176,7 @@ query searchAcrossLineage {
 }
 ```
 
-This example shows using lineage degrees as a filter, but additional search filters can be included here as well. 
+This example shows using lineage degrees as a filter, but additional search filters can be included here as well.
 
 </TabItem>
 <TabItem value="curl" label="Curl">
@@ -188,7 +184,7 @@ This example shows using lineage degrees as a filter, but additional search filt
 ```shell
 curl --location --request POST 'http://localhost:8080/api/graphql' \
 --header 'Authorization: Bearer <my-access-token>' \
---header 'Content-Type: application/json'  --data-raw '{ { "query": "mutation searchAcrossLineage { searchAcrossLineage( input: { query: \"*\" urn: \"urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.adoption.human_profiles,PROD)\" start: 0 count: 10 direction: DOWNSTREAM orFilters: [ { and: [ { condition: EQUAL negated: false field: \"degree\" values: [\"1\", \"2\", \"3+\"] } ] } ] } ) { searchResults { degree entity { urn type } } }}"
+--header 'Content-Type: application/json'  --data-raw '{ { "query": "query searchAcrossLineage { searchAcrossLineage( input: { query: \"*\" urn: \"urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.adoption.human_profiles,PROD)\" start: 0 count: 10 direction: DOWNSTREAM orFilters: [ { and: [ { condition: EQUAL negated: false field: \"degree\" values: [\"1\", \"2\", \"3+\"] } ] } ] } ) { searchResults { degree entity { urn type } } }}"
 }}'
 ```
 
diff --git a/metadata-ingestion/examples/library/read_lineage_rest.py b/metadata-ingestion/examples/library/read_lineage_rest.py
index 34437ed86280d..bd9b4e8651dba 100644
--- a/metadata-ingestion/examples/library/read_lineage_rest.py
+++ b/metadata-ingestion/examples/library/read_lineage_rest.py
@@ -6,7 +6,7 @@
 
 # Query multiple aspects from entity
 query = """
-mutation searchAcrossLineage {
+query searchAcrossLineage {
   searchAcrossLineage(
     input: {
       query: "*"

From 3f4ab44a91bff734e0a0437622d7579410875ec5 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Sun, 29 Oct 2023 16:26:05 -0500
Subject: [PATCH 122/259] feat(kafka): increase kafka message size and enable
 compression (#9038)

Co-authored-by: Pedro Silva <pedro@acryl.io>
---
 .github/workflows/docker-unified.yml          | 10 +++-
 build.gradle                                  |  2 +-
 .../app/client/KafkaTrackingProducer.java     | 10 ++--
 .../app/config/ConfigurationProvider.java     |  6 ++-
 docker/broker/env/docker.env                  |  4 +-
 docker/datahub-frontend/Dockerfile            |  4 +-
 docker/datahub-gms/Dockerfile                 |  4 +-
 docker/datahub-mae-consumer/Dockerfile        |  4 +-
 docker/datahub-mce-consumer/Dockerfile        |  4 +-
 docker/datahub-upgrade/Dockerfile             |  4 +-
 docker/kafka-setup/kafka-config.sh            |  2 +
 docker/kafka-setup/kafka-setup.sh             | 46 ++++++++++++-------
 docker/kafka-setup/kafka-topic-workers.sh     | 10 +++-
 .../docker-compose-m1.quickstart.yml          |  2 +
 ...er-compose-without-neo4j-m1.quickstart.yml |  2 +
 ...ocker-compose-without-neo4j.quickstart.yml |  2 +
 .../quickstart/docker-compose.quickstart.yml  |  2 +
 docs/deploy/environment-vars.md               | 22 +++++----
 .../config/kafka/ConsumerConfiguration.java   | 10 ++++
 .../config/kafka/KafkaConfiguration.java      |  2 +
 .../config/kafka/ProducerConfiguration.java   |  4 ++
 .../src/main/resources/application.yml        |  4 ++
 .../kafka/DataHubKafkaProducerFactory.java    |  2 +
 .../kafka/KafkaEventConsumerFactory.java      |  4 ++
 .../kafka/SimpleKafkaConsumerFactory.java     |  9 +++-
 25 files changed, 135 insertions(+), 40 deletions(-)
 create mode 100644 metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index 8666a5e2e2171..5f5a62de6288c 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -851,8 +851,14 @@ jobs:
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
-          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
+          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()
diff --git a/build.gradle b/build.gradle
index cf55a59cfe694..bd282535fa13c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -39,7 +39,7 @@ buildscript {
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id 'com.palantir.docker' version '0.35.0'
+  id 'com.palantir.docker' version '0.35.0' apply false
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }
diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java
index fab17f9215d4a..59e91a6d5a0f7 100644
--- a/datahub-frontend/app/client/KafkaTrackingProducer.java
+++ b/datahub-frontend/app/client/KafkaTrackingProducer.java
@@ -1,6 +1,8 @@
 package client;
 
+import com.linkedin.metadata.config.kafka.ProducerConfiguration;
 import com.typesafe.config.Config;
+import config.ConfigurationProvider;
 import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -35,12 +37,12 @@ public class KafkaTrackingProducer {
     private final KafkaProducer<String, String> _producer;
 
     @Inject
-    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle) {
+    public KafkaTrackingProducer(@Nonnull Config config, ApplicationLifecycle lifecycle, final ConfigurationProvider configurationProvider) {
         _isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled");
 
         if (_isEnabled) {
             _logger.debug("Analytics tracking is enabled");
-            _producer = createKafkaProducer(config);
+            _producer = createKafkaProducer(config, configurationProvider.getKafka().getProducer());
 
             lifecycle.addStopHook(
                     () -> {
@@ -62,13 +64,15 @@ public void send(ProducerRecord<String, String> record) {
         _producer.send(record);
     }
 
-    private static KafkaProducer createKafkaProducer(Config config) {
+    private static KafkaProducer createKafkaProducer(Config config, ProducerConfiguration producerConfiguration) {
         final Properties props = new Properties();
         props.put(ProducerConfig.CLIENT_ID_CONFIG, "datahub-frontend");
         props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.getString("analytics.kafka.delivery.timeout.ms"));
         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getString("analytics.kafka.bootstrap.server"));
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // Actor urn.
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // JSON object.
+        props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, producerConfiguration.getMaxRequestSize());
+        props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerConfiguration.getCompressionType());
 
         final String securityProtocolConfig = "analytics.kafka.security.protocol";
         if (config.hasPath(securityProtocolConfig)
diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java
index 00a5472ec3476..8f526c831b5c9 100644
--- a/datahub-frontend/app/config/ConfigurationProvider.java
+++ b/datahub-frontend/app/config/ConfigurationProvider.java
@@ -1,6 +1,7 @@
 package config;
 
 import com.linkedin.metadata.config.cache.CacheConfiguration;
+import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import lombok.Data;
 
@@ -11,7 +12,6 @@
 
 /**
  * Minimal sharing between metadata-service and frontend
- * Initially for use of client caching configuration.
  * Does not use the factories module to avoid transitive dependencies.
  */
 @EnableConfigurationProperties
@@ -19,6 +19,10 @@
 @ConfigurationProperties
 @Data
 public class ConfigurationProvider {
+    /**
+     * Kafka related configs.
+     */
+    private KafkaConfiguration kafka;
 
     /**
      * Configuration for caching
diff --git a/docker/broker/env/docker.env b/docker/broker/env/docker.env
index 18115697c2832..6eb958609daf1 100644
--- a/docker/broker/env/docker.env
+++ b/docker/broker/env/docker.env
@@ -5,4 +5,6 @@ KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9
 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
 KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
-KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
\ No newline at end of file
+KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+KAFKA_MESSAGE_MAX_BYTES=5242880
+KAFKA_MAX_MESSAGE_BYTES=5242880
\ No newline at end of file
diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile
index 9efc0d2ce8753..9c13e73078042 100644
--- a/docker/datahub-frontend/Dockerfile
+++ b/docker/datahub-frontend/Dockerfile
@@ -8,10 +8,12 @@ RUN addgroup -S datahub && adduser -S datahub -G datahub
 # Upgrade Alpine and base packages
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl sqlite \
+    && apk --no-cache add curl sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 
 COPY ./datahub-frontend.zip /
diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
index f5428f7480403..e271188a703cc 100644
--- a/docker/datahub-gms/Dockerfile
+++ b/docker/datahub-gms/Dockerfile
@@ -18,7 +18,7 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash coreutils gcompat sqlite \
+    && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
@@ -29,6 +29,8 @@ RUN apk --no-cache --update-cache --available upgrade \
     && cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 COPY war.war /datahub/datahub-gms/bin/war.war
 COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-gms/resources/entity-registry.yml
diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
index 4b321b1639c1b..ec3da4de71d15 100644
--- a/docker/datahub-mae-consumer/Dockerfile
+++ b/docker/datahub-mae-consumer/Dockerfile
@@ -18,7 +18,7 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash coreutils sqlite \
+    && apk --no-cache add curl bash coreutils sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
@@ -26,6 +26,8 @@ RUN apk --no-cache --update-cache --available upgrade \
     && cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 COPY mae-consumer-job.jar /datahub/datahub-mae-consumer/bin/
 COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-mae-consumer/resources/entity-registry.yml
diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
index 4d38ee6daa235..f9c47f77a98f5 100644
--- a/docker/datahub-mce-consumer/Dockerfile
+++ b/docker/datahub-mce-consumer/Dockerfile
@@ -18,7 +18,7 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash sqlite \
+    && apk --no-cache add curl bash sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
@@ -33,6 +33,8 @@ COPY docker/datahub-mce-consumer/start.sh /datahub/datahub-mce-consumer/scripts/
 COPY docker/monitoring/client-prometheus-config.yaml /datahub/datahub-mce-consumer/scripts/prometheus-config.yaml
 RUN chmod +x /datahub/datahub-mce-consumer/scripts/start.sh
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as dev-install
 # Dummy stage for development. Assumes code is built on your machine and mounted to this image.
 # See this excellent thread https://github.com/docker/cli/issues/1134
diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
index 945be54678a24..f08e7268e4018 100644
--- a/docker/datahub-upgrade/Dockerfile
+++ b/docker/datahub-upgrade/Dockerfile
@@ -18,7 +18,7 @@ FROM alpine:3 AS base
 ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add curl bash coreutils gcompat sqlite \
+    && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
@@ -28,6 +28,8 @@ RUN apk --no-cache --update-cache --available upgrade \
     && cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
 COPY --from=binary /go/bin/dockerize /usr/local/bin
 
+ENV LD_LIBRARY_PATH="/lib:/lib64"
+
 FROM base as prod-install
 COPY datahub-upgrade.jar /datahub/datahub-upgrade/bin/
 COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-gms/resources/entity-registry.yml
diff --git a/docker/kafka-setup/kafka-config.sh b/docker/kafka-setup/kafka-config.sh
index 2ba8e2d7c5d47..4d5698ccc3856 100644
--- a/docker/kafka-setup/kafka-config.sh
+++ b/docker/kafka-setup/kafka-config.sh
@@ -2,6 +2,7 @@
 
 : ${PARTITIONS:=1}
 : ${REPLICATION_FACTOR:=1}
+: ${MAX_MESSAGE_BYTES:=5242880}
 
 : ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:=PLAINTEXT}
 
@@ -12,3 +13,4 @@ export KAFKA_HEAP_OPTS="-Xmx64M"
 CONNECTION_PROPERTIES_PATH=/tmp/connection.properties
 
 WORKERS=4
+DELIMITER=";"
diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh
index b5024e49e59f1..439ffb4d4d829 100755
--- a/docker/kafka-setup/kafka-setup.sh
+++ b/docker/kafka-setup/kafka-setup.sh
@@ -102,24 +102,43 @@ exec 4<&-
 send() {
   work_id=$1
   topic_args=$2
-  echo sending $work_id $topic_args
-  echo "$work_id" "$topic_args" 1>&3 ## the fifo is fd 3
+  topic_config=$3
+
+  echo -e "sending $work_id\n   worker_args: ${topic_args}${DELIMITER}${topic_config}"
+  echo "$work_id" "${topic_args}${DELIMITER}${topic_config}" 1>&3 ## the fifo is fd 3
 }
 
 ## Produce the jobs to run.
-send "$METADATA_AUDIT_EVENT_NAME" "--partitions $PARTITIONS --topic $METADATA_AUDIT_EVENT_NAME"
-send "$METADATA_CHANGE_EVENT_NAME" "--partitions $PARTITIONS --topic $METADATA_CHANGE_EVENT_NAME"
-send "$FAILED_METADATA_CHANGE_EVENT_NAME" "--partitions $PARTITIONS --topic $FAILED_METADATA_CHANGE_EVENT_NAME"
-send "$METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME" "--partitions $PARTITIONS --topic $METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME"
+send "$METADATA_AUDIT_EVENT_NAME" "--partitions $PARTITIONS --topic $METADATA_AUDIT_EVENT_NAME" \
+     "--entity-type topics --entity-name $METADATA_AUDIT_EVENT_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
+
+send "$METADATA_CHANGE_EVENT_NAME" "--partitions $PARTITIONS --topic $METADATA_CHANGE_EVENT_NAME" \
+     "--entity-type topics --entity-name $METADATA_CHANGE_EVENT_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
+send "$FAILED_METADATA_CHANGE_EVENT_NAME" "--partitions $PARTITIONS --topic $FAILED_METADATA_CHANGE_EVENT_NAME" \
+     "--entity-type topics --entity-name $FAILED_METADATA_CHANGE_EVENT_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
+
+send "$METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME" "--partitions $PARTITIONS --topic $METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME" \
+     "--entity-type topics --entity-name $METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
 
 # Set retention to 90 days
-send "$METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME" "--partitions $PARTITIONS --config retention.ms=7776000000 --topic $METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME"
-send "$METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--partitions $PARTITIONS --topic $METADATA_CHANGE_PROPOSAL_TOPIC_NAME"
-send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--partitions $PARTITIONS --topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME"
-send "$PLATFORM_EVENT_TOPIC_NAME" "--partitions $PARTITIONS --topic $PLATFORM_EVENT_TOPIC_NAME"
+send "$METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME" "--partitions $PARTITIONS --config retention.ms=7776000000 --topic $METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME" \
+     "--entity-type topics --entity-name $METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
+
+send "$METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--partitions $PARTITIONS --topic $METADATA_CHANGE_PROPOSAL_TOPIC_NAME" \
+     "--entity-type topics --entity-name $METADATA_CHANGE_PROPOSAL_TOPIC_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
+send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--partitions $PARTITIONS --topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" \
+     "--entity-type topics --entity-name $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
+
+send "$PLATFORM_EVENT_TOPIC_NAME" "--partitions $PARTITIONS --topic $PLATFORM_EVENT_TOPIC_NAME" \
+     "--entity-type topics --entity-name $PLATFORM_EVENT_TOPIC_NAME --alter --add-config max.message.bytes=$MAX_MESSAGE_BYTES"
 
 # Infinite retention upgrade topic
-send "$DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" "--partitions 1 --config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC_NAME"
+  # Make sure the retention.ms config for $DATAHUB_UPGRADE_HISTORY_TOPIC_NAME is configured to infinite
+  # Please see the bug report below for details
+  # https://github.com/datahub-project/datahub/issues/7882
+send "$DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" "--partitions 1 --config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" \
+     "--entity-type topics --entity-name "$DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" --alter --add-config retention.ms=-1"
+
 # Create topic for datahub usage event
 if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then
   send "$DATAHUB_USAGE_EVENT_NAME" "--partitions $PARTITIONS --topic $DATAHUB_USAGE_EVENT_NAME"
@@ -150,8 +169,3 @@ if [[ $USE_CONFLUENT_SCHEMA_REGISTRY == "TRUE" ]]; then
       --entity-name _schemas \
       --alter --add-config cleanup.policy=compact
 fi
-
-# Make sure the retention.ms config for $DATAHUB_UPGRADE_HISTORY_TOPIC_NAME is configured to infinite
-# Please see the bug report below for details
-# https://github.com/datahub-project/datahub/issues/7882
-kafka-configs.sh --command-config $CONNECTION_PROPERTIES_PATH --bootstrap-server $KAFKA_BOOTSTRAP_SERVER --entity-type topics --entity-name "$DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" --alter --add-config retention.ms=-1
diff --git a/docker/kafka-setup/kafka-topic-workers.sh b/docker/kafka-setup/kafka-topic-workers.sh
index fd0d45c3f4611..3ddf41abbabf5 100644
--- a/docker/kafka-setup/kafka-topic-workers.sh
+++ b/docker/kafka-setup/kafka-topic-workers.sh
@@ -11,10 +11,18 @@ START_LOCK=$4
 ## the queue workers are supposed to be doing
 job() {
   i=$1
-  topic_args=$2
+  worker_args=$2
+  topic_args=$(echo $worker_args | cut -d "$DELIMITER" -f 1)
+  topic_config=$(echo $worker_args | cut -d "$DELIMITER" -f 2)
+
+  echo "   $i: kafka-topics.sh --create --if-not-exist $topic_args"
   kafka-topics.sh --create --if-not-exists --command-config $CONNECTION_PROPERTIES_PATH --bootstrap-server $KAFKA_BOOTSTRAP_SERVER \
      --replication-factor $REPLICATION_FACTOR \
      $topic_args
+  if [[ ! -z "$topic_config" ]]; then
+    echo "   $i: kafka-configs.sh $topic_config"
+    kafka-configs.sh --command-config $CONNECTION_PROPERTIES_PATH --bootstrap-server $KAFKA_BOOTSTRAP_SERVER $topic_config
+  fi
 }
 
 ## This is the worker to read from the queue.
diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml
index 89e9aaa0defd6..c5de687d335b9 100644
--- a/docker/quickstart/docker-compose-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-m1.quickstart.yml
@@ -16,6 +16,8 @@ services:
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
     - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+    - KAFKA_MESSAGE_MAX_BYTES=5242880
+    - KAFKA_MAX_MESSAGE_BYTES=5242880
     healthcheck:
       interval: 1s
       retries: 5
diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
index f6284edc83648..b6935f24c5ce2 100644
--- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
@@ -16,6 +16,8 @@ services:
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
     - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+    - KAFKA_MESSAGE_MAX_BYTES=5242880
+    - KAFKA_MAX_MESSAGE_BYTES=5242880
     healthcheck:
       interval: 1s
       retries: 5
diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
index 4e3503e35c0db..4ff8bbd70da85 100644
--- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -16,6 +16,8 @@ services:
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
     - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+    - KAFKA_MESSAGE_MAX_BYTES=5242880
+    - KAFKA_MAX_MESSAGE_BYTES=5242880
     healthcheck:
       interval: 1s
       retries: 5
diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
index e2f52064389e0..f2950ebab2c9d 100644
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@@ -16,6 +16,8 @@ services:
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
     - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
+    - KAFKA_MESSAGE_MAX_BYTES=5242880
+    - KAFKA_MAX_MESSAGE_BYTES=5242880
     healthcheck:
       interval: 1s
       retries: 5
diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md
index 779c3d3d7c432..4c7b249349ca0 100644
--- a/docs/deploy/environment-vars.md
+++ b/docs/deploy/environment-vars.md
@@ -67,15 +67,19 @@ In general, there are **lots** of Kafka configuration environment variables for
 These environment variables follow the standard Spring representation of properties as environment variables.
 Simply replace the dot, `.`, with an underscore, `_`, and convert to uppercase.
 
-| Variable                                            | Default                                      | Unit/Type | Components                              | Description                                                                                                                                                                      |
-|-----------------------------------------------------|----------------------------------------------|-----------|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `KAFKA_LISTENER_CONCURRENCY`                        | 1                                            | integer   | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Number of Kafka consumer threads. Optimize throughput by matching to topic partitions.                                                                                           |
-| `SPRING_KAFKA_PRODUCER_PROPERTIES_MAX_REQUEST_SIZE` | 1048576                                      | bytes     | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Max produced message size. Note that the topic configuration is not controlled by this variable.                                                                                 |
-| `SCHEMA_REGISTRY_TYPE`                              | `INTERNAL`                                   | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Schema registry implementation. One of `INTERNAL` or `KAFKA` or `AWS_GLUE`                                                                                                       |
-| `KAFKA_SCHEMAREGISTRY_URL`                          | `http://localhost:8080/schema-registry/api/` | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Schema registry url. Used for `INTERNAL` and `KAFKA`. The default value is for the `GMS` component. The `MCE Consumer` and `MAE Consumer` should be the `GMS` hostname and port. |
-| `AWS_GLUE_SCHEMA_REGISTRY_REGION`                   | `us-east-1`                                  | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`] | If using `AWS_GLUE` in the `SCHEMA_REGISTRY_TYPE` variable for the schema registry implementation.                                                                               |
-| `AWS_GLUE_SCHEMA_REGISTRY_NAME`                     | ``                                           | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`] | If using `AWS_GLUE` in the `SCHEMA_REGISTRY_TYPE` variable for the schema registry.                                                                                              |
-| `USE_CONFLUENT_SCHEMA_REGISTRY`                     | `true`                                       | boolean   | [`kafka-setup`]                         | Enable Confluent schema registry configuration.                                                                                                                                  |
+| Variable                                            | Default                                      | Unit/Type | Components                                             | Description                                                                                                                                                                                                                                                                                         |
+|-----------------------------------------------------|----------------------------------------------|-----------|--------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `KAFKA_LISTENER_CONCURRENCY`                        | 1                                            | integer   | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | Number of Kafka consumer threads. Optimize throughput by matching to topic partitions.                                                                                                                                                                                                              |
+| `SPRING_KAFKA_PRODUCER_PROPERTIES_MAX_REQUEST_SIZE` | 1048576                                      | bytes     | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | Max produced message size. Note that the topic configuration is not controlled by this variable.                                                                                                                                                                                                    |
+| `SCHEMA_REGISTRY_TYPE`                              | `INTERNAL`                                   | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | Schema registry implementation. One of `INTERNAL` or `KAFKA` or `AWS_GLUE`                                                                                                                                                                                                                          |
+| `KAFKA_SCHEMAREGISTRY_URL`                          | `http://localhost:8080/schema-registry/api/` | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | Schema registry url. Used for `INTERNAL` and `KAFKA`. The default value is for the `GMS` component. The `MCE Consumer` and `MAE Consumer` should be the `GMS` hostname and port.                                                                                                                    |
+| `AWS_GLUE_SCHEMA_REGISTRY_REGION`                   | `us-east-1`                                  | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | If using `AWS_GLUE` in the `SCHEMA_REGISTRY_TYPE` variable for the schema registry implementation.                                                                                                                                                                                                  |
+| `AWS_GLUE_SCHEMA_REGISTRY_NAME`                     | ``                                           | string    | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | If using `AWS_GLUE` in the `SCHEMA_REGISTRY_TYPE` variable for the schema registry.                                                                                                                                                                                                                 |
+| `USE_CONFLUENT_SCHEMA_REGISTRY`                     | `true`                                       | boolean   | [`kafka-setup`]                                        | Enable Confluent schema registry configuration.                                                                                                                                                                                                                                                     |
+| `KAFKA_PRODUCER_MAX_REQUEST_SIZE`                   | `5242880`                                    | integer   | [`Frontend`, `GMS`, `MCE Consumer`, `MAE Consumer`]    | Max produced message size. Note that the topic configuration is not controlled by this variable.                                                                                                                                                                                                    |
+| `KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES`          | `5242880`                                    | integer   | [`GMS`, `MCE Consumer`, `MAE Consumer`]                | The maximum amount of data per-partition the server will return. Records are fetched in batches by the consumer. If the first record batch in the first non-empty partition of the fetch is larger than this limit, the batch will still be returned to ensure that the consumer can make progress. |
+| `MAX_MESSAGE_BYTES`                                 | `5242880`                                    | integer   | [`kafka-setup`]                                        | Sets the max message size on the kakfa topics.                                                                                                                                                                                                                                                      |
+| `KAFKA_PRODUCER_COMPRESSION_TYPE`                   | `snappy`                                     | string    | [`Frontend`, `GMS`, `MCE Consumer`, `MAE Consumer`]    | The compression used by the producer.                                                                                                                                                                                                                                                               |
 
 ## Frontend
 
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java
new file mode 100644
index 0000000000000..7a93119226a2d
--- /dev/null
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java
@@ -0,0 +1,10 @@
+package com.linkedin.metadata.config.kafka;
+
+import lombok.Data;
+
+
+@Data
+public class ConsumerConfiguration {
+
+  private int maxPartitionFetchBytes;
+}
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java
index 2966abfc63396..2345f88352c17 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java
@@ -12,4 +12,6 @@ public class KafkaConfiguration {
   private SchemaRegistryConfiguration schemaRegistry;
 
   private ProducerConfiguration producer;
+
+  private ConsumerConfiguration consumer;
 }
diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java
index 2bf4cea3f0c18..26a8c6b649133 100644
--- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java
+++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java
@@ -13,4 +13,8 @@ public class ProducerConfiguration {
   private int requestTimeout;
 
   private int backoffTimeout;
+
+  private String compressionType;
+
+  private int maxRequestSize;
 }
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 5d72e24748072..b817208672e08 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -228,6 +228,10 @@ kafka:
     deliveryTimeout: ${KAFKA_PRODUCER_DELIVERY_TIMEOUT:30000}
     requestTimeout: ${KAFKA_PRODUCER_REQUEST_TIMEOUT:3000}
     backoffTimeout: ${KAFKA_PRODUCER_BACKOFF_TIMEOUT:500}
+    compressionType: ${KAFKA_PRODUCER_COMPRESSION_TYPE:snappy} # producer's compression algorithm
+    maxRequestSize: ${KAFKA_PRODUCER_MAX_REQUEST_SIZE:5242880} # the max bytes sent by the producer, also see kafka-setup MAX_MESSAGE_BYTES for matching value
+  consumer:
+    maxPartitionFetchBytes: ${KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES:5242880} # the max bytes consumed per partition
   schemaRegistry:
     type: ${SCHEMA_REGISTRY_TYPE:KAFKA} # INTERNAL or KAFKA or AWS_GLUE
     url: ${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081}
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java
index c67a2e704681f..78b3de501e0e5 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java
@@ -59,6 +59,8 @@ public static Map<String, Object> buildProducerProperties(SchemaRegistryConfig s
     props.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, kafkaConfiguration.getProducer().getDeliveryTimeout());
     props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, kafkaConfiguration.getProducer().getRequestTimeout());
     props.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, kafkaConfiguration.getProducer().getBackoffTimeout());
+    props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, kafkaConfiguration.getProducer().getCompressionType());
+    props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, kafkaConfiguration.getProducer().getMaxRequestSize());
 
     // Override KafkaProperties with SchemaRegistryConfig only for non-empty values
     schemaRegistryConfig.getProperties().entrySet()
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java
index ba18be6834d14..7a9e80781d639 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java
@@ -70,6 +70,7 @@ private static Map<String, Object> buildCustomizedProperties(KafkaProperties bas
         consumerProps.setEnableAutoCommit(true);
         consumerProps.setAutoCommitInterval(Duration.ofSeconds(10));
 
+
         // KAFKA_BOOTSTRAP_SERVER has precedence over SPRING_KAFKA_BOOTSTRAP_SERVERS
         if (kafkaConfiguration.getBootstrapServers() != null && kafkaConfiguration.getBootstrapServers().length() > 0) {
             consumerProps.setBootstrapServers(Arrays.asList(kafkaConfiguration.getBootstrapServers().split(",")));
@@ -84,6 +85,9 @@ private static Map<String, Object> buildCustomizedProperties(KafkaProperties bas
                 .filter(entry -> entry.getValue() != null && !entry.getValue().toString().isEmpty())
                 .forEach(entry -> customizedProperties.put(entry.getKey(), entry.getValue()));
 
+        customizedProperties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG,
+                kafkaConfiguration.getConsumer().getMaxPartitionFetchBytes());
+
         return customizedProperties;
     }
 
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java
index 05ebfdddf8b80..e12cbec87fe45 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java
@@ -4,8 +4,11 @@
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.Map;
+
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.generic.GenericRecord;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
@@ -40,10 +43,14 @@ protected KafkaListenerContainerFactory<?> createInstance(@Qualifier("configurat
       consumerProps.setBootstrapServers(Arrays.asList(kafkaConfiguration.getBootstrapServers().split(",")));
     } // else we rely on KafkaProperties which defaults to localhost:9092
 
+    Map<String, Object> customizedProperties = consumerProps.buildProperties();
+    customizedProperties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG,
+            kafkaConfiguration.getConsumer().getMaxPartitionFetchBytes());
+
     ConcurrentKafkaListenerContainerFactory<String, GenericRecord> factory =
         new ConcurrentKafkaListenerContainerFactory<>();
     factory.setContainerCustomizer(new ThreadPoolContainerCustomizer());
-    factory.setConsumerFactory(new DefaultKafkaConsumerFactory<>(properties.buildConsumerProperties()));
+    factory.setConsumerFactory(new DefaultKafkaConsumerFactory<>(customizedProperties));
 
     log.info("Simple KafkaListenerContainerFactory built successfully");
 

From 758ed47644b330efbbee4e61dde71f6ff5808e23 Mon Sep 17 00:00:00 2001
From: Dmytro Kulyk <34435869+KulykDmytro@users.noreply.github.com>
Date: Mon, 30 Oct 2023 06:14:32 +0200
Subject: [PATCH 123/259] feat(ingest/jsonschema) enable schema-aware
 `JsonSchemaTranslator` (#8971)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../src/datahub/ingestion/extractor/json_schema_util.py        | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py
index c943b83a887ed..360ddf1129154 100644
--- a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py
+++ b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py
@@ -598,7 +598,8 @@ def get_fields_from_schema(
                     jsonref_schema_dict = schema_dict
                 else:
                     # first validate the schema using a json validator
-                    jsonschema.Draft7Validator.check_schema(schema_dict)
+                    validator = jsonschema.validators.validator_for(schema_dict)
+                    validator.check_schema(schema_dict)
                     # then apply jsonref
                     jsonref_schema_dict = jsonref.loads(schema_string)
             except Exception as e:

From 2c019148ad451752eff582c3206df75c83fe2a63 Mon Sep 17 00:00:00 2001
From: Alex Klavens <123000295+alexklavensnyt@users.noreply.github.com>
Date: Mon, 30 Oct 2023 04:43:52 -0400
Subject: [PATCH 124/259] =?UTF-8?q?fix(metadata-ingestion):=20adds=20defau?=
 =?UTF-8?q?lt=20value=20to=20=5Fresolved=5Fdomain=5Furn=20i=E2=80=A6=20(#9?=
 =?UTF-8?q?115)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Alex Klavens <alexklavens@A14603.lan>
---
 .../src/datahub/api/entities/dataproduct/dataproduct.py         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
index 2d9b14ceb2d06..28e4a03b8f75f 100644
--- a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
+++ b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
@@ -104,7 +104,7 @@ class DataProduct(ConfigModel):
 
     id: str
     domain: str
-    _resolved_domain_urn: Optional[str]
+    _resolved_domain_urn: Optional[str] = None
     assets: Optional[List[str]] = None
     display_name: Optional[str] = None
     owners: Optional[List[Union[str, Ownership]]] = None

From f5c8192cca6eacc7e21e62204883854f0f6bcbdb Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Mon, 30 Oct 2023 18:32:51 +0530
Subject: [PATCH 125/259] ci: tweak to only run relevant workflows (#9052)

---
 .github/workflows/airflow-plugin.yml     |  2 +-
 .github/workflows/check-datahub-jars.yml | 12 ++++--------
 .github/workflows/documentation.yml      |  8 ++++++++
 .github/workflows/metadata-ingestion.yml |  2 +-
 .github/workflows/metadata-model.yml     |  5 ++---
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
index 54042d104d906..d0c0f52781b9a 100644
--- a/.github/workflows/airflow-plugin.yml
+++ b/.github/workflows/airflow-plugin.yml
@@ -12,7 +12,7 @@ on:
     branches:
       - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/airflow-plugin.yml"
       - "metadata-ingestion-modules/airflow-plugin/**"
       - "metadata-ingestion/**"
       - "metadata-models/**"
diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
index 9a17a70e7f8d4..41f9ea91a94e2 100644
--- a/.github/workflows/check-datahub-jars.yml
+++ b/.github/workflows/check-datahub-jars.yml
@@ -4,17 +4,13 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docker/**"
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-integration"
   pull_request:
     branches:
       - "**"
-    paths-ignore:
-      - "docker/**"
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-integration"
   release:
     types: [published]
 
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
index ebe2990f3a3cd..c94282938120e 100644
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@@ -4,9 +4,17 @@ on:
   pull_request:
     branches:
       - "**"
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docs-website/**"
   push:
     branches:
       - master
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docs-website/**"
   # release:
   #   types: [published, edited]
 
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
index 699ca330ce0ac..ec6bd4141cc6f 100644
--- a/.github/workflows/metadata-ingestion.yml
+++ b/.github/workflows/metadata-ingestion.yml
@@ -11,7 +11,7 @@ on:
     branches:
       - "**"
     paths:
-      - ".github/**"
+      - ".github/workflows/metadata-ingestion.yml"
       - "metadata-ingestion/**"
       - "metadata-models/**"
   release:
diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
index 9d54c88eee591..4bae5ccc9a266 100644
--- a/.github/workflows/metadata-model.yml
+++ b/.github/workflows/metadata-model.yml
@@ -3,9 +3,8 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
+    paths:
+      - "metadata-models/**"
   release:
     types: [published]
 

From 9c72bd9ed7f1c2b4228fc656c5f8b6f31bf0d431 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Mon, 30 Oct 2023 14:12:07 -0400
Subject: [PATCH 126/259] fix(test): Fix for flaky download_lineage_results
 cypress test (#9132)

---
 .../cypress/cypress/e2e/lineage/download_lineage_results.js   | 3 +++
 .../tests/cypress/cypress/e2e/mutations/dataset_ownership.js  | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
index 315aa7b22b9da..dc6efc9f7df66 100644
--- a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
@@ -27,6 +27,9 @@ const downloadCsvFile = (filename) => {
 };
 
 describe("download lineage results to .csv file", () => {
+     beforeEach(() => {
+        cy.on('uncaught:exception', (err, runnable) => { return false; });
+      });
 
     it("download and verify lineage results for 1st, 2nd and 3+ degree of dependencies", () => {
       cy.loginWithCredentials();
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js
index 99ad9a68d35e1..465d7998b9f9a 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/dataset_ownership.js
@@ -29,6 +29,10 @@ const addOwner = (owner, type, elementId) => {
 }
 
 describe("add, remove ownership for dataset", () => {
+    beforeEach(() => {
+        cy.on('uncaught:exception', (err, runnable) => { return false; });
+      });
+
     it("create test user and test group, add user to a group", () => {
         cy.loginWithCredentials();
         cy.createUser(username, password, email);

From 300cea373d6a94f05cf3bd95ab69bc503a28538e Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Mon, 30 Oct 2023 20:50:42 +0000
Subject: [PATCH 127/259] docs: Update updating-datahub.md (#9131)

---
 docs/how/updating-datahub.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 4d1535f28fa0a..28f11e4b6d707 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -4,10 +4,20 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ## Next
 
-- #9010 - In Redshift source's config `incremental_lineage` is set default to off.
+### Breaking Changes
+
+### Potential Downtime
+
+### Deprecations
+
+### Other Notable Changes
+
+## 0.12.0
 
 ### Breaking Changes
 
+- #9044 - GraphQL APIs for adding ownership now expect either an `ownershipTypeUrn` referencing a customer ownership type or a (deprecated) `type`. Where before adding an ownership without a concrete type was allowed, this is no longer the case. For simplicity you can use the `type` parameter which will get translated to a custom ownership type internally if one exists for the type being added.
+- #9010 - In Redshift source's config `incremental_lineage` is set default to off.
 - #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
 - #8942 - Removed `urn:li:corpuser:datahub` owner for the `Measure`, `Dimension` and `Temporal` tags emitted 
   by Looker and LookML source connectors.

From 58bcedcd6a091263c6dc3e1181c260233a80575d Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 30 Oct 2023 14:18:48 -0700
Subject: [PATCH 128/259] fix(ingest/clickhouse): pin version to solve column
 reflection regression (#9143)

---
 metadata-ingestion/setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 7f7826abe2095..b1c5510efd923 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -173,7 +173,9 @@
 
 clickhouse_common = {
     # Clickhouse 0.2.0 adds support for SQLAlchemy 1.4.x
-    "clickhouse-sqlalchemy>=0.2.0",
+    # Disallow 0.2.5 because of https://github.com/xzkostyan/clickhouse-sqlalchemy/issues/272.
+    # Note that there's also a known issue around nested map types: https://github.com/xzkostyan/clickhouse-sqlalchemy/issues/269.
+    "clickhouse-sqlalchemy>=0.2.0,<0.2.5",
 }
 
 redshift_common = {

From 51d6d1f4531dad133e06db75267fbea77e424d00 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 30 Oct 2023 14:19:52 -0700
Subject: [PATCH 129/259] feat(ingest/looker): cleanup error handling (#9135)

---
 .../src/datahub/ingestion/api/workunit.py          |  6 +++++-
 .../ingestion/source/looker/looker_lib_wrapper.py  |  8 ++++++--
 .../ingestion/source/looker/looker_source.py       | 14 ++------------
 .../ingestion/source/looker/lookml_source.py       |  5 +----
 4 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/api/workunit.py b/metadata-ingestion/src/datahub/ingestion/api/workunit.py
index 8eea3514a22af..b1c003ee27e12 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/workunit.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/workunit.py
@@ -22,7 +22,11 @@ class MetadataWorkUnit(WorkUnit):
     metadata: Union[
         MetadataChangeEvent, MetadataChangeProposal, MetadataChangeProposalWrapper
     ]
-    # A workunit creator can determine if this workunit is allowed to fail
+
+    # A workunit creator can determine if this workunit is allowed to fail.
+    # TODO: This flag was initially added during the rollout of the subType aspect
+    # to improve backwards compatibility, but is not really needed anymore and so
+    # should be removed.
     treat_errors_as_warnings: bool = False
 
     # When this is set to false, this MWU will be ignored by automatic helpers
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
index cf132b7ef27f7..b00f74b71e792 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
@@ -123,8 +123,12 @@ def get_user(self, id_: str, user_fields: str) -> Optional[User]:
                 transport_options=self.transport_options,
             )
         except SDKError as e:
-            logger.warning(f"Could not find user with id {id_}")
-            logger.warning(f"Failure was {e}")
+            if "Looker Not Found (404)" in str(e):
+                # User not found
+                logger.info(f"Could not find user with id {id_}: 404 error")
+            else:
+                logger.warning(f"Could not find user with id {id_}")
+                logger.warning(f"Failure was {e}")
         # User not found
         return None
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
index a3df977582ca4..09683d790c14c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -926,14 +926,7 @@ def process_metrics_dimensions_and_fields_for_dashboard(
         mcps = chart_mcps
         mcps.append(dashboard_mcp)
 
-        workunits = [
-            MetadataWorkUnit(
-                id=f"looker-{mcp.aspectName}-{mcp.entityUrn}",
-                mcp=mcp,
-                treat_errors_as_warnings=True,
-            )
-            for mcp in mcps
-        ]
+        workunits = [mcp.as_workunit() for mcp in mcps]
 
         return workunits
 
@@ -1320,10 +1313,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     id=f"looker-{event.proposedSnapshot.urn}", mce=event
                 )
             elif isinstance(event, MetadataChangeProposalWrapper):
-                # We want to treat subtype aspects as optional, so allowing failures in this aspect to be treated as warnings rather than failures
-                yield event.as_workunit(
-                    treat_errors_as_warnings=event.aspectName in ["subTypes"]
-                )
+                yield event.as_workunit()
             else:
                 raise Exception(f"Unexpected type of event {event}")
         self.reporter.report_stage_end("explore_metadata")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
index e69c3b6e601bd..e6b78cc7a7745 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
@@ -2171,10 +2171,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                                     for mcp in self._build_dataset_mcps(
                                         maybe_looker_view
                                     ):
-                                        # We want to treat mcp aspects as optional, so allowing failures in this aspect to be treated as warnings rather than failures
-                                        yield mcp.as_workunit(
-                                            treat_errors_as_warnings=True
-                                        )
+                                        yield mcp.as_workunit()
                                 else:
                                     (
                                         prev_model_name,

From 0bd2d9a36cdf18575ac4e54126db5be33ec59d8a Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 30 Oct 2023 14:22:05 -0700
Subject: [PATCH 130/259] feat(ingest): add `entity_supports_aspect` helper
 (#9120)

---
 .../src/datahub/emitter/mcp_builder.py              | 13 ++++++++++++-
 metadata-ingestion/tests/unit/test_mcp_builder.py   |  9 +++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
index 65e0c0d6ba60d..d50feba8b119c 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -1,9 +1,10 @@
-from typing import Dict, Iterable, List, Optional, TypeVar
+from typing import Dict, Iterable, List, Optional, Type, TypeVar
 
 from pydantic.fields import Field
 from pydantic.main import BaseModel
 
 from datahub.emitter.mce_builder import (
+    Aspect,
     datahub_guid,
     make_container_urn,
     make_data_platform_urn,
@@ -18,6 +19,7 @@
 )
 from datahub.metadata.com.linkedin.pegasus2avro.container import ContainerProperties
 from datahub.metadata.schema_classes import (
+    KEY_ASPECTS,
     ContainerClass,
     DomainsClass,
     EmbedClass,
@@ -306,3 +308,12 @@ def create_embed_mcp(urn: str, embed_url: str) -> MetadataChangeProposalWrapper:
         entityUrn=urn,
         aspect=EmbedClass(renderUrl=embed_url),
     )
+
+
+def entity_supports_aspect(entity_type: str, aspect_type: Type[Aspect]) -> bool:
+    entity_key_aspect = KEY_ASPECTS[entity_type]
+    aspect_name = aspect_type.get_aspect_name()
+
+    supported_aspects = entity_key_aspect.ASPECT_INFO["entityAspects"]
+
+    return aspect_name in supported_aspects
diff --git a/metadata-ingestion/tests/unit/test_mcp_builder.py b/metadata-ingestion/tests/unit/test_mcp_builder.py
index 561b782ef9e46..e304edb24789c 100644
--- a/metadata-ingestion/tests/unit/test_mcp_builder.py
+++ b/metadata-ingestion/tests/unit/test_mcp_builder.py
@@ -1,4 +1,5 @@
 import datahub.emitter.mcp_builder as builder
+from datahub.metadata.schema_classes import StatusClass, TelemetryClientIdClass
 
 
 def test_guid_generator():
@@ -83,3 +84,11 @@ def test_guid_generators():
 
     guid = key.guid()
     assert guid == guid_datahub
+
+
+def test_entity_supports_aspect():
+    assert builder.entity_supports_aspect("dataset", StatusClass)
+    assert not builder.entity_supports_aspect("telemetry", StatusClass)
+
+    assert not builder.entity_supports_aspect("dataset", TelemetryClientIdClass)
+    assert builder.entity_supports_aspect("telemetry", TelemetryClientIdClass)

From ce0f36b8bc74e3f0bab447408096347617804d92 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 30 Oct 2023 14:23:19 -0700
Subject: [PATCH 131/259] feat(sqlparser): support more update syntaxes + fix
 bug with subqueries (#9105)

---
 .../src/datahub/utilities/sqlglot_lineage.py  | 57 ++++++++++-
 .../test_postgres_select_subquery.json        | 64 ++++++++++++
 .../test_snowflake_update_from_table.json     |  1 +
 .../test_snowflake_update_hardcoded.json      |  4 +-
 .../goldens/test_snowflake_update_self.json   | 29 ++++++
 .../unit/sql_parsing/test_sqlglot_lineage.py  | 98 +++++++++++++++++++
 6 files changed, 247 insertions(+), 6 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_self.json

diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 1d74b20569814..388388f9f4b38 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -12,8 +12,8 @@
 import sqlglot.errors
 import sqlglot.lineage
 import sqlglot.optimizer.annotate_types
+import sqlglot.optimizer.optimizer
 import sqlglot.optimizer.qualify
-import sqlglot.optimizer.qualify_columns
 from pydantic import BaseModel
 from typing_extensions import TypedDict
 
@@ -48,6 +48,19 @@
 SQL_PARSE_RESULT_CACHE_SIZE = 1000
 
 
+RULES_BEFORE_TYPE_ANNOTATION: tuple = tuple(
+    filter(
+        # Skip pushdown_predicates because it sometimes throws exceptions, and we
+        # don't actually need it for anything.
+        lambda func: func.__name__ not in {"pushdown_predicates"},
+        itertools.takewhile(
+            lambda func: func != sqlglot.optimizer.annotate_types.annotate_types,
+            sqlglot.optimizer.optimizer.RULES,
+        ),
+    )
+)
+
+
 class GraphQLSchemaField(TypedDict):
     fieldPath: str
     nativeDataType: str
@@ -289,6 +302,10 @@ def _table_level_lineage(
     )
     # TODO: If a CTAS has "LIMIT 0", it's not really lineage, just copying the schema.
 
+    # Update statements implicitly read from the table being updated, so add those back in.
+    if isinstance(statement, sqlglot.exp.Update):
+        tables = tables | modified
+
     return tables, modified
 
 
@@ -568,17 +585,20 @@ def _schema_aware_fuzzy_column_resolve(
         # - the select instead of the full outer statement
         # - schema info
         # - column qualification enabled
+        # - running the full pre-type annotation optimizer
 
         # logger.debug("Schema: %s", sqlglot_db_schema.mapping)
-        statement = sqlglot.optimizer.qualify.qualify(
+        statement = sqlglot.optimizer.optimizer.optimize(
             statement,
             dialect=dialect,
             schema=sqlglot_db_schema,
+            qualify_columns=True,
             validate_qualify_columns=False,
             identify=True,
             # sqlglot calls the db -> schema -> table hierarchy "catalog", "db", "table".
             catalog=default_db,
             db=default_schema,
+            rules=RULES_BEFORE_TYPE_ANNOTATION,
         )
     except (sqlglot.errors.OptimizeError, ValueError) as e:
         raise SqlUnderstandingError(
@@ -748,6 +768,7 @@ def _extract_select_from_create(
 _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT: Set[str] = set(
     sqlglot.exp.Update.arg_types.keys()
 ) - set(sqlglot.exp.Select.arg_types.keys())
+_UPDATE_FROM_TABLE_ARGS_TO_MOVE = {"joins", "laterals", "pivot"}
 
 
 def _extract_select_from_update(
@@ -774,17 +795,43 @@ def _extract_select_from_update(
             # they'll get caught later.
             new_expressions.append(expr)
 
-    return sqlglot.exp.Select(
+    # Special translation for the `from` clause.
+    extra_args = {}
+    original_from = statement.args.get("from")
+    if original_from and isinstance(original_from.this, sqlglot.exp.Table):
+        # Move joins, laterals, and pivots from the Update->From->Table->field
+        # to the top-level Select->field.
+
+        for k in _UPDATE_FROM_TABLE_ARGS_TO_MOVE:
+            if k in original_from.this.args:
+                # Mutate the from table clause in-place.
+                extra_args[k] = original_from.this.args.pop(k)
+
+    select_statement = sqlglot.exp.Select(
         **{
             **{
                 k: v
                 for k, v in statement.args.items()
                 if k not in _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
             },
+            **extra_args,
             "expressions": new_expressions,
         }
     )
 
+    # Update statements always implicitly have the updated table in context.
+    # TODO: Retain table name alias.
+    if select_statement.args.get("from"):
+        # select_statement = sqlglot.parse_one(select_statement.sql(dialect=dialect))
+
+        select_statement = select_statement.join(
+            statement.this, append=True, join_kind="cross"
+        )
+    else:
+        select_statement = select_statement.from_(statement.this)
+
+    return select_statement
+
 
 def _is_create_table_ddl(statement: sqlglot.exp.Expression) -> bool:
     return isinstance(statement, sqlglot.exp.Create) and isinstance(
@@ -955,7 +1002,7 @@ def _sqlglot_lineage_inner(
     # Fetch schema info for the relevant tables.
     table_name_urn_mapping: Dict[_TableName, str] = {}
     table_name_schema_mapping: Dict[_TableName, SchemaInfo] = {}
-    for table in itertools.chain(tables, modified):
+    for table in tables | modified:
         # For select statements, qualification will be a no-op. For other statements, this
         # is where the qualification actually happens.
         qualified_table = table.qualified(
@@ -971,7 +1018,7 @@ def _sqlglot_lineage_inner(
         # Also include the original, non-qualified table name in the urn mapping.
         table_name_urn_mapping[table] = urn
 
-    total_tables_discovered = len(tables) + len(modified)
+    total_tables_discovered = len(tables | modified)
     total_schemas_resolved = len(table_name_schema_mapping)
     debug_info = SqlParsingDebugInfo(
         confidence=0.9 if total_tables_discovered == total_schemas_resolved
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json
new file mode 100644
index 0000000000000..0c40ce120c934
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json
@@ -0,0 +1,64 @@
+{
+    "query_type": "SELECT",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table2,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "a",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table1,PROD)",
+                    "column": "a"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "b",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table1,PROD)",
+                    "column": "b"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "c",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                    }
+                },
+                "native_column_type": "INT[]"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table2,PROD)",
+                    "column": "c"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
index e2baa34e7fe28..d51001f969799 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_from_table.json
@@ -1,6 +1,7 @@
 {
     "query_type": "UPDATE",
     "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
         "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
         "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)"
     ],
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
index b41ed61b37cdb..f421b28530c64 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_hardcoded.json
@@ -1,6 +1,8 @@
 {
     "query_type": "UPDATE",
-    "in_tables": [],
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
+    ],
     "out_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
     ],
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_self.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_self.json
new file mode 100644
index 0000000000000..c8cc32164a3eb
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_update_self.json
@@ -0,0 +1,29 @@
+{
+    "query_type": "UPDATE",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
+    ],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                "column": "orderkey",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "DECIMAL"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)",
+                    "column": "orderkey"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index dfc5b486abd35..5559ebe1756a6 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -768,3 +768,101 @@ def test_snowflake_update_from_table():
         },
         expected_file=RESOURCE_DIR / "test_snowflake_update_from_table.json",
     )
+
+
+def test_snowflake_update_self():
+    assert_sql_result(
+        """
+UPDATE snowflake_sample_data.tpch_sf1.orders
+SET orderkey = orderkey + 1
+""",
+        dialect="snowflake",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": {
+                "orderkey": "NUMBER(38,0)",
+                "totalprice": "NUMBER(12,2)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_update_self.json",
+    )
+
+
+def test_postgres_select_subquery():
+    assert_sql_result(
+        """
+SELECT
+    a,
+    b,
+    (SELECT c FROM table2 WHERE table2.id = table1.id) as c
+FROM table1
+""",
+        dialect="postgres",
+        default_db="my_db",
+        default_schema="my_schema",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table1,PROD)": {
+                "id": "INTEGER",
+                "a": "INTEGER",
+                "b": "INTEGER",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.table2,PROD)": {
+                "id": "INTEGER",
+                "c": "INTEGER",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_postgres_select_subquery.json",
+    )
+
+
+@pytest.mark.skip(reason="We can't parse column-list syntax with sub-selects yet")
+def test_postgres_update_subselect():
+    assert_sql_result(
+        """
+UPDATE accounts SET sales_person_name =
+    (SELECT name FROM employees
+     WHERE employees.id = accounts.sales_person_id)
+""",
+        dialect="postgres",
+        default_db="my_db",
+        default_schema="my_schema",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)": {
+                "id": "INTEGER",
+                "sales_person_id": "INTEGER",
+                "sales_person_name": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.employees,PROD)": {
+                "id": "INTEGER",
+                "name": "VARCHAR(16777216)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_postgres_update_subselect.json",
+    )
+
+
+@pytest.mark.skip(reason="We can't parse column-list syntax with sub-selects yet")
+def test_postgres_complex_update():
+    # Example query from the postgres docs:
+    # https://www.postgresql.org/docs/current/sql-update.html
+    assert_sql_result(
+        """
+UPDATE accounts SET (contact_first_name, contact_last_name) =
+    (SELECT first_name, last_name FROM employees
+     WHERE employees.id = accounts.sales_person);
+""",
+        dialect="postgres",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)": {
+                "id": "INTEGER",
+                "contact_first_name": "VARCHAR(16777216)",
+                "contact_last_name": "VARCHAR(16777216)",
+                "sales_person": "INTEGER",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.employees,PROD)": {
+                "id": "INTEGER",
+                "first_name": "VARCHAR(16777216)",
+                "last_name": "VARCHAR(16777216)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_postgres_complex_update.json",
+    )

From 94d438d44f2d18def4a422cd60150d2c9a78be49 Mon Sep 17 00:00:00 2001
From: sachinsaju <33017477+sachinsaju@users.noreply.github.com>
Date: Tue, 31 Oct 2023 08:54:56 +0530
Subject: [PATCH 132/259] docs: correct broken doc links (#9137)

Co-authored-by: Hyejin Yoon <0327jane@gmail.com>
---
 docs/deploy/aws.md                       | 2 +-
 docs/what-is-datahub/datahub-concepts.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md
index e0f57b4a0b0cb..6598b93c25e9a 100644
--- a/docs/deploy/aws.md
+++ b/docs/deploy/aws.md
@@ -15,7 +15,7 @@ This guide requires the following tools:
 - [kubectl](https://kubernetes.io/docs/tasks/tools/) to manage kubernetes resources
 - [helm](https://helm.sh/docs/intro/install/) to deploy the resources based on helm charts. Note, we only support Helm
     3.
-- [eksctl](https://eksctl.io/introduction/#installation) to create and manage clusters on EKS
+- [eksctl](https://eksctl.io/installation/) to create and manage clusters on EKS
 - [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) to manage AWS resources
 
 To use the above tools, you need to set up AWS credentials by following
diff --git a/docs/what-is-datahub/datahub-concepts.md b/docs/what-is-datahub/datahub-concepts.md
index 6328d97fa6a50..03b86fab0ede4 100644
--- a/docs/what-is-datahub/datahub-concepts.md
+++ b/docs/what-is-datahub/datahub-concepts.md
@@ -99,7 +99,7 @@ List of Data Platforms
 - Tableau
 - Vertica
 
-Reference : [data_platforms.json](https://github.com/acryldata/datahub-fork/blob/acryl-main/metadata-service/war/src/main/resources/boot/data_platforms.json)
+Reference : [data_platforms.json](https://github.com/datahub-project/datahub/blob/master/metadata-service/war/src/main/resources/boot/data_platforms.json)
 
 </details>
 

From ea1273281e3a65ab4d94d002ee19f91907a3eb84 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 30 Oct 2023 20:57:59 -0700
Subject: [PATCH 133/259] feat(ingest): sql parser perf + asyncio fixes (#9119)

---
 metadata-ingestion/setup.py                   |  2 +-
 .../src/datahub/cli/docker_cli.py             |  5 ++
 .../src/datahub/upgrade/upgrade.py            | 12 ++---
 .../src/datahub/utilities/sqlglot_lineage.py  |  5 +-
 .../goldens/test_select_from_union.json       |  2 +-
 .../test_teradata_strange_operators.json      | 46 +++++++++++++++++++
 .../unit/sql_parsing/test_sqlglot_lineage.py  | 14 ++++++
 7 files changed, 73 insertions(+), 13 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_strange_operators.json

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index b1c5510efd923..151842bd84d0a 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -108,7 +108,7 @@
 sqlglot_lib = {
     # Using an Acryl fork of sqlglot.
     # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
-    "acryl-sqlglot==18.5.2.dev45",
+    "acryl-sqlglot==18.17.1.dev16",
 }
 
 sql_common = (
diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py
index 4afccfe711e34..77e3285d359ef 100644
--- a/metadata-ingestion/src/datahub/cli/docker_cli.py
+++ b/metadata-ingestion/src/datahub/cli/docker_cli.py
@@ -5,6 +5,7 @@
 import os
 import pathlib
 import platform
+import signal
 import subprocess
 import sys
 import tempfile
@@ -770,6 +771,10 @@ def quickstart(  # noqa: C901
                     logger.debug("docker compose up still running, sending SIGKILL")
                     up_process.kill()
                     up_process.wait()
+            else:
+                # If the docker process got a keyboard interrupt, raise one here.
+                if up_process.returncode in {128 + signal.SIGINT, -signal.SIGINT}:
+                    raise KeyboardInterrupt
 
         # Check docker health every few seconds.
         status = check_docker_quickstart()
diff --git a/metadata-ingestion/src/datahub/upgrade/upgrade.py b/metadata-ingestion/src/datahub/upgrade/upgrade.py
index 30f19b8b84f35..acc7954ad25a6 100644
--- a/metadata-ingestion/src/datahub/upgrade/upgrade.py
+++ b/metadata-ingestion/src/datahub/upgrade/upgrade.py
@@ -1,6 +1,5 @@
 import asyncio
 import contextlib
-import functools
 import logging
 import sys
 from datetime import datetime, timedelta, timezone
@@ -374,17 +373,14 @@ def check_upgrade(func: Callable[..., T]) -> Callable[..., T]:
     @wraps(func)
     def async_wrapper(*args: Any, **kwargs: Any) -> Any:
         async def run_inner_func():
-            loop = asyncio.get_event_loop()
-            return await loop.run_in_executor(
-                None, functools.partial(func, *args, **kwargs)
-            )
+            return func(*args, **kwargs)
 
         async def run_func_check_upgrade():
             version_stats_future = asyncio.ensure_future(retrieve_version_stats())
-            the_one_future = asyncio.ensure_future(run_inner_func())
-            ret = await the_one_future
+            main_func_future = asyncio.ensure_future(run_inner_func())
+            ret = await main_func_future
 
-            # the one future has returned
+            # the main future has returned
             # we check the other futures quickly
             try:
                 version_stats = await asyncio.wait_for(version_stats_future, 0.5)
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 388388f9f4b38..6413275ac63a6 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -106,6 +106,7 @@ def get_query_type_of_sql(expression: sqlglot.exp.Expression) -> QueryType:
         sqlglot.exp.Update: QueryType.UPDATE,
         sqlglot.exp.Delete: QueryType.DELETE,
         sqlglot.exp.Merge: QueryType.MERGE,
+        sqlglot.exp.Subqueryable: QueryType.SELECT,  # unions, etc. are also selects
     }
 
     for cls, query_type in mapping.items():
@@ -820,10 +821,8 @@ def _extract_select_from_update(
     )
 
     # Update statements always implicitly have the updated table in context.
-    # TODO: Retain table name alias.
+    # TODO: Retain table name alias, if one was present.
     if select_statement.args.get("from"):
-        # select_statement = sqlglot.parse_one(select_statement.sql(dialect=dialect))
-
         select_statement = select_statement.join(
             statement.this, append=True, join_kind="cross"
         )
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json
index 902aa010c8afc..5d1d421f49a2a 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json
@@ -1,5 +1,5 @@
 {
-    "query_type": "UNKNOWN",
+    "query_type": "SELECT",
     "in_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf10.orders,PROD)",
         "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf100.orders,PROD)"
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_strange_operators.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_strange_operators.json
new file mode 100644
index 0000000000000..4b21a2512ccd1
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_teradata_strange_operators.json
@@ -0,0 +1,46 @@
+{
+    "query_type": "SELECT",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:teradata,dbc.table1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:teradata,dbc.table2,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "col1",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,dbc.table1,PROD)",
+                    "column": "col1"
+                },
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,dbc.table2,PROD)",
+                    "column": "col1"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "col2",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,dbc.table1,PROD)",
+                    "column": "col2"
+                },
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:teradata,dbc.table2,PROD)",
+                    "column": "col2"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index 5559ebe1756a6..3b9fa0d55f18d 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -675,6 +675,20 @@ def test_teradata_default_normalization():
     )
 
 
+def test_teradata_strange_operators():
+    assert_sql_result(
+        """
+select col1, col2 from dbc.table1
+where col1 eq 'value1'
+minus
+select col1, col2 from dbc.table2
+""",
+        dialect="teradata",
+        default_schema="dbc",
+        expected_file=RESOURCE_DIR / "test_teradata_strange_operators.json",
+    )
+
+
 def test_snowflake_update_hardcoded():
     assert_sql_result(
         """

From b565a657d2235b82e65dfbe0bfcc11c97c3d9b79 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 30 Oct 2023 23:35:12 -0700
Subject: [PATCH 134/259] feat(quickstart): fix broker
 InconsistentClusterIdException issues (#9148)

---
 docker/docker-compose-with-cassandra.yml                    | 6 +++++-
 docker/docker-compose-without-neo4j.yml                     | 6 +++++-
 docker/docker-compose.yml                                   | 6 +++++-
 docker/quickstart/docker-compose-m1.quickstart.yml          | 4 +++-
 .../docker-compose-without-neo4j-m1.quickstart.yml          | 4 +++-
 .../quickstart/docker-compose-without-neo4j.quickstart.yml  | 4 +++-
 docker/quickstart/docker-compose.quickstart.yml             | 4 +++-
 7 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml
index 9543e67da07f2..39f4341600572 100644
--- a/docker/docker-compose-with-cassandra.yml
+++ b/docker/docker-compose-with-cassandra.yml
@@ -200,7 +200,10 @@ services:
       retries: 5
       timeout: 5s
     volumes:
-      - zkdata:/var/lib/zookeeper
+    # See https://stackoverflow.com/a/61008432 for why we need two volumes.
+    # See also: https://docs.confluent.io/platform/current/installation/docker/operations/external-volumes.html#data-volumes-for-kafka-and-zk
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 networks:
   default:
     name: datahub_network
@@ -210,3 +213,4 @@ volumes:
   neo4jdata:
   broker:
   zkdata:
+  zklogs:
diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml
index 022362782f742..235e89e340551 100644
--- a/docker/docker-compose-without-neo4j.yml
+++ b/docker/docker-compose-without-neo4j.yml
@@ -174,7 +174,10 @@ services:
       retries: 3
       timeout: 5s
     volumes:
-    - zkdata:/var/lib/zookeeper
+    # See https://stackoverflow.com/a/61008432 for why we need two volumes.
+    # See also: https://docs.confluent.io/platform/current/installation/docker/operations/external-volumes.html#data-volumes-for-kafka-and-zk
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 networks:
   default:
     name: datahub_network
@@ -182,3 +185,4 @@ volumes:
   esdata:
   broker:
   zkdata:
+  zklogs:
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index a486689e050a2..46da8c6fdbd2a 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -195,7 +195,10 @@ services:
       retries: 3
       timeout: 5s
     volumes:
-    - zkdata:/var/lib/zookeeper
+    # See https://stackoverflow.com/a/61008432 for why we need two volumes.
+    # See also: https://docs.confluent.io/platform/current/installation/docker/operations/external-volumes.html#data-volumes-for-kafka-and-zk
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 networks:
   default:
     name: datahub_network
@@ -204,3 +207,4 @@ volumes:
   neo4jdata:
   broker:
   zkdata:
+  zklogs:
diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml
index c5de687d335b9..3b6d02c83d0f0 100644
--- a/docker/quickstart/docker-compose-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-m1.quickstart.yml
@@ -300,7 +300,8 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/lib/zookeeper
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 version: '3.9'
 volumes:
   broker: null
@@ -308,3 +309,4 @@ volumes:
   mysqldata: null
   neo4jdata: null
   zkdata: null
+  zklogs: null
diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
index b6935f24c5ce2..e45bafc3da480 100644
--- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
@@ -274,10 +274,12 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/lib/zookeeper
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 version: '3.9'
 volumes:
   broker: null
   esdata: null
   mysqldata: null
   zkdata: null
+  zklogs: null
diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
index 4ff8bbd70da85..020ef5e9a97b9 100644
--- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -274,10 +274,12 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/lib/zookeeper
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 version: '3.9'
 volumes:
   broker: null
   esdata: null
   mysqldata: null
   zkdata: null
+  zklogs: null
diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
index f2950ebab2c9d..8adc2b9063b84 100644
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@@ -300,7 +300,8 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/lib/zookeeper
+    - zkdata:/var/lib/zookeeper/data
+    - zklogs:/var/lib/zookeeper/log
 version: '3.9'
 volumes:
   broker: null
@@ -308,3 +309,4 @@ volumes:
   mysqldata: null
   neo4jdata: null
   zkdata: null
+  zklogs: null

From 2e8954f33a10f3e11af22fe6198fea43d65d580c Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Tue, 31 Oct 2023 22:25:48 +0530
Subject: [PATCH 135/259] fix(policies): remove non-existent policies, fix name
 (#9150)

---
 .../war/src/main/resources/boot/policies.json   | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json
index 18cb48bfcf1f0..b7ffc11c08f05 100644
--- a/metadata-service/war/src/main/resources/boot/policies.json
+++ b/metadata-service/war/src/main/resources/boot/policies.json
@@ -56,7 +56,7 @@
         "EDIT_ENTITY",
         "VIEW_ENTITY_PAGE",
         "EDIT_LINEAGE",
-        "EDIT_ENTITY_ASSERTIONS_PRIVILEGE",
+        "EDIT_ENTITY_ASSERTIONS",
         "SEARCH_PRIVILEGE",
         "GET_COUNTS_PRIVILEGE",
         "GET_TIMESERIES_ASPECT_PRIVILEGE",
@@ -251,11 +251,6 @@
         "EDIT_GROUP_MEMBERS",
         "EDIT_USER_PROFILE",
         "EDIT_CONTACT_INFO",
-        "MANAGE_ENTITY_TAGS_PRIVILEGE",
-        "MANAGE_ENTITY_GLOSSARY_TERMS_PRIVILEGE",
-        "MANAGE_DATASET_COL_GLOSSARY_TERMS_PRIVILEGE",
-        "MANAGE_DATASET_COL_TAGS_PRIVILEGE",
-        "EDIT_ENTITY_ASSERTIONS_PRIVILEGE",
         "EDIT_LINEAGE",
         "EDIT_ENTITY_QUERIES",
         "SEARCH_PRIVILEGE",
@@ -336,11 +331,6 @@
         "EDIT_GROUP_MEMBERS",
         "EDIT_USER_PROFILE",
         "EDIT_CONTACT_INFO",
-        "MANAGE_ENTITY_TAGS_PRIVILEGE",
-        "MANAGE_ENTITY_GLOSSARY_TERMS_PRIVILEGE",
-        "MANAGE_DATASET_COL_GLOSSARY_TERMS_PRIVILEGE",
-        "MANAGE_DATASET_COL_TAGS_PRIVILEGE",
-        "EDIT_ENTITY_ASSERTIONS_PRIVILEGE",
         "EDIT_LINEAGE",
         "EDIT_ENTITY_QUERIES",
         "SEARCH_PRIVILEGE",
@@ -441,11 +431,6 @@
         "EDIT_GROUP_MEMBERS",
         "EDIT_USER_PROFILE",
         "EDIT_CONTACT_INFO",
-        "MANAGE_ENTITY_TAGS_PRIVILEGE",
-        "MANAGE_ENTITY_GLOSSARY_TERMS_PRIVILEGE",
-        "MANAGE_DATASET_COL_GLOSSARY_TERMS_PRIVILEGE",
-        "MANAGE_DATASET_COL_TAGS_PRIVILEGE",
-        "EDIT_ENTITY_ASSERTIONS_PRIVILEGE",
         "EDIT_LINEAGE",
         "EDIT_ENTITY_QUERIES",
         "GET_TIMELINE_PRIVILEGE",

From b8dcc86281d06fcde35773fd4ef4933f5b553fd7 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:48:34 -0400
Subject: [PATCH 136/259] refactor(smoke): Fix for a test that passed on Oss
 and failed on Saas (#9147)

---
 .../cypress/cypress/e2e/lineage/download_lineage_results.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
index dc6efc9f7df66..ed4167b87c506 100644
--- a/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/download_lineage_results.js
@@ -37,7 +37,7 @@ describe("download lineage results to .csv file", () => {
       cy.openEntityTab("Lineage");
 
       // Verify 1st degree of dependencies
-      cy.contains(/1 - 3 of 3/);
+      cy.contains(/1 - [3-4] of [3-4]/);
       downloadCsvFile("first_degree_results.csv");
       let first_degree_csv = cy.readFile('cypress/downloads/first_degree_results.csv');
       first_degree.forEach(function (urn) {
@@ -52,7 +52,7 @@ describe("download lineage results to .csv file", () => {
 
       // Verify 1st and 2nd degree of dependencies
       cy.get('[data-testid="facet-degree-2"]').click().wait(5000);
-      cy.contains(/1 - 7 of 7/);
+      cy.contains(/1 - [7-8] of [7-8]/);
       downloadCsvFile("second_degree_results.csv");
       let second_degree_csv = cy.readFile('cypress/downloads/second_degree_results.csv');
       first_degree.forEach(function (urn) {
@@ -67,7 +67,7 @@ describe("download lineage results to .csv file", () => {
 
       // Verify 1st 2nd and 3+ degree of dependencies(Verify multi page download)
       cy.get('[data-testid="facet-degree-3+"]').click().wait(5000);
-      cy.contains(/1 - 10 of 13/);
+      cy.contains(/1 - 10 of 1[3-4]/);
       downloadCsvFile("third_plus_degree_results.csv");
       let third_degree_csv = cy.readFile('cypress/downloads/third_plus_degree_results.csv');
       first_degree.forEach(function (urn) {

From dae320c9bc28b80c6110395092d4223e9a37258b Mon Sep 17 00:00:00 2001
From: sachinsaju <33017477+sachinsaju@users.noreply.github.com>
Date: Wed, 1 Nov 2023 04:28:39 +0530
Subject: [PATCH 137/259] docs(teradata): teradata doc external link 404 fix
 (#9152)

---
 metadata-ingestion/docs/sources/teradata/teradata_pre.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/docs/sources/teradata/teradata_pre.md b/metadata-ingestion/docs/sources/teradata/teradata_pre.md
index 7263a59f5ea3d..7b4da1255d575 100644
--- a/metadata-ingestion/docs/sources/teradata/teradata_pre.md
+++ b/metadata-ingestion/docs/sources/teradata/teradata_pre.md
@@ -25,4 +25,4 @@ will fit for your queries (the default query text size Teradata captures is max
     REPLACE QUERY LOGGING LIMIT SQLTEXT=2000 ON ALL;
     ```
    See more here about query logging:
-      [https://docs.teradata.com/r/Teradata-VantageCloud-Lake/Database-Reference/Database-Administration/Tracking-Query-Behavior-with-Database-Query-Logging-Operational-DBAs]()
+      [https://docs.teradata.com/r/Teradata-VantageCloud-Lake/Database-Reference/Database-Administration/Tracking-Query-Behavior-with-Database-Query-Logging-Operational-DBAs](https://docs.teradata.com/r/Teradata-VantageCloud-Lake/Database-Reference/Database-Administration/Tracking-Query-Behavior-with-Database-Query-Logging-Operational-DBAs)

From 7a31950f794b2b0527ad685cbd08e967b524bfec Mon Sep 17 00:00:00 2001
From: "Jia (Jason) Teoh" <jiateoh@users.noreply.github.com>
Date: Tue, 31 Oct 2023 18:41:16 -0700
Subject: [PATCH 138/259] fix(datahub-client): Include relocation for snakeyaml
 dependency. (#8911)

Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
---
 metadata-integration/java/datahub-client/build.gradle | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle
index e6210f1f073f6..0bf6b18fa5073 100644
--- a/metadata-integration/java/datahub-client/build.gradle
+++ b/metadata-integration/java/datahub-client/build.gradle
@@ -97,6 +97,7 @@ shadowJar {
   // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first
   relocate 'org.springframework', 'datahub.shaded.org.springframework'
   relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson'
+  relocate 'org.yaml', 'io.acryl.shaded.org.yaml' // Required for shading snakeyaml
   relocate 'net.jcip.annotations', 'datahub.shaded.annotations'
   relocate 'javassist', 'datahub.shaded.javassist'
   relocate 'edu.umd.cs.findbugs', 'datahub.shaded.findbugs'
@@ -242,4 +243,4 @@ checkstyleMain.exclude '**/generated/**'
 
 clean {
   project.delete("$projectDir/generated")
-}
\ No newline at end of file
+}

From 73514ad9c5643cc5fbbb1edb0991d4aea0812459 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 31 Oct 2023 21:28:38 -0700
Subject: [PATCH 139/259] fix(ingest): cleanup large images in CI (#9153)

---
 .../tests/integration/sql_server/test_sql_server.py          | 5 ++++-
 metadata-ingestion/tests/integration/vertica/test_vertica.py | 5 ++++-
 metadata-ingestion/tests/test_helpers/docker_helpers.py      | 5 +++++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py
index 099690fed34c2..f439a322c2677 100644
--- a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py
+++ b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py
@@ -6,7 +6,7 @@
 
 from tests.test_helpers import mce_helpers
 from tests.test_helpers.click_helpers import run_datahub_cmd
-from tests.test_helpers.docker_helpers import wait_for_port
+from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port
 
 
 @pytest.fixture(scope="module")
@@ -29,6 +29,9 @@ def mssql_runner(docker_compose_runner, pytestconfig):
         assert ret.returncode == 0
         yield docker_services
 
+    # The image is pretty large, so we remove it after the test.
+    cleanup_image("mcr.microsoft.com/mssql/server")
+
 
 SOURCE_FILES_PATH = "./tests/integration/sql_server/source_files"
 config_file = os.listdir(SOURCE_FILES_PATH)
diff --git a/metadata-ingestion/tests/integration/vertica/test_vertica.py b/metadata-ingestion/tests/integration/vertica/test_vertica.py
index 94ad33ba21ce4..d7b4c390f75d9 100644
--- a/metadata-ingestion/tests/integration/vertica/test_vertica.py
+++ b/metadata-ingestion/tests/integration/vertica/test_vertica.py
@@ -6,7 +6,7 @@
 
 from tests.test_helpers import mce_helpers
 from tests.test_helpers.click_helpers import run_datahub_cmd
-from tests.test_helpers.docker_helpers import wait_for_port
+from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port
 
 FROZEN_TIME = "2020-04-14 07:00:00"
 
@@ -49,6 +49,9 @@ def vertica_runner(docker_compose_runner, test_resources_dir):
 
         yield docker_services
 
+    # The image is pretty large, so we remove it after the test.
+    cleanup_image("vertica/vertica-ce")
+
 
 @freeze_time(FROZEN_TIME)
 @pytest.mark.integration
diff --git a/metadata-ingestion/tests/test_helpers/docker_helpers.py b/metadata-ingestion/tests/test_helpers/docker_helpers.py
index 30157c3a78094..2eb61068196a2 100644
--- a/metadata-ingestion/tests/test_helpers/docker_helpers.py
+++ b/metadata-ingestion/tests/test_helpers/docker_helpers.py
@@ -1,5 +1,6 @@
 import contextlib
 import logging
+import os
 import subprocess
 from typing import Callable, Optional, Union
 
@@ -78,6 +79,10 @@ def run(
 def cleanup_image(image_name: str) -> None:
     assert ":" not in image_name, "image_name should not contain a tag"
 
+    if not os.environ.get("CI"):
+        logger.debug("Not cleaning up images to speed up local development")
+        return
+
     images_proc = subprocess.run(
         f"docker image ls --filter 'reference={image_name}*' -q",
         shell=True,

From d2314976033e42c13b7897f46ea0f227afb7c90b Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 31 Oct 2023 21:37:11 -0700
Subject: [PATCH 140/259] build: increase gradle retries (#9091)

---
 gradle.properties | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/gradle.properties b/gradle.properties
index 2b211e725359a..1cd349344b432 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -6,9 +6,16 @@ org.gradle.caching=false
 # Increase gradle JVM memory to 3GB to allow tests to run locally
 org.gradle.jvmargs=-Xmx3000m
 # Increase retries to 5 (from default of 3) and increase interval from 125ms to 1s.
+# Based on this thread https://github.com/gradle/gradle/issues/4629, it's unclear
+# if we should be using systemProp or not.  We're using both for now.
 org.gradle.internal.repository.max.retries=5
 org.gradle.internal.repository.max.tentatives=5
 org.gradle.internal.repository.initial.backoff=1000
+systemProp.org.gradle.internal.http.connectionTimeout=120000
+systemProp.org.gradle.internal.http.socketTimeout=120000
+systemProp.org.gradle.internal.repository.max.retries=5
+systemProp.org.gradle.internal.repository.max.tentatives=5
+systemProp.org.gradle.internal.repository.initial.backoff=1000
 
 # Needed to publish to Nexus from a sub-module
 gnsp.disableApplyOnlyOnRootProjectEnforcement=true

From 55f14530a397f75a9201db11c13bd7bbbb25162c Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Wed, 1 Nov 2023 00:12:52 -0700
Subject: [PATCH 141/259] feat(ingest): bump sqlglot parser (#9155)

---
 metadata-ingestion/setup.py                   |  2 +-
 .../unit/sql_parsing/test_sqlglot_lineage.py  | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 151842bd84d0a..afce8dcee840b 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -108,7 +108,7 @@
 sqlglot_lib = {
     # Using an Acryl fork of sqlglot.
     # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
-    "acryl-sqlglot==18.17.1.dev16",
+    "acryl-sqlglot==19.0.2.dev10",
 }
 
 sql_common = (
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index 3b9fa0d55f18d..c420f2b8438ce 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -676,9 +676,13 @@ def test_teradata_default_normalization():
 
 
 def test_teradata_strange_operators():
+    # This is a test for the following operators:
+    # - `SEL` (select)
+    # - `EQ` (equals)
+    # - `MINUS` (except)
     assert_sql_result(
         """
-select col1, col2 from dbc.table1
+sel col1, col2 from dbc.table1
 where col1 eq 'value1'
 minus
 select col1, col2 from dbc.table2
@@ -689,6 +693,19 @@ def test_teradata_strange_operators():
     )
 
 
+@pytest.mark.skip("sqlglot doesn't support this cast syntax yet")
+def test_teradata_cast_syntax():
+    assert_sql_result(
+        """
+SELECT my_table.date_col MONTH(4) AS month_col
+FROM my_table
+""",
+        dialect="teradata",
+        default_schema="dbc",
+        expected_file=RESOURCE_DIR / "test_teradata_cast_syntax.json",
+    )
+
+
 def test_snowflake_update_hardcoded():
     assert_sql_result(
         """

From 876de214c9a11f8928d8eafe5c7f658d5b9dc61f Mon Sep 17 00:00:00 2001
From: Tony Ouyang <tony@terratrue.com>
Date: Wed, 1 Nov 2023 00:13:17 -0700
Subject: [PATCH 142/259] feat(ingest/mongodb): support stateful ingestion
 (#9118)

---
 .../src/datahub/ingestion/source/mongodb.py   |   74 +-
 .../mongodb/mongodb_mces_golden.json          | 8320 +++++++++--------
 2 files changed, 4270 insertions(+), 4124 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
index 890c5c64bd5e6..ce2b9ce2981e0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
@@ -15,7 +15,12 @@
     EnvConfigMixin,
     PlatformInstanceConfigMixin,
 )
-from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
+from datahub.emitter.mce_builder import (
+    make_data_platform_urn,
+    make_dataplatform_instance_urn,
+    make_dataset_urn_with_platform_instance,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -25,14 +30,21 @@
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.schema_inference.object import (
     SchemaDescription,
     construct_schema,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
-from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+    StaleEntityRemovalSourceReport,
+    StatefulIngestionConfigBase,
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionSourceBase,
+)
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     ArrayTypeClass,
     BooleanTypeClass,
@@ -48,7 +60,10 @@
     TimeTypeClass,
     UnionTypeClass,
 )
-from datahub.metadata.schema_classes import DatasetPropertiesClass
+from datahub.metadata.schema_classes import (
+    DataPlatformInstanceClass,
+    DatasetPropertiesClass,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +74,9 @@
 DENY_DATABASE_LIST = set(["admin", "config", "local"])
 
 
-class MongoDBConfig(PlatformInstanceConfigMixin, EnvConfigMixin):
+class MongoDBConfig(
+    PlatformInstanceConfigMixin, EnvConfigMixin, StatefulIngestionConfigBase
+):
     # See the MongoDB authentication docs for details and examples.
     # https://pymongo.readthedocs.io/en/stable/examples/authentication.html
     connect_uri: str = Field(
@@ -99,6 +116,8 @@ class MongoDBConfig(PlatformInstanceConfigMixin, EnvConfigMixin):
         default=AllowDenyPattern.allow_all(),
         description="regex patterns for collections to filter in ingestion.",
     )
+    # Custom Stateful Ingestion settings
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
 
     @validator("maxDocumentSize")
     def check_max_doc_size_filter_is_valid(cls, doc_size_filter_value):
@@ -108,7 +127,7 @@ def check_max_doc_size_filter_is_valid(cls, doc_size_filter_value):
 
 
 @dataclass
-class MongoDBSourceReport(SourceReport):
+class MongoDBSourceReport(StaleEntityRemovalSourceReport):
     filtered: List[str] = field(default_factory=list)
 
     def report_dropped(self, name: str) -> None:
@@ -129,6 +148,7 @@ def report_dropped(self, name: str) -> None:
     bson.timestamp.Timestamp: "timestamp",
     bson.dbref.DBRef: "dbref",
     bson.objectid.ObjectId: "oid",
+    bson.Decimal128: "numberDecimal",
     "mixed": "mixed",
 }
 
@@ -145,6 +165,7 @@ def report_dropped(self, name: str) -> None:
     bson.timestamp.Timestamp: TimeTypeClass,
     bson.dbref.DBRef: BytesTypeClass,
     bson.objectid.ObjectId: BytesTypeClass,
+    bson.Decimal128: NumberTypeClass,
     dict: RecordTypeClass,
     "mixed": UnionTypeClass,
 }
@@ -206,7 +227,7 @@ def construct_schema_pymongo(
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @dataclass
-class MongoDBSource(Source):
+class MongoDBSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:
 
@@ -227,7 +248,7 @@ class MongoDBSource(Source):
     mongo_client: MongoClient
 
     def __init__(self, ctx: PipelineContext, config: MongoDBConfig):
-        super().__init__(ctx)
+        super().__init__(config, ctx)
         self.config = config
         self.report = MongoDBSourceReport()
 
@@ -254,6 +275,14 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> "MongoDBSource":
         config = MongoDBConfig.parse_obj(config_dict)
         return cls(ctx, config)
 
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
+        ]
+
     def get_pymongo_type_string(
         self, field_type: Union[Type, str], collection_name: str
     ) -> str:
@@ -332,16 +361,18 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     platform_instance=self.config.platform_instance,
                 )
 
-                dataset_snapshot = DatasetSnapshot(
-                    urn=dataset_urn,
-                    aspects=[],
-                )
+                if self.config.platform_instance:
+                    data_platform_instance = DataPlatformInstanceClass(
+                        platform=make_data_platform_urn(platform),
+                        instance=make_dataplatform_instance_urn(
+                            platform, self.config.platform_instance
+                        ),
+                    )
 
                 dataset_properties = DatasetPropertiesClass(
                     tags=[],
                     customProperties={},
                 )
-                dataset_snapshot.aspects.append(dataset_properties)
 
                 if self.config.enableSchemaInference:
                     assert self.config.maxDocumentSize is not None
@@ -412,13 +443,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                         fields=canonical_schema,
                     )
 
-                    dataset_snapshot.aspects.append(schema_metadata)
-
                 # TODO: use list_indexes() or index_information() to get index information
                 # See https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.list_indexes.
 
-                mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
-                yield MetadataWorkUnit(id=dataset_name, mce=mce)
+                yield from [
+                    mcp.as_workunit()
+                    for mcp in MetadataChangeProposalWrapper.construct_many(
+                        entityUrn=dataset_urn,
+                        aspects=[
+                            schema_metadata,
+                            dataset_properties,
+                            data_platform_instance,
+                        ],
+                    )
+                ]
 
     def is_server_version_gte_4_4(self) -> bool:
         try:
diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
index e16101b137ac9..ec3fd80e6a6ea 100644
--- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
+++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json
@@ -1,4136 +1,4240 @@
 [
 {
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {},
-                        "tags": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "emptyCollection",
-                        "platform": "urn:li:dataPlatform:mongodb",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.Schemaless": {}
-                        },
-                        "fields": []
-                    }
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "emptyCollection",
+            "platform": "urn:li:dataPlatform:mongodb",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.Schemaless": {}
+            },
+            "fields": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.emptyCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mongodb",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mongodb,instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "firstCollection",
+            "platform": "urn:li:dataPlatform:mongodb",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.Schemaless": {}
+            },
+            "fields": [
+                {
+                    "fieldPath": "_id",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BytesType": {}
+                        }
+                    },
+                    "nativeDataType": "oid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "age",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "canSwim",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BooleanType": {}
+                        }
+                    },
+                    "nativeDataType": "boolean",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "emptyObject",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.RecordType": {}
+                        }
+                    },
+                    "nativeDataType": "OBJECT",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteColor",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.RecordType": {}
+                        }
+                    },
+                    "nativeDataType": "OBJECT",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.calories",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.emptyObject",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.RecordType": {}
+                        }
+                    },
+                    "nativeDataType": "OBJECT",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.ingredients",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.ArrayType": {}
+                        }
+                    },
+                    "nativeDataType": "ARRAY",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.ingredients.color",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.ingredients.from",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.ingredients.name",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.name",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "favoriteFood.servings",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "legs",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "mixedType",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.UnionType": {}
+                        }
+                    },
+                    "nativeDataType": "mixed",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "seen",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "servings",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "sometimesNull",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "tags",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.ArrayType": {}
+                        }
+                    },
+                    "nativeDataType": "ARRAY",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "type",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mongodb",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mongodb,instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "largeCollection",
+            "platform": "urn:li:dataPlatform:mongodb",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.Schemaless": {}
+            },
+            "fields": [
+                {
+                    "fieldPath": "_id",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BytesType": {}
+                        }
+                    },
+                    "nativeDataType": "oid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_200",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_201",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_202",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_203",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_204",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_205",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_206",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_207",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_208",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_209",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_210",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_211",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_212",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_213",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_214",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_215",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_216",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_217",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_218",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_219",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_220",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_221",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_222",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_223",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_224",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_225",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_226",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_227",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_228",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_229",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_230",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_231",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_232",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_233",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_234",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_235",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_236",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_237",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_238",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_239",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_240",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_241",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_242",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_243",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_244",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_245",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_246",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_247",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_248",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_249",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_250",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_251",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_252",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_253",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_254",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_255",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_256",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_257",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_258",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_259",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_260",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_261",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_262",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_263",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_264",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_265",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_266",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_267",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_268",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_269",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_270",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_271",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_272",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_273",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_274",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_275",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_276",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_277",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_278",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_279",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_280",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_281",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_282",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_283",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_284",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_285",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_286",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_287",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_288",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_289",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_290",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_291",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_292",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_293",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_294",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_295",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_296",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_297",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_298",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_299",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_300",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_301",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_302",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_303",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_304",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_305",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_306",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_307",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_308",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_309",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_310",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_311",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_312",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_313",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_314",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_315",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_316",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_317",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_318",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_319",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_320",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_321",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_322",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_323",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_324",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_325",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_326",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_327",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_328",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_329",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_330",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_331",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_332",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_333",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_334",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_335",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_336",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_337",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_338",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_339",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_340",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_341",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_342",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_343",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_344",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_345",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_346",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_347",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_348",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_349",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_350",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_351",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_352",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_353",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_354",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_355",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_356",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_357",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_358",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_359",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_360",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_361",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_362",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_363",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_364",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_365",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_366",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_367",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_368",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_369",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_370",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_371",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_372",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_374",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_375",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_376",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_377",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_378",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_379",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_380",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_381",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_382",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_383",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_384",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_385",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_386",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_387",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_388",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_389",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_390",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_391",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_392",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_393",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_394",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_395",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_396",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_397",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_398",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_399",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_400",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_401",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_402",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_403",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_404",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_405",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_406",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_407",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_408",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_409",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_410",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_411",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_412",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_413",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_414",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_415",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_416",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_417",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_418",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_419",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_420",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_421",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_422",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_423",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_424",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_425",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_426",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_427",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_428",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_429",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_430",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_431",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_432",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_433",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_434",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_435",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_436",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_437",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_438",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_439",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_440",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_441",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_442",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_443",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_444",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_445",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_446",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_447",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_448",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_449",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_450",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_451",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_452",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_453",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_454",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_455",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_456",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_457",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_458",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_459",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_460",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_461",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_462",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_463",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_464",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_465",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_466",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_467",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_468",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_469",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_470",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_471",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_472",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_473",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_474",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_475",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_476",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_477",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_478",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_479",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_480",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_481",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_482",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_483",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_484",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_485",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_486",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_487",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_488",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_489",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_490",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_491",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_492",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_493",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_494",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_495",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_496",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_497",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_498",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "field_499",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.firstCollection,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {},
-                        "tags": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "firstCollection",
-                        "platform": "urn:li:dataPlatform:mongodb",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.Schemaless": {}
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "_id",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BytesType": {}
-                                    }
-                                },
-                                "nativeDataType": "oid",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "age",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "float",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "canSwim",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
-                                    }
-                                },
-                                "nativeDataType": "boolean",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "emptyObject",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
-                                    }
-                                },
-                                "nativeDataType": "OBJECT",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteColor",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
-                                    }
-                                },
-                                "nativeDataType": "OBJECT",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.calories",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "integer",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.emptyObject",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
-                                    }
-                                },
-                                "nativeDataType": "OBJECT",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.ingredients",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
-                                    }
-                                },
-                                "nativeDataType": "ARRAY",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.ingredients.color",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.ingredients.from",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.ingredients.name",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.name",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "favoriteFood.servings",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "float",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "legs",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "integer",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "mixedType",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.UnionType": {}
-                                    }
-                                },
-                                "nativeDataType": "mixed",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "name",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "seen",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "float",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "servings",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "integer",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "sometimesNull",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "tags",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
-                                    }
-                                },
-                                "nativeDataType": "ARRAY",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "type",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            }
-                        ]
-                    }
-                }
-            ]
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "schema.downsampled": "True",
+                "schema.totalFields": "501"
+            },
+            "tags": []
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "schema.downsampled": "True",
-                            "schema.totalFields": "501"
-                        },
-                        "tags": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "largeCollection",
-                        "platform": "urn:li:dataPlatform:mongodb",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.Schemaless": {}
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "_id",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BytesType": {}
-                                    }
-                                },
-                                "nativeDataType": "oid",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_200",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_201",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_202",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_203",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_204",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_205",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_206",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_207",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_208",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_209",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_210",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_211",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_212",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_213",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_214",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_215",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_216",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_217",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_218",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_219",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_220",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_221",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_222",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_223",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_224",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_225",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_226",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_227",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_228",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_229",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_230",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_231",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_232",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_233",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_234",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_235",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_236",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_237",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_238",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_239",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_240",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_241",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_242",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_243",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_244",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_245",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_246",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_247",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_248",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_249",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_250",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_251",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_252",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_253",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_254",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_255",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_256",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_257",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_258",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_259",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_260",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_261",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_262",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_263",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_264",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_265",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_266",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_267",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_268",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_269",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_270",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_271",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_272",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_273",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_274",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_275",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_276",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_277",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_278",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_279",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_280",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_281",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_282",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_283",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_284",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_285",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_286",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_287",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_288",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_289",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_290",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_291",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_292",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_293",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_294",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_295",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_296",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_297",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_298",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_299",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_300",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_301",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_302",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_303",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_304",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_305",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_306",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_307",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_308",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_309",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_310",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_311",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_312",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_313",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_314",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_315",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_316",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_317",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_318",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_319",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_320",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_321",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_322",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_323",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_324",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_325",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_326",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_327",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_328",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_329",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_330",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_331",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_332",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_333",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_334",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_335",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_336",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_337",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_338",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_339",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_340",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_341",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_342",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_343",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_344",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_345",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_346",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_347",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_348",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_349",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_350",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_351",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_352",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_353",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_354",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_355",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_356",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_357",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_358",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_359",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_360",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_361",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_362",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_363",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_364",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_365",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_366",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_367",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_368",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_369",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_370",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_371",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_372",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_374",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_375",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_376",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_377",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_378",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_379",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_380",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_381",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_382",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_383",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_384",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_385",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_386",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_387",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_388",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_389",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_390",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_391",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_392",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_393",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_394",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_395",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_396",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_397",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_398",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_399",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_400",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_401",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_402",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_403",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_404",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_405",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_406",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_407",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_408",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_409",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_410",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_411",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_412",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_413",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_414",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_415",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_416",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_417",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_418",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_419",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_420",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_421",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_422",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_423",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_424",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_425",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_426",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_427",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_428",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_429",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_430",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_431",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_432",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_433",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_434",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_435",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_436",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_437",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_438",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_439",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_440",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_441",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_442",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_443",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_444",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_445",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_446",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_447",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_448",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_449",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_450",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_451",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_452",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_453",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_454",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_455",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_456",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_457",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_458",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_459",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_460",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_461",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_462",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_463",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_464",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_465",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_466",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_467",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_468",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_469",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_470",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_471",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_472",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_473",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_474",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_475",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_476",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_477",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_478",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_479",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_480",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_481",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_482",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_483",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_484",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_485",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_486",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_487",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_488",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_489",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_490",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_491",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_492",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_493",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_494",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_495",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_496",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_497",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_498",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "field_499",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            }
-                        ]
-                    }
-                }
-            ]
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.largeCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mongodb",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mongodb,instance)"
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {},
-                        "tags": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "secondCollection",
-                        "platform": "urn:li:dataPlatform:mongodb",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.Schemaless": {}
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "_id",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BytesType": {}
-                                    }
-                                },
-                                "nativeDataType": "oid",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "mixedType",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.UnionType": {}
-                                    }
-                                },
-                                "nativeDataType": "mixed",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "mixedType.fieldA",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "mixedType.fieldTwo",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "integer",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "name",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "nullableMixedType",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.UnionType": {}
-                                    }
-                                },
-                                "nativeDataType": "mixed",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "rating",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "float",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "tasty",
-                                "nullable": false,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
-                                    }
-                                },
-                                "nativeDataType": "boolean",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "varieties",
-                                "nullable": true,
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
-                                    }
-                                },
-                                "nativeDataType": "ARRAY",
-                                "recursive": false,
-                                "isPartOfKey": false
-                            }
-                        ]
-                    }
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "schemaMetadata",
+    "aspect": {
+        "json": {
+            "schemaName": "secondCollection",
+            "platform": "urn:li:dataPlatform:mongodb",
+            "version": 0,
+            "created": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            },
+            "hash": "",
+            "platformSchema": {
+                "com.linkedin.schema.Schemaless": {}
+            },
+            "fields": [
+                {
+                    "fieldPath": "_id",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BytesType": {}
+                        }
+                    },
+                    "nativeDataType": "oid",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "mixedType",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.UnionType": {}
+                        }
+                    },
+                    "nativeDataType": "mixed",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "mixedType.fieldA",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "mixedType.fieldTwo",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "integer",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "name",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.StringType": {}
+                        }
+                    },
+                    "nativeDataType": "string",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "nullableMixedType",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.UnionType": {}
+                        }
+                    },
+                    "nativeDataType": "mixed",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "rating",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.NumberType": {}
+                        }
+                    },
+                    "nativeDataType": "float",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "tasty",
+                    "nullable": false,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.BooleanType": {}
+                        }
+                    },
+                    "nativeDataType": "boolean",
+                    "recursive": false,
+                    "isPartOfKey": false
+                },
+                {
+                    "fieldPath": "varieties",
+                    "nullable": true,
+                    "type": {
+                        "type": {
+                            "com.linkedin.schema.ArrayType": {}
+                        }
+                    },
+                    "nativeDataType": "ARRAY",
+                    "recursive": false,
+                    "isPartOfKey": false
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "datasetProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,instance.mngdb.secondCollection,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:mongodb",
+            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:mongodb,instance)"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4145,7 +4249,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4160,7 +4265,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4175,7 +4281,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -4190,7 +4297,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "mongodb-test"
+        "runId": "mongodb-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file

From f2eb0cf3073967d505004e9f2df58d5475cadec1 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Wed, 1 Nov 2023 15:41:02 -0400
Subject: [PATCH 143/259] smoke-test: API test for managing secrets privilege
 (#9121)

---
 .../tests/privileges/test_privileges.py       | 241 ++++++++++++++++++
 smoke-test/tests/privileges/utils.py          | 218 ++++++++++++++++
 2 files changed, 459 insertions(+)
 create mode 100644 smoke-test/tests/privileges/test_privileges.py
 create mode 100644 smoke-test/tests/privileges/utils.py

diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py
new file mode 100644
index 0000000000000..13d6b6cf3415a
--- /dev/null
+++ b/smoke-test/tests/privileges/test_privileges.py
@@ -0,0 +1,241 @@
+import pytest
+import tenacity
+
+from tests.utils import (get_frontend_session, wait_for_writes_to_sync, wait_for_healthcheck_util,
+                        get_frontend_url, get_admin_credentials,get_sleep_info)
+from tests.privileges.utils import *
+
+sleep_sec, sleep_times = get_sleep_info()
+
+@pytest.fixture(scope="session")
+def wait_for_healthchecks():
+    wait_for_healthcheck_util()
+    yield
+
+
+@pytest.mark.dependency()
+def test_healthchecks(wait_for_healthchecks):
+    # Call to wait_for_healthchecks fixture will do the actual functionality.
+    pass
+
+
+@pytest.fixture(scope="session")
+def admin_session(wait_for_healthchecks):
+    yield get_frontend_session()
+
+
+@pytest.mark.dependency(depends=["test_healthchecks"])
+@pytest.fixture(scope="module", autouse=True)
+def privileges_and_test_user_setup(admin_session):
+    """Fixture to execute setup before and tear down after all tests are run"""
+    # Disable 'All users' privileges
+    set_base_platform_privileges_policy_status("INACTIVE", admin_session)
+    set_view_dataset_sensitive_info_policy_status("INACTIVE", admin_session)
+    set_view_entity_profile_privileges_policy_status("INACTIVE", admin_session)
+    # Sleep for eventual consistency
+    wait_for_writes_to_sync()
+
+    # Create a new user   
+    admin_session = create_user(admin_session, "user", "user")
+
+    yield
+
+    # Remove test user
+    remove_user(admin_session, "urn:li:corpuser:user")
+
+    # Restore All users privileges
+    set_base_platform_privileges_policy_status("ACTIVE", admin_session)
+    set_view_dataset_sensitive_info_policy_status("ACTIVE", admin_session)
+    set_view_entity_profile_privileges_policy_status("ACTIVE", admin_session)
+
+    # Sleep for eventual consistency
+    wait_for_writes_to_sync()
+
+
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(sleep_sec)
+)
+def _ensure_can_create_secret(session, json, urn):
+    create_secret_success = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=json)
+    create_secret_success.raise_for_status()
+    secret_data = create_secret_success.json()
+
+    assert secret_data
+    assert secret_data["data"]
+    assert secret_data["data"]["createSecret"]
+    assert secret_data["data"]["createSecret"] == urn
+    
+
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
+)
+def _ensure_cant_create_secret(session, json):
+    create_secret_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=json)
+    create_secret_response.raise_for_status()
+    create_secret_data = create_secret_response.json()
+
+    assert create_secret_data["errors"][0]["extensions"]["code"] == 403
+    assert create_secret_data["errors"][0]["extensions"]["type"] == "UNAUTHORIZED"
+    assert create_secret_data["data"]["createSecret"] == None
+
+
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(sleep_sec)
+)
+def _ensure_can_create_ingestion_source(session, json):
+    create_ingestion_success = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=json)
+    create_ingestion_success.raise_for_status()
+    ingestion_data = create_ingestion_success.json()
+
+    assert ingestion_data
+    assert ingestion_data["data"]
+    assert ingestion_data["data"]["createIngestionSource"]
+    assert ingestion_data["data"]["createIngestionSource"] is not None
+
+    return ingestion_data["data"]["createIngestionSource"]
+    
+
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
+)
+def _ensure_cant_create_ingestion_source(session, json):
+    create_source_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=json)
+    create_source_response.raise_for_status()
+    create_source_data = create_source_response.json()
+
+    assert create_source_data["errors"][0]["extensions"]["code"] == 403
+    assert create_source_data["errors"][0]["extensions"]["type"] == "UNAUTHORIZED"
+    assert create_source_data["data"]["createIngestionSource"] == None
+
+
+@pytest.mark.dependency(depends=["test_healthchecks"])
+def test_privilege_to_create_and_manage_secrets():
+
+    (admin_user, admin_pass) = get_admin_credentials()
+    admin_session = login_as(admin_user, admin_pass)
+    user_session = login_as("user", "user")
+    secret_urn = "urn:li:dataHubSecret:TestSecretName"
+
+    # Verify new user can't create secrets
+    create_secret = { 
+        "query": """mutation createSecret($input: CreateSecretInput!) {\n
+            createSecret(input: $input)\n}""",
+        "variables": {
+            "input":{
+                "name":"TestSecretName",
+                "value":"Test Secret Value",
+                "description":"Test Secret Description"
+                }
+        },
+    }
+    _ensure_cant_create_secret(user_session, create_secret)
+
+
+    # Assign privileges to the new user to manage secrets
+    policy_urn = create_user_policy("urn:li:corpuser:user", ["MANAGE_SECRETS"], admin_session)
+
+    # Verify new user can create and manage secrets
+    # Create a secret
+    _ensure_can_create_secret(user_session, create_secret, secret_urn)
+
+
+    # Remove a secret
+    remove_secret = { 
+        "query": """mutation deleteSecret($urn: String!) {\n
+            deleteSecret(urn: $urn)\n}""",
+        "variables": {
+            "urn": secret_urn
+        },
+    }
+
+    remove_secret_response = user_session.post(f"{get_frontend_url()}/api/v2/graphql", json=remove_secret)
+    remove_secret_response.raise_for_status()
+    secret_data = remove_secret_response.json()
+
+    assert secret_data
+    assert secret_data["data"]
+    assert secret_data["data"]["deleteSecret"]
+    assert secret_data["data"]["deleteSecret"] == secret_urn
+
+
+    # Remove the policy
+    remove_policy(policy_urn, admin_session)
+
+    # Ensure user can't create secret after policy is removed
+    _ensure_cant_create_secret(user_session, create_secret)
+
+
+@pytest.mark.dependency(depends=["test_healthchecks"])
+def test_privilege_to_create_and_manage_ingestion_source():
+
+    (admin_user, admin_pass) = get_admin_credentials()
+    admin_session = login_as(admin_user, admin_pass)
+    user_session = login_as("user", "user")
+
+    # Verify new user can't create ingestion source
+    create_ingestion_source = { 
+        "query": """mutation createIngestionSource($input: UpdateIngestionSourceInput!) {\n
+            createIngestionSource(input: $input)\n}""",
+        "variables": {"input":{"type":"snowflake","name":"test","config":
+            {"recipe":
+                "{\"source\":{\"type\":\"snowflake\",\"config\":{\"account_id\":null,\"include_table_lineage\":true,\"include_view_lineage\":true,\"include_tables\":true,\"include_views\":true,\"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},\"stateful_ingestion\":{\"enabled\":true}}}}",
+                "executorId":"default","debugMode":False,"extraArgs":[]}}},
+    }
+
+    _ensure_cant_create_ingestion_source(user_session, create_ingestion_source)
+
+
+     # Assign privileges to the new user to manage ingestion source
+    policy_urn = create_user_policy("urn:li:corpuser:user", ["MANAGE_INGESTION"], admin_session)
+   
+    # Verify new user can create and manage ingestion source(edit, delete)
+    ingestion_source_urn = _ensure_can_create_ingestion_source(user_session, create_ingestion_source)
+
+    # Edit ingestion source
+    update_ingestion_source = { 
+        "query": """mutation updateIngestionSource($urn: String!, $input: UpdateIngestionSourceInput!) {\n
+            updateIngestionSource(urn: $urn, input: $input)\n}""",
+        "variables": {"urn":ingestion_source_urn,
+              "input":{"type":"snowflake","name":"test updated",
+                       "config":{"recipe":"{\"source\":{\"type\":\"snowflake\",\"config\":{\"account_id\":null,\"include_table_lineage\":true,\"include_view_lineage\":true,\"include_tables\":true,\"include_views\":true,\"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},\"stateful_ingestion\":{\"enabled\":true}}}}",
+                        "executorId":"default","debugMode":False,"extraArgs":[]}}}
+    }
+
+    update_ingestion_success = user_session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=update_ingestion_source)
+    update_ingestion_success.raise_for_status()
+    ingestion_data = update_ingestion_success.json()
+
+    assert ingestion_data
+    assert ingestion_data["data"]
+    assert ingestion_data["data"]["updateIngestionSource"]
+    assert ingestion_data["data"]["updateIngestionSource"] == ingestion_source_urn
+
+
+    # Delete ingestion source
+    remove_ingestion_source = { 
+        "query": """mutation deleteIngestionSource($urn: String!) {\n
+            deleteIngestionSource(urn: $urn)\n}""",
+        "variables": {
+            "urn": ingestion_source_urn
+        },
+    }
+
+    remove_ingestion_response = user_session.post(f"{get_frontend_url()}/api/v2/graphql", json=remove_ingestion_source)
+    remove_ingestion_response.raise_for_status()
+    ingestion_data = remove_ingestion_response.json()
+
+    assert ingestion_data
+    assert ingestion_data["data"]
+    assert ingestion_data["data"]["deleteIngestionSource"]
+    assert ingestion_data["data"]["deleteIngestionSource"] == ingestion_source_urn
+
+    # Remove the policy
+    remove_policy(policy_urn, admin_session)
+
+    # Ensure that user can't create ingestion source after policy is removed
+    _ensure_cant_create_ingestion_source(user_session, create_ingestion_source)
\ No newline at end of file
diff --git a/smoke-test/tests/privileges/utils.py b/smoke-test/tests/privileges/utils.py
new file mode 100644
index 0000000000000..ea1f565f6f5ac
--- /dev/null
+++ b/smoke-test/tests/privileges/utils.py
@@ -0,0 +1,218 @@
+import requests_wrapper as requests
+from tests.consistency_utils import wait_for_writes_to_sync
+from tests.utils import (get_frontend_url, wait_for_writes_to_sync, get_admin_credentials)
+
+
+def set_base_platform_privileges_policy_status(status, session):
+    base_platform_privileges = { 
+        "query": """mutation updatePolicy($urn: String!, $input: PolicyUpdateInput!) {\n
+            updatePolicy(urn: $urn, input: $input) }""",
+        "variables": {
+            "urn": "urn:li:dataHubPolicy:7",
+            "input": {
+                "type": "PLATFORM",
+                "state": status,
+                "name": "All Users - Base Platform Privileges",
+                "description": "Grants base platform privileges to ALL users of DataHub. Change this policy to alter that behavior.",
+                "privileges": ["MANAGE_INGESTION",
+                               "MANAGE_SECRETS",
+                               "MANAGE_USERS_AND_GROUPS",
+                               "VIEW_ANALYTICS",
+                               "GENERATE_PERSONAL_ACCESS_TOKENS",
+                               "MANAGE_DOMAINS",
+                               "MANAGE_GLOBAL_ANNOUNCEMENTS",
+                               "MANAGE_TESTS",
+                               "MANAGE_GLOSSARIES",
+                               "MANAGE_TAGS",
+                               "MANAGE_GLOBAL_VIEWS",
+                               "MANAGE_GLOBAL_OWNERSHIP_TYPES"],
+                "actors": {
+                    "users": [],
+                    "groups": None,
+                    "resourceOwners": False,
+                    "allUsers": True,
+                    "allGroups": False,
+                    "resourceOwnersTypes": None,
+                },
+            },
+        },
+    }
+    base_privileges_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=base_platform_privileges)
+    base_privileges_response.raise_for_status()
+    base_res_data = base_privileges_response.json()
+    assert base_res_data["data"]["updatePolicy"] == "urn:li:dataHubPolicy:7"
+
+def set_view_dataset_sensitive_info_policy_status(status, session):
+    dataset_sensitive_information = { 
+        "query": """mutation updatePolicy($urn: String!, $input: PolicyUpdateInput!) {\n
+            updatePolicy(urn: $urn, input: $input) }""",
+        "variables": {
+            "urn": "urn:li:dataHubPolicy:view-dataset-sensitive",
+            "input": {
+                "type": "METADATA",
+                "state": status,
+                "name": "All Users - View Dataset Sensitive Information",
+                "description": "Grants viewing privileges of usage and profile information of all datasets for all users",
+                "privileges": ["VIEW_DATASET_USAGE","VIEW_DATASET_PROFILE"],
+                "actors": {
+                    "users": [],
+                    "groups": None,
+                    "resourceOwners": False,
+                    "allUsers": True,
+                    "allGroups": False,
+                    "resourceOwnersTypes": None,
+                },
+            },
+        },
+    }
+    sensitive_info_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=dataset_sensitive_information)
+    sensitive_info_response.raise_for_status()
+    sens_info_data = sensitive_info_response.json()
+    assert sens_info_data["data"]["updatePolicy"] == "urn:li:dataHubPolicy:view-dataset-sensitive"
+
+def set_view_entity_profile_privileges_policy_status(status, session):
+    view_entity_page = { 
+        "query": """mutation updatePolicy($urn: String!, $input: PolicyUpdateInput!) {\n
+            updatePolicy(urn: $urn, input: $input) }""",
+        "variables": {
+            "urn": "urn:li:dataHubPolicy:view-entity-page-all",
+            "input": {
+                "type": "METADATA",
+                "state": status,
+                "name": "All Users - View Entity Page",
+                "description": "Grants entity view to all users",
+                "privileges": ["VIEW_ENTITY_PAGE",
+                               "SEARCH_PRIVILEGE",
+                               "GET_COUNTS_PRIVILEGE",
+                               "GET_TIMESERIES_ASPECT_PRIVILEGE",
+                               "GET_ENTITY_PRIVILEGE",
+                               "GET_TIMELINE_PRIVILEGE"],
+                "actors": {
+                    "users": [],
+                    "groups": None,
+                    "resourceOwners": False,
+                    "allUsers": True,
+                    "allGroups": False,
+                    "resourceOwnersTypes": None,
+                },
+            },
+        },
+    }
+    view_entity_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=view_entity_page)
+    view_entity_response.raise_for_status()
+    view_entity_data = view_entity_response.json()
+    assert view_entity_data["data"]["updatePolicy"] == "urn:li:dataHubPolicy:view-entity-page-all"
+
+def create_user(session, email, password):
+    # Remove user if exists
+    res_data = remove_user(session, f"urn:li:corpuser:{email}")
+    assert res_data
+    assert "error" not in res_data
+    # Get the invite token
+    get_invite_token_json = {
+        "query": """query getInviteToken($input: GetInviteTokenInput!) {\n
+            getInviteToken(input: $input){\n
+              inviteToken\n
+            }\n
+        }""",
+        "variables": {"input": {}},
+    }
+    get_invite_token_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=get_invite_token_json
+    )
+    get_invite_token_response.raise_for_status()
+    get_invite_token_res_data = get_invite_token_response.json()
+    invite_token = get_invite_token_res_data["data"]["getInviteToken"]["inviteToken"]
+    assert invite_token is not None
+    assert "error" not in invite_token
+    # Create a new user using the invite token
+    sign_up_json = {
+        "fullName": "Test User",
+        "email": email,
+        "password": password,
+        "title": "Data Engineer",
+        "inviteToken": invite_token,
+    }
+    sign_up_response = session.post(
+        f"{get_frontend_url()}/signUp", json=sign_up_json
+    )
+    sign_up_response.raise_for_status()
+    assert sign_up_response
+    assert "error" not in sign_up_response
+    wait_for_writes_to_sync()
+    session.cookies.clear()
+    (admin_user, admin_pass) = get_admin_credentials()
+    admin_session = login_as(admin_user, admin_pass)
+    return admin_session
+
+
+def login_as(username, password):
+    session = requests.Session()
+    headers = {
+        "Content-Type": "application/json",
+    }
+    data = '{"username":"' + username + '", "password":"' + password + '"}'
+    response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data)
+    response.raise_for_status()
+    return session
+
+def remove_user(session, urn):
+    json = {
+        "query": """mutation removeUser($urn: String!) {\n
+            removeUser(urn: $urn)
+        }""",
+        "variables": {"urn": urn},
+    }
+    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
+    response.raise_for_status()
+    return response.json()
+
+def create_user_policy(user_urn, privileges, session):
+    policy = {
+        "query": """mutation createPolicy($input: PolicyUpdateInput!) {\n
+            createPolicy(input: $input) }""",
+        "variables": {
+            "input": {
+                "type": "PLATFORM",
+                "name": "Policy Name",
+                "description": "Policy Description",
+                "state": "ACTIVE",
+                "resources": {"filter":{"criteria":[]}},
+                "privileges": privileges,
+                "actors": {
+                    "users": [user_urn],
+                    "resourceOwners": False,
+                    "allUsers": False,
+                    "allGroups": False,
+                },
+            }
+        },
+    }
+
+    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=policy)
+    response.raise_for_status()
+    res_data = response.json()
+
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["createPolicy"]
+    return res_data["data"]["createPolicy"]
+
+def remove_policy(urn, session):
+    remove_policy_json = {
+        "query": """mutation deletePolicy($urn: String!) {\n
+            deletePolicy(urn: $urn) }""",
+        "variables": {"urn": urn},
+    }
+
+    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=remove_policy_json)
+    response.raise_for_status()
+    res_data = response.json()
+
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["deletePolicy"]
+    assert res_data["data"]["deletePolicy"] == urn
\ No newline at end of file

From 95d9ff2cc2b71c5062454f6da1eca5084d6dd6eb Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Thu, 2 Nov 2023 01:44:16 +0530
Subject: [PATCH 144/259] fix(ingest): handle exceptions in min, max, mean
 profiling (#9129)

---
 .../ingestion/source/ge_data_profiler.py      | 36 +++++++++++++++++--
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
index 9f6ac9dd21164..6b97d2eb456da 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
@@ -406,22 +406,52 @@ def _get_dataset_rows(self, dataset_profile: DatasetProfileClass) -> None:
     def _get_dataset_column_min(
         self, column_profile: DatasetFieldProfileClass, column: str
     ) -> None:
-        if self.config.include_field_min_value:
+        if not self.config.include_field_min_value:
+            return
+        try:
             column_profile.min = str(self.dataset.get_column_min(column))
+        except Exception as e:
+            logger.debug(
+                f"Caught exception while attempting to get column min for column {column}. {e}"
+            )
+            self.report.report_warning(
+                "Profiling - Unable to get column min",
+                f"{self.dataset_name}.{column}",
+            )
 
     @_run_with_query_combiner
     def _get_dataset_column_max(
         self, column_profile: DatasetFieldProfileClass, column: str
     ) -> None:
-        if self.config.include_field_max_value:
+        if not self.config.include_field_max_value:
+            return
+        try:
             column_profile.max = str(self.dataset.get_column_max(column))
+        except Exception as e:
+            logger.debug(
+                f"Caught exception while attempting to get column max for column {column}. {e}"
+            )
+            self.report.report_warning(
+                "Profiling - Unable to get column max",
+                f"{self.dataset_name}.{column}",
+            )
 
     @_run_with_query_combiner
     def _get_dataset_column_mean(
         self, column_profile: DatasetFieldProfileClass, column: str
     ) -> None:
-        if self.config.include_field_mean_value:
+        if not self.config.include_field_mean_value:
+            return
+        try:
             column_profile.mean = str(self.dataset.get_column_mean(column))
+        except Exception as e:
+            logger.debug(
+                f"Caught exception while attempting to get column mean for column {column}. {e}"
+            )
+            self.report.report_warning(
+                "Profiling - Unable to get column mean",
+                f"{self.dataset_name}.{column}",
+            )
 
     @_run_with_query_combiner
     def _get_dataset_column_median(

From 932eebea353cf6f31bc489428feb54b43d647075 Mon Sep 17 00:00:00 2001
From: kushagra-apptware <81357546+kushagra-apptware@users.noreply.github.com>
Date: Thu, 2 Nov 2023 01:50:15 +0530
Subject: [PATCH 145/259] feat: rename Assets tab to Owner Of (#9141)

Co-authored-by: John Joyce <john@acryl.io>
---
 datahub-web-react/src/app/entity/group/GroupProfile.tsx | 2 +-
 datahub-web-react/src/app/entity/user/UserProfile.tsx   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/datahub-web-react/src/app/entity/group/GroupProfile.tsx b/datahub-web-react/src/app/entity/group/GroupProfile.tsx
index 53d2062277dec..11ed31e00003f 100644
--- a/datahub-web-react/src/app/entity/group/GroupProfile.tsx
+++ b/datahub-web-react/src/app/entity/group/GroupProfile.tsx
@@ -16,7 +16,7 @@ import NonExistentEntityPage from '../shared/entity/NonExistentEntityPage';
 const messageStyle = { marginTop: '10%' };
 
 export enum TabType {
-    Assets = 'Assets',
+    Assets = 'Owner Of',
     Members = 'Members',
 }
 
diff --git a/datahub-web-react/src/app/entity/user/UserProfile.tsx b/datahub-web-react/src/app/entity/user/UserProfile.tsx
index 1d20072c4ea8f..e8284ba61afe4 100644
--- a/datahub-web-react/src/app/entity/user/UserProfile.tsx
+++ b/datahub-web-react/src/app/entity/user/UserProfile.tsx
@@ -17,7 +17,7 @@ export interface Props {
 }
 
 export enum TabType {
-    Assets = 'Assets',
+    Assets = 'Owner Of',
     Groups = 'Groups',
 }
 const ENABLED_TAB_TYPES = [TabType.Assets, TabType.Groups];

From 50789224a12e0f48d6b4ca2ef3876498f7738d9e Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Wed, 1 Nov 2023 16:58:37 -0700
Subject: [PATCH 146/259] fix(ingest/mongodb): fix schema inference for lists
 of values (#9145)

---
 .../datahub/ingestion/source/schema_inference/object.py    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/object.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/object.py
index 5797d66aa4d19..b58bdf41ccaa5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/object.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/object.py
@@ -16,7 +16,7 @@ class SchemaDescription(BasicSchemaDescription):
     nullable: bool  # if field is ever missing
 
 
-def is_field_nullable(doc: Dict[str, Any], field_path: Tuple) -> bool:
+def is_field_nullable(doc: Dict[str, Any], field_path: Tuple[str, ...]) -> bool:
     """
     Check if a nested field is nullable in a document from a collection.
 
@@ -54,7 +54,10 @@ def is_field_nullable(doc: Dict[str, Any], field_path: Tuple) -> bool:
             # count empty lists of nested objects as nullable
             if len(value) == 0:
                 return True
-            return any(is_field_nullable(x, remaining_fields) for x in doc[field])
+            return any(
+                isinstance(x, dict) and is_field_nullable(x, remaining_fields)
+                for x in doc[field]
+            )
 
         # any other types to check?
         # raise ValueError("Nested type not 'list' or 'dict' encountered")

From f7cd80283ad768afe14e3cf53b9c38fe912be570 Mon Sep 17 00:00:00 2001
From: deepgarg-visa <149145061+deepgarg-visa@users.noreply.github.com>
Date: Thu, 2 Nov 2023 09:16:58 +0530
Subject: [PATCH 147/259] fix(ingest/db2): fix handling for table properties
 (#9128)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../src/datahub/ingestion/source/sql/sql_common.py             | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
index 51909eaf4ed55..80f828e9ea2fd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -20,6 +20,7 @@
 import sqlalchemy.dialects.postgresql.base
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.engine.reflection import Inspector
+from sqlalchemy.engine.row import LegacyRow
 from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.sql import sqltypes as types
 from sqlalchemy.types import TypeDecorator, TypeEngine
@@ -784,7 +785,7 @@ def get_table_properties(
             table_info: dict = inspector.get_table_comment(table, f'"{schema}"')  # type: ignore
 
         description = table_info.get("text")
-        if type(description) is tuple:
+        if isinstance(description, LegacyRow):
             # Handling for value type tuple which is coming for dialect 'db2+ibm_db'
             description = table_info["text"][0]
 

From bab9d1c93196b7a7181525609e854640c9a13712 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Wed, 1 Nov 2023 20:47:18 -0700
Subject: [PATCH 148/259] fix(ingest): fully support MCPs in urn_iter primitive
 (#9157)

---
 .../src/datahub/utilities/urns/urn_iter.py    | 16 +++++++-------
 .../tests/unit/serde/test_urn_iterator.py     | 21 +++++++++++++------
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py b/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
index e13d439161064..169a4ac3649a3 100644
--- a/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
+++ b/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
@@ -117,17 +117,17 @@ def _modify_at_path(
         if isinstance(path[0], int):
             assert isinstance(model, list)
             model[path[0]] = new_value
-        elif isinstance(model, MetadataChangeProposalWrapper):
-            setattr(model, path[0], new_value)
-        else:
-            assert isinstance(model, DictWrapper)
+        elif isinstance(model, DictWrapper):
             model._inner_dict[path[0]] = new_value
+        else:  # MCPW
+            setattr(model, path[0], new_value)
     elif isinstance(path[0], int):
         assert isinstance(model, list)
-        return _modify_at_path(model[path[0]], path[1:], new_value)
-    else:
-        assert isinstance(model, DictWrapper)
-        return _modify_at_path(model._inner_dict[path[0]], path[1:], new_value)
+        _modify_at_path(model[path[0]], path[1:], new_value)
+    elif isinstance(model, DictWrapper):
+        _modify_at_path(model._inner_dict[path[0]], path[1:], new_value)
+    else:  # MCPW
+        _modify_at_path(getattr(model, path[0]), path[1:], new_value)
 
 
 def _lowercase_dataset_urn(dataset_urn: str) -> str:
diff --git a/metadata-ingestion/tests/unit/serde/test_urn_iterator.py b/metadata-ingestion/tests/unit/serde/test_urn_iterator.py
index 9657ac45068da..135580dcdff13 100644
--- a/metadata-ingestion/tests/unit/serde/test_urn_iterator.py
+++ b/metadata-ingestion/tests/unit/serde/test_urn_iterator.py
@@ -1,4 +1,5 @@
 import datahub.emitter.mce_builder as builder
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageTypeClass,
     FineGrainedLineage,
@@ -10,11 +11,11 @@
 from datahub.utilities.urns.urn_iter import list_urns_with_path, lowercase_dataset_urns
 
 
-def _datasetUrn(tbl):
+def _datasetUrn(tbl: str) -> str:
     return builder.make_dataset_urn("bigquery", tbl, "PROD")
 
 
-def _fldUrn(tbl, fld):
+def _fldUrn(tbl: str, fld: str) -> str:
     return builder.make_schema_field_urn(_datasetUrn(tbl), fld)
 
 
@@ -114,8 +115,10 @@ def test_upstream_lineage_urn_iterator():
     ]
 
 
-def _make_test_lineage_obj(upstream: str, downstream: str) -> UpstreamLineage:
-    return UpstreamLineage(
+def _make_test_lineage_obj(
+    table: str, upstream: str, downstream: str
+) -> MetadataChangeProposalWrapper:
+    lineage = UpstreamLineage(
         upstreams=[
             Upstream(
                 dataset=_datasetUrn(upstream),
@@ -132,11 +135,17 @@ def _make_test_lineage_obj(upstream: str, downstream: str) -> UpstreamLineage:
         ],
     )
 
+    return MetadataChangeProposalWrapper(entityUrn=_datasetUrn(table), aspect=lineage)
+
 
 def test_dataset_urn_lowercase_transformer():
-    original = _make_test_lineage_obj("upstreamTable", "downstreamTable")
+    original = _make_test_lineage_obj(
+        "mainTableName", "upstreamTable", "downstreamTable"
+    )
 
-    expected = _make_test_lineage_obj("upstreamtable", "downstreamtable")
+    expected = _make_test_lineage_obj(
+        "maintablename", "upstreamtable", "downstreamtable"
+    )
 
     assert original != expected  # sanity check
 

From 12b41713b46ab474f0d55ea81fe0e854526036a9 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Thu, 2 Nov 2023 10:05:24 +0530
Subject: [PATCH 149/259] =?UTF-8?q?fix(ingest/bigquery):=20use=20correct?=
 =?UTF-8?q?=20row=20count=20in=20null=20count=20profiling=20c=E2=80=A6=20(?=
 =?UTF-8?q?#9123)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
Co-authored-by: Aseem Bansal <asmbansal2@gmail.com>
---
 .../datahub/ingestion/source/ge_data_profiler.py    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
index 6b97d2eb456da..c334a97680e3e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
@@ -659,7 +659,16 @@ def generate_dataset_profile(  # noqa: C901 (complexity)
         self.query_combiner.flush()
 
         assert profile.rowCount is not None
-        row_count: int = profile.rowCount
+        row_count: int  # used for null counts calculation
+        if profile.partitionSpec and "SAMPLE" in profile.partitionSpec.partition:
+            # We can alternatively use `self._get_dataset_rows(profile)` to get
+            # exact count of rows in sample, as actual rows involved in sample
+            # may be slightly different (more or less) than configured `sample_size`.
+            # However not doing so to start with, as that adds another query overhead
+            # plus approximate metrics should work for sampling based profiling.
+            row_count = self.config.sample_size
+        else:
+            row_count = profile.rowCount
 
         for column_spec in columns_profiling_queue:
             column = column_spec.column
@@ -811,7 +820,7 @@ def update_dataset_batch_use_sampling(self, profile: DatasetProfileClass) -> Non
             sample_pc = 100 * self.config.sample_size / profile.rowCount
             sql = (
                 f"SELECT * FROM {str(self.dataset._table)} "
-                + f"TABLESAMPLE SYSTEM ({sample_pc:.3f} percent)"
+                + f"TABLESAMPLE SYSTEM ({sample_pc:.8f} percent)"
             )
             temp_table_name = create_bigquery_temp_table(
                 self,

From 11d8988868357b956e7b6ccfa905689d6163f814 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Thu, 2 Nov 2023 17:46:27 +0900
Subject: [PATCH 150/259] docs: add feature guides for subscriptions and
 notifications (#9122)

---
 docs-website/sidebars.js                      |   5 +
 docs/managed-datahub/saas-slack-setup.md      |   4 +-
 .../subscription-and-notification.md          | 130 ++++++++++++++++++
 3 files changed, 137 insertions(+), 2 deletions(-)
 create mode 100644 docs/managed-datahub/subscription-and-notification.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 39eaea57444ed..ab4c1311d5fc7 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -625,6 +625,11 @@ module.exports = {
           type: "doc",
           id: "docs/managed-datahub/chrome-extension",
         },
+        {
+          type: "doc",
+          id: "docs/managed-datahub/subscription-and-notification",
+          className: "saasOnly",
+        },
         {
           "Managed DataHub Release History": [
             "docs/managed-datahub/release-notes/v_0_2_12",
diff --git a/docs/managed-datahub/saas-slack-setup.md b/docs/managed-datahub/saas-slack-setup.md
index 68f947f171715..8d4519b878cd8 100644
--- a/docs/managed-datahub/saas-slack-setup.md
+++ b/docs/managed-datahub/saas-slack-setup.md
@@ -1,6 +1,6 @@
 import FeatureAvailability from '@site/src/components/FeatureAvailability';
 
-# Configure Slack Notifications
+# Configure Slack For Notifications
 
 <FeatureAvailability saasOnly />
 
@@ -108,4 +108,4 @@ For now we support sending notifications to
 - Click on “More”
 - Click on “Copy member ID”
 
-![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_user_id.png)
+![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_user_id.png)
\ No newline at end of file
diff --git a/docs/managed-datahub/subscription-and-notification.md b/docs/managed-datahub/subscription-and-notification.md
new file mode 100644
index 0000000000000..b30a03de16511
--- /dev/null
+++ b/docs/managed-datahub/subscription-and-notification.md
@@ -0,0 +1,130 @@
+import FeatureAvailability from '@site/src/components/FeatureAvailability';
+
+# Subscriptions & Notifications
+
+<FeatureAvailability saasOnly />
+
+DataHub's Subscriptions and Notifications feature gives you real-time change alerts on data assets of your choice.
+With this feature, you can set up subscriptions to specific changes for an Entity – and DataHub will notify you when those changes happen. Currently, DataHub supports notifications on Slack, with support for Microsoft Teams and email subscriptions forthcoming.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-user-notifications-in-slack.png"/>
+</p>
+
+This feature is especially useful in helping you stay on top of any upstream changes that could impact the assets you or your stakeholders rely on. It eliminates the need for you and your team to manually check for upstream changes, or for upstream stakeholders to identify and notify impacted users.
+As a user, you can subscribe to and receive notifications about changes such as deprecations, schema changes, changes in ownership, assertions, or incidents. You’ll always been in the know about potential data quality issues so you can proactively manage your data resources.
+
+## Prerequisites
+
+Once you have [configured Slack within your DataHub instance](saas-slack-setup.md), you will be able to subscribe to any Entity in DataHub and begin recieving notifications via DM.
+If you want to create and manage group-level Subscriptions for your team, you will need [the following privileges](../../docs/authorization/roles.md#role-privileges):
+
+- Manage Group Notification Settings
+- Manage Group Subscriptions
+
+## Using DataHub’s Subscriptions and Notifications Feature
+
+The first step is identifying the assets you want to subscribe to. 
+DataHub’s [Lineage and Impact Analysis features](../../docs/act-on-metadata/impact-analysis.md#lineage-impact-analysis-setup-prerequisites-and-permissions) can help you identify upstream entities that could impact the assets you use and are responsible for.
+You can use the Subscriptions and Notifications feature to sign up for updates for your entire team, or just for yourself.
+
+### Subscribing Your Team/Group to Notifications
+
+The dropdown menu next to the Subscribe button lets you choose who the subscription is for. To create a group subscription, click on Manage Group Subscriptions.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-subscription-dropdown.png"/>
+</p>
+
+Next, customize the group’s subscriptions by selecting the types of changes you want the group to be notified about.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-group-subscription-settings.png"/>
+</p>
+
+Connect to Slack. Currently, Acryl's Subscriptions and Notifications feature integrates only with Slack. Add your group’s Slack Channel ID to receive notifications on Slack.
+(You can find your Channel ID in the About section of your channel on Slack.)
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-group-set-channel-id.png"/>
+</p>
+
+### Individually Subscribing to an Entity
+
+Select the **Subscribe Me** option in the Subscriptions dropdown menu.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-subscription-dropdown-zoom.png"/>
+</p>
+
+Pick the updates you want to be notified about, and connect your Slack account by using your Slack Member ID.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-user-subscription-settings.png"/>
+</p>
+
+:::note
+You can find your Slack Member ID in your profile settings.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-user-set-user-id.png"/>
+</p>
+:::
+
+### Managing Your Subscriptions
+
+You can enable, disable, or manage notifications at any time to ensure that you receive relevant updates.
+
+Simply use the Dropdown menu next to the Subscribe button to unsubscribe from the asset, or to manage/modify your subscription (say, to modify the changes you want to be updated about).
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-subscription-dropdown.png"/>
+</p>
+
+You can also view and manage your subscriptions in your DataHub settings page.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-user-subscriptions.png"/>
+</p>
+
+You can view and manage the group’s subscriptions on the group’s page on DataHub.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-group-subscriptions.png"/>
+</p>
+
+## FAQ
+
+<details>
+<summary>
+What changes can I be notified about using this feature?
+</summary>
+You can subscribe to deprecations, Assertion status changes, Incident status changes, Schema changes, Ownership changes, Glossary Term changes, and Tag changes.
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-user-subscription-settings-zoom.jpg"/>
+</p>
+</details>
+
+<details>
+<summary>
+What if I no longer want to receive updates about a data asset?
+</summary>
+You can unsubscribe from any asset to stop receiving notifications about it. On the asset’s DataHub page, simply use the dropdown menu next to the Subscribe button to unsubscribe from the asset.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/subscription-and-notification/s_n-subscription-dropdown.png"/>
+</p>
+</details>
+
+<details>
+<summary>
+What if I want to be notified about different changes?
+</summary>
+To modify your subscription, use the dropdown menu next to the Subscribe button to modify the changes you want to be notified about.
+</details>
+
+## Reference
+
+- [DataHub Blog - Simplifying Data Monitoring & Management with Subscriptions and Notifications with Acryl DataHub](https://www.acryldata.io/blog/simplifying-data-monitoring-and-management-with-subscriptions-and-notifications-with-acryl-datahub)
+- Video Guide - Getting Started with Subscription & Notifications
+    <iframe width="560" height="315" src="https://www.loom.com/embed/f02fe71e09494b5e82904c8a47f06ac1?sid=ef041cb7-9c06-4926-8e0c-e948b1dc3af0" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>

From f42cb95b928c071b8309cf7c3e9a0fe8b41d3a90 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Thu, 2 Nov 2023 17:46:49 +0900
Subject: [PATCH 151/259] docs: unify oidc guides using tabs (#9068)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 docs-website/sidebars.js                      |  11 +-
 .../guides/sso/configure-oidc-behind-proxy.md |  18 +-
 .../guides/sso/configure-oidc-react-azure.md  | 127 -------
 .../guides/sso/configure-oidc-react-google.md | 118 ------
 .../guides/sso/configure-oidc-react-okta.md   | 124 ------
 .../guides/sso/configure-oidc-react.md        | 355 +++++++++++++-----
 6 files changed, 263 insertions(+), 490 deletions(-)
 delete mode 100644 docs/authentication/guides/sso/configure-oidc-react-azure.md
 delete mode 100644 docs/authentication/guides/sso/configure-oidc-react-google.md
 delete mode 100644 docs/authentication/guides/sso/configure-oidc-react-okta.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index ab4c1311d5fc7..9cc035f3e29e0 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -171,15 +171,8 @@ module.exports = {
             {
               "Frontend Authentication": [
                 "docs/authentication/guides/jaas",
-                {
-                  "OIDC Authentication": [
-                    "docs/authentication/guides/sso/configure-oidc-react",
-                    "docs/authentication/guides/sso/configure-oidc-react-google",
-                    "docs/authentication/guides/sso/configure-oidc-react-okta",
-                    "docs/authentication/guides/sso/configure-oidc-react-azure",
-                    "docs/authentication/guides/sso/configure-oidc-behind-proxy",
-                  ],
-                },
+                "docs/authentication/guides/sso/configure-oidc-react",
+                "docs/authentication/guides/sso/configure-oidc-behind-proxy",
               ],
             },
             "docs/authentication/introducing-metadata-service-authentication",
diff --git a/docs/authentication/guides/sso/configure-oidc-behind-proxy.md b/docs/authentication/guides/sso/configure-oidc-behind-proxy.md
index c998816e04735..684bf768f2baf 100644
--- a/docs/authentication/guides/sso/configure-oidc-behind-proxy.md
+++ b/docs/authentication/guides/sso/configure-oidc-behind-proxy.md
@@ -1,8 +1,9 @@
-# Configuring Frontend to use a Proxy when communicating with SSO Provider
-*Authored on 22/08/2023*
+# OIDC Proxy Configuration
 
-The `datahub-frontend-react` server can be configured to use an http proxy when retrieving the openid-configuration. 
-This can be needed if your infrastructure is locked down and disallows connectivity by default, using proxies for fine-grained egress control. 
+_Authored on 22/08/2023_
+
+The `datahub-frontend-react` server can be configured to use an http proxy when retrieving the openid-configuration.
+This can be needed if your infrastructure is locked down and disallows connectivity by default, using proxies for fine-grained egress control.
 
 ## Configure http proxy and non proxy hosts
 
@@ -17,7 +18,8 @@ HTTP_NON_PROXY_HOSTS=localhost|datahub-gms (or any other hosts that you would li
 ```
 
 ## Optional: provide custom truststore
-If your upstream proxy performs SSL termination to inspect traffic, this will result in different (self-signed) certificates for HTTPS connections. 
+
+If your upstream proxy performs SSL termination to inspect traffic, this will result in different (self-signed) certificates for HTTPS connections.
 The default truststore used in the `datahub-frontend-react` docker image will not trust these kinds of connections.
 To address this, you can copy or mount your own truststore (provided by the proxy or network administrators) into the docker container.
 
@@ -36,8 +38,8 @@ FROM linkedin/datahub-frontend-react:<version>
 COPY /truststore-directory /certificates
 ```
 
-Building this Dockerfile will result in your own custom docker image on your local machine. 
-You will then be able to tag it, publish it to your own registry, etc. 
+Building this Dockerfile will result in your own custom docker image on your local machine.
+You will then be able to tag it, publish it to your own registry, etc.
 
 #### Option b) Mount truststore from your host machine using a docker volume
 
@@ -51,7 +53,7 @@ Adapt your docker-compose.yml to include a new volume mount in the `datahub-fron
       - /truststore-directory:/certificates
 ```
 
-### Reference new truststore 
+### Reference new truststore
 
 Add the following environment values to the `datahub-frontend-react` container:
 
diff --git a/docs/authentication/guides/sso/configure-oidc-react-azure.md b/docs/authentication/guides/sso/configure-oidc-react-azure.md
deleted file mode 100644
index 177387327c0e8..0000000000000
--- a/docs/authentication/guides/sso/configure-oidc-react-azure.md
+++ /dev/null
@@ -1,127 +0,0 @@
-# Configuring Azure Authentication for React App (OIDC)
-*Authored on 21/12/2021*
-
-`datahub-frontend` server can be configured to authenticate users over OpenID Connect (OIDC). As such, it can be configured to
-delegate authentication responsibility to identity providers like Microsoft Azure.
-
-This guide will provide steps for configuring DataHub authentication using Microsoft Azure.
-
-:::caution
-Even when OIDC is configured, the root user can still login without OIDC by going
-to `/login` URL endpoint. It is recommended that you don't use the default
-credentials by mounting a different file in the front end container. To do this
-please see [this guide](../jaas.md) to mount a custom user.props file for a JAAS authenticated deployment.
-:::
-
-## Steps
-
-### 1. Create an application registration in Microsoft Azure portal
-
-a. Using an account linked to your organization, navigate to the [Microsoft Azure Portal](https://portal.azure.com).
-
-b. Select **App registrations**, then **New registration** to register a new app.
-
-c. Name your app registration and choose who can access your application.
-
-d. Select `Web` as the **Redirect URI** type and enter the following:
-```
-https://your-datahub-domain.com/callback/oidc
-```
-If you are just testing locally, the following can be used: `http://localhost:9002/callback/oidc`.
-Azure supports more than one redirect URI, so both can be configured at the same time from the **Authentication** tab once the registration is complete.
-
-At this point, your app registration should look like the following:
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-app-registration.png"/>
-</p>
-
-
-e. Click **Register**.
-
-### 2. Configure Authentication (optional)
-
-Once registration is done, you will land on the app registration **Overview** tab.  On the left-side navigation bar, click on **Authentication** under **Manage** and add extra redirect URIs if need be (if you want to support both local testing and Azure deployments).
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-authentication.png"/>
-</p>
-
-
-Click **Save**.
-
-### 3. Configure Certificates & secrets
-
-On the left-side navigation bar, click on **Certificates & secrets** under **Manage**.  
-Select **Client secrets**, then **New client secret**.  Type in a meaningful description for your secret and select an expiry.  Click the **Add** button when you are done.
-
-**IMPORTANT:** Copy the `value` of your newly create secret since Azure will never display its value afterwards.
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-certificates-secrets.png"/>
-</p>
-
-
-### 4. Configure API permissions
-
-On the left-side navigation bar, click on **API permissions** under **Manage**.  DataHub requires the following four Microsoft Graph APIs:
-
-1. `User.Read` *(should be already configured)*
-2. `profile`
-3. `email`
-4. `openid`
-
-Click on **Add a permission**, then from the **Microsoft APIs** tab select **Microsoft Graph**, then **Delegated permissions**.  From the **OpenId permissions** category, select `email`, `openid`, `profile` and click **Add permissions**.
-
-At this point, you should be looking at a screen like the following:
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-api-permissions.png"/>
-</p>
-
-
-### 5. Obtain Application (Client) ID
-
-On the left-side navigation bar, go back to the **Overview** tab.  You should see the `Application (client) ID`. Save its value for the next step.
-
-### 6. Obtain Discovery URI
-
-On the same page, you should see a `Directory (tenant) ID`. Your OIDC discovery URI will be formatted as follows:
-
-```
-https://login.microsoftonline.com/{tenant ID}/v2.0/.well-known/openid-configuration
-```
-
-### 7. Configure `datahub-frontend` to enable OIDC authentication
-
-a. Open the file `docker/datahub-frontend/env/docker.env`
-
-b. Add the following configuration values to the file:
-
-```
-AUTH_OIDC_ENABLED=true
-AUTH_OIDC_CLIENT_ID=your-client-id
-AUTH_OIDC_CLIENT_SECRET=your-client-secret
-AUTH_OIDC_DISCOVERY_URI=https://login.microsoftonline.com/{tenant ID}/v2.0/.well-known/openid-configuration
-AUTH_OIDC_BASE_URL=your-datahub-url
-AUTH_OIDC_SCOPE="openid profile email"
-```
-
-Replacing the placeholders above with the client id (step 5), client secret (step 3) and tenant ID (step 6) received from Microsoft Azure.
-
-### 9. Restart `datahub-frontend-react` docker container
-
-Now, simply restart the `datahub-frontend-react` container to enable the integration.
-
-```
-docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml  up datahub-frontend-react
-```
-
-Navigate to your DataHub domain to see SSO in action.
-
-## Resources
-- [Microsoft identity platform and OpenID Connect protocol](https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-protocols-oidc/)
\ No newline at end of file
diff --git a/docs/authentication/guides/sso/configure-oidc-react-google.md b/docs/authentication/guides/sso/configure-oidc-react-google.md
deleted file mode 100644
index af62185e6e787..0000000000000
--- a/docs/authentication/guides/sso/configure-oidc-react-google.md
+++ /dev/null
@@ -1,118 +0,0 @@
-# Configuring Google Authentication for React App (OIDC)
-*Authored on 3/10/2021*
-
-`datahub-frontend` server can be configured to authenticate users over OpenID Connect (OIDC). As such, it can be configured to delegate 
-authentication responsibility to identity providers like Google. 
-
-This guide will provide steps for configuring DataHub authentication using Google.
-
-:::caution
-Even when OIDC is configured, the root user can still login without OIDC by going
-to `/login` URL endpoint. It is recommended that you don't use the default
-credentials by mounting a different file in the front end container. To do this
-please see [this guide](../jaas.md) to mount a custom user.props file for a JAAS authenticated deployment.
-:::
-
-## Steps
-
-### 1. Create a project in the Google API Console
-
-Using an account linked to your organization, navigate to the [Google API Console](https://console.developers.google.com/) and select **New project**. 
-Within this project, we will configure the OAuth2.0 screen and credentials. 
-
-### 2. Create OAuth2.0 consent screen
-
-a. Navigate to `OAuth consent screen`. This is where you'll configure the screen your users see when attempting to 
-log in to DataHub. 
-
-b. Select `Internal` (if you only want your company users to have access) and then click **Create**. 
-Note that in order to complete this step you should be logged into a Google account associated with your organization. 
-
-c. Fill out the details in the App Information & Domain sections. Make sure the 'Application Home Page' provided matches where DataHub is deployed
-at your organization. 
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google-setup-1.png"/>
-</p>
-
-
-Once you've completed this, **Save & Continue**. 
-
-d. Configure the scopes: Next, click **Add or Remove Scopes**. Select the following scopes:
-    
-    - `.../auth/userinfo.email`
-    - `.../auth/userinfo.profile`
-    - `openid`
-
-Once you've selected these, **Save & Continue**. 
-
-### 3. Configure client credentials
-
-Now navigate to the **Credentials** tab. This is where you'll obtain your client id & secret, as well as configure info 
-like the redirect URI used after a user is authenticated. 
-
-a. Click **Create Credentials** & select `OAuth client ID` as the credential type.
-
-b. On the following screen, select `Web application` as your Application Type. 
-
-c. Add the domain where DataHub is hosted to your 'Authorized Javascript Origins'.
-
-```
-https://your-datahub-domain.com
-```
-
-d. Add the domain where DataHub is hosted with the path `/callback/oidc` appended to 'Authorized Redirect URLs'. 
-
-```
-https://your-datahub-domain.com/callback/oidc
-```
-
-e. Click **Create**
-
-f. You will now receive a pair of values, a client id and a client secret. Bookmark these for the next step.
-
-At this point, you should be looking at a screen like the following:
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google-setup-2.png"/>
-</p>
-
-
-Success!
-
-### 4. Configure `datahub-frontend` to enable OIDC authentication
-
-a. Open the file `docker/datahub-frontend/env/docker.env`
-
-b. Add the following configuration values to the file: 
-
-```
-AUTH_OIDC_ENABLED=true
-AUTH_OIDC_CLIENT_ID=your-client-id
-AUTH_OIDC_CLIENT_SECRET=your-client-secret
-AUTH_OIDC_DISCOVERY_URI=https://accounts.google.com/.well-known/openid-configuration
-AUTH_OIDC_BASE_URL=your-datahub-url
-AUTH_OIDC_SCOPE="openid profile email"
-AUTH_OIDC_USER_NAME_CLAIM=email
-AUTH_OIDC_USER_NAME_CLAIM_REGEX=([^@]+)
-```
-
-Replacing the placeholders above with the client id & client secret received from Google in Step 3f.  
-
-
-### 5. Restart `datahub-frontend-react` docker container
-
-Now, simply restart the `datahub-frontend-react` container to enable the integration. 
-
-```
-docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml  up datahub-frontend-react
-```
-
-Navigate to your DataHub domain to see SSO in action. 
-
-
-## References
-
-- [OpenID Connect in Google Identity](https://developers.google.com/identity/protocols/oauth2/openid-connect)
\ No newline at end of file
diff --git a/docs/authentication/guides/sso/configure-oidc-react-okta.md b/docs/authentication/guides/sso/configure-oidc-react-okta.md
deleted file mode 100644
index 320b887a28f16..0000000000000
--- a/docs/authentication/guides/sso/configure-oidc-react-okta.md
+++ /dev/null
@@ -1,124 +0,0 @@
-# Configuring Okta Authentication for React App (OIDC)
-*Authored on 3/10/2021*
-
-`datahub-frontend` server can be configured to authenticate users over OpenID Connect (OIDC). As such, it can be configured to
-delegate authentication responsibility to identity providers like Okta.
-
-This guide will provide steps for configuring DataHub authentication using Okta.
-
-:::caution
-Even when OIDC is configured, the root user can still login without OIDC by going
-to `/login` URL endpoint. It is recommended that you don't use the default
-credentials by mounting a different file in the front end container. To do this
-please see [this guide](../jaas.md) to mount a custom user.props file for a JAAS authenticated deployment.
-:::
-
-## Steps
-
-### 1. Create an application in Okta Developer Console
-
-a. Log in to your Okta admin account & navigate to the developer console
-
-b. Select **Applications**, then **Add Application**, the **Create New App** to create a new app.
-
-c. Select `Web` as the **Platform**, and `OpenID Connect` as the **Sign on method**
-
-d. Click **Create**
-
-e. Under 'General Settings', name your application
-
-f. Below, add a **Login Redirect URI**. This should be formatted as
-
-```
-https://your-datahub-domain.com/callback/oidc
-```
-
-If you're just testing locally, this can be `http://localhost:9002/callback/oidc`.
-
-g. Below, add a **Logout Redirect URI**. This should be formatted as
-
-```
-https://your-datahub-domain.com
-```
-
-h. [Optional] If you're enabling DataHub login as an Okta tile, you'll need to provide the **Initiate Login URI**. You
-can set if to
-
-```
-https://your-datahub-domain.com/authenticate
-```
-
-If you're just testing locally, this can be `http://localhost:9002`.
-
-i. Click **Save**
-
-
-### 2. Obtain Client Credentials
-
-On the subsequent screen, you should see the client credentials. Bookmark the `Client id` and `Client secret` for the next step.
-
-### 3. Obtain Discovery URI
-
-On the same page, you should see an `Okta Domain`. Your OIDC discovery URI will be formatted as follows:
-
-```
-https://your-okta-domain.com/.well-known/openid-configuration
-```
-
-for example, `https://dev-33231928.okta.com/.well-known/openid-configuration`.
-
-At this point, you should be looking at a screen like the following:
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-1.png"/>
-</p>
-
-
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-2.png"/>
-</p>
-
-
-Success!
-
-### 4. Configure `datahub-frontend` to enable OIDC authentication
-
-a. Open the file `docker/datahub-frontend/env/docker.env`
-
-b. Add the following configuration values to the file:
-
-```
-AUTH_OIDC_ENABLED=true
-AUTH_OIDC_CLIENT_ID=your-client-id
-AUTH_OIDC_CLIENT_SECRET=your-client-secret
-AUTH_OIDC_DISCOVERY_URI=https://your-okta-domain.com/.well-known/openid-configuration
-AUTH_OIDC_BASE_URL=your-datahub-url
-AUTH_OIDC_SCOPE="openid profile email groups"
-```
-
-Replacing the placeholders above with the client id & client secret received from Okta in Step 2.
-
-> **Pro Tip!** You can easily enable Okta to return the groups that a user is associated with, which will be provisioned in DataHub, along with the user logging in. This can be enabled by setting the `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` flag to `true`. 
-> if they do not already exist in DataHub. You can enable your Okta application to return a 'groups' claim from the Okta Console at Applications > Your Application -> Sign On -> OpenID Connect ID Token Settings (Requires an edit).
-> 
-> By default, we assume that the groups will appear in a claim named "groups". This can be customized using the `AUTH_OIDC_GROUPS_CLAIM` container configuration. 
-> 
-> 
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-groups-claim.png"/>
-</p>
-
-
-### 5. Restart `datahub-frontend-react` docker container
-
-Now, simply restart the `datahub-frontend-react` container to enable the integration.
-
-```
-docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml  up datahub-frontend-react
-```
-
-Navigate to your DataHub domain to see SSO in action.
-
-## Resources
-- [OAuth 2.0 and OpenID Connect Overview](https://developer.okta.com/docs/concepts/oauth-openid/)
diff --git a/docs/authentication/guides/sso/configure-oidc-react.md b/docs/authentication/guides/sso/configure-oidc-react.md
index 1671673c09318..9b4af80bb0ccd 100644
--- a/docs/authentication/guides/sso/configure-oidc-react.md
+++ b/docs/authentication/guides/sso/configure-oidc-react.md
@@ -1,59 +1,201 @@
-# Overview
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# OIDC Authentication
 
 The DataHub React application supports OIDC authentication built on top of the [Pac4j Play](https://github.com/pac4j/play-pac4j) library.
 This enables operators of DataHub to integrate with 3rd party identity providers like Okta, Google, Keycloak, & more to authenticate their users.
 
-When configured, OIDC auth will be enabled between clients of the DataHub UI & `datahub-frontend` server. Beyond this point is considered
-to be a secure environment and as such authentication is validated & enforced only at the "front door" inside datahub-frontend.
+## 1. Register an app with your Identity Provider
 
-:::caution
-Even if OIDC is configured the root user can still login without OIDC by going
-to `/login` URL endpoint. It is recommended that you don't use the default
-credentials by mounting a different file in the front end container. To do this
-please see [this guide](../jaas.md) to mount a custom user.props file for a JAAS authenticated deployment.
+<Tabs>
+<TabItem value="google" label="Google Identity">
+
+#### Create a project in the Google API Console
+
+Using an account linked to your organization, navigate to the [Google API Console](https://console.developers.google.com/) and select **New project**.
+Within this project, we will configure the OAuth2.0 screen and credentials.
+
+#### Create OAuth2.0 consent screen
+
+Navigate to **OAuth consent screen**. This is where you'll configure the screen your users see when attempting to
+log in to DataHub. Select **Internal** (if you only want your company users to have access) and then click **Create**.
+Note that in order to complete this step you should be logged into a Google account associated with your organization.
+
+Fill out the details in the App Information & Domain sections. Make sure the 'Application Home Page' provided matches where DataHub is deployed
+at your organization. Once you've completed this, **Save & Continue**.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/google-setup-1.png"/>
+</p>
+
+#### Configure the scopes
+
+Next, click **Add or Remove Scopes**. Select the following scope and click **Save & Continue**.
+
+- .../auth/userinfo.email
+- .../auth/userinfo.profile
+- openid
+
+</TabItem>
+<TabItem value="okta" label="Okta">
+
+#### Create an application in Okta Developer Console
+
+Log in to your Okta admin account & navigate to the developer console. Select **Applications**, then **Add Application**, the **Create New App** to create a new app.
+Select `Web` as the **Platform**, and `OpenID Connect` as the **Sign on method**.
+
+Click **Create** and name your application under **General Settings** and save.
+
+- **Login Redirect URI** : `https://your-datahub-domain.com/callback/oidc`.
+- **Logout Redirect URI**. `https://your-datahub-domain.com`
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-2.png"/>
+</p>
+
+:::note Optional
+If you're enabling DataHub login as an Okta tile, you'll need to provide the **Initiate Login URI**. You
+can set if to `https://your-datahub-domain.com/authenticate`. If you're just testing locally, this can be `http://localhost:9002`.
 :::
 
-## Provider-Specific Guides
+</TabItem>
+<TabItem value="azure" label="Azure">
 
-1. [Configuring OIDC using Google](configure-oidc-react-google.md)
-2. [Configuring OIDC using Okta](configure-oidc-react-okta.md)
-3. [Configuring OIDC using Azure](configure-oidc-react-azure.md)
+#### Create an application registration in Microsoft Azure portal
 
-## Configuring OIDC in React
+Using an account linked to your organization, navigate to the [Microsoft Azure Portal](https://portal.azure.com). Select **App registrations**, then **New registration** to register a new app.
 
-### 1. Register an app with your Identity Provider
+Name your app registration and choose who can access your application.
 
-To configure OIDC in React, you will most often need to register yourself as a client with your identity provider (Google, Okta, etc). Each provider may
-have their own instructions. Provided below are links to examples for Okta, Google, Azure AD, & Keycloak.
+- **Redirect URI** : Select **Web** as type and enter `https://your-datahub-domain.com/callback/oidc`
 
-- [Registering an App in Okta](https://developer.okta.com/docs/guides/add-an-external-idp/openidconnect/main/)
-- [OpenID Connect in Google Identity](https://developers.google.com/identity/protocols/oauth2/openid-connect)
-- [OpenID Connect authentication with Azure Active Directory](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/auth-oidc)
-- [Keycloak - Securing Applications and Services Guide](https://www.keycloak.org/docs/latest/securing_apps/)
+Azure supports more than one redirect URI, so both can be configured at the same time from the **Authentication** tab once the registration is complete.
+At this point, your app registration should look like the following. Finally, click **Register**.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-app-registration.png"/>
+</p>
 
-During the registration process, you'll need to provide a login redirect URI to the identity provider. This tells the identity provider
-where to redirect to once they've authenticated the end user.
+:::note Optional
+Once registration is done, you will land on the app registration **Overview** tab.  
+On the left-side navigation bar, click on **Authentication** under **Manage** and add extra redirect URIs if need be (if you want to support both local testing and Azure deployments). Finally, click **Save**.
 
-By default, the URL will be constructed as follows:
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-authentication.png"/>
+</p>
 
-> "http://your-datahub-domain.com/callback/oidc"
+:::
+
+#### Configure Certificates & secrets
+
+On the left-side navigation bar, click on **Certificates & secrets** under **Manage**.  
+Select **Client secrets**, then **New client secret**. Type in a meaningful description for your secret and select an expiry. Click the **Add** button when you are done.
+Copy the value of your newly create secret since Azure will never display its value afterwards.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-certificates-secrets.png"/>
+</p>
+
+#### Configure API permissions
+
+On the left-side navigation bar, click on **API permissions** under **Manage**. DataHub requires the following four Microsoft Graph APIs:
 
-For example, if you're hosted DataHub at `datahub.myorg.com`, this
-value would be `http://datahub.myorg.com/callback/oidc`. For testing purposes you can also specify localhost as the domain name
-directly: `http://localhost:9002/callback/oidc`
+- User.Read _(should be already configured)_
+- profile
+- email
+- openid
+
+Click on **Add a permission**, then from the **Microsoft APIs** tab select **Microsoft Graph**, then **Delegated permissions**. From the **OpenId permissions** category, select `email`, `openid`, `profile` and click **Add permissions**.
+
+At this point, you should be looking at a screen like the following:
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/azure-setup-api-permissions.png"/>
+</p>
+
+</TabItem>
+</Tabs>
+
+## 2. Obtain Client Credentials & Discovery URL
 
 The goal of this step should be to obtain the following values, which will need to be configured before deploying DataHub:
 
-1. **Client ID** - A unique identifier for your application with the identity provider
-2. **Client Secret** - A shared secret to use for exchange between you and your identity provider
-3. **Discovery URL** - A URL where the OIDC API of your identity provider can be discovered. This should suffixed by
-   `.well-known/openid-configuration`. Sometimes, identity providers will not explicitly include this URL in their setup guides, though
-   this endpoint *will* exist as per the OIDC specification. For more info see http://openid.net/specs/openid-connect-discovery-1_0.html.
+- **Client ID** - A unique identifier for your application with the identity provider
+- **Client Secret** - A shared secret to use for exchange between you and your identity provider
+- **Discovery URL** - A URL where the OIDC API of your identity provider can be discovered. This should suffixed by
+  `.well-known/openid-configuration`. Sometimes, identity providers will not explicitly include this URL in their setup guides, though
+  this endpoint _will_ exist as per the OIDC specification. For more info see http://openid.net/specs/openid-connect-discovery-1_0.html.
+
+<Tabs>
+
+<TabItem value="google" label="Google Identity">
+
+**Obtain Client Credentials**
+
+Navigate to the **Credentials** tab. Click **Create Credentials** & select **OAuth client ID** as the credential type.
+
+On the following screen, select **Web application** as your Application Type.
+Add the domain where DataHub is hosted to your 'Authorized Javascript Origins'.
+
+```
+https://your-datahub-domain.com
+```
+
+Add the domain where DataHub is hosted with the path `/callback/oidc` appended to 'Authorized Redirect URLs'. Finally, click **Create**
+
+```
+https://your-datahub-domain.com/callback/oidc
+```
+
+You will now receive a pair of values, a client id and a client secret. Bookmark these for the next step.
+
+</TabItem>
+<TabItem value="okta" label="Okta">
+
+**Obtain Client Credentials**
+
+After registering the app, you should see the client credentials. Bookmark the `Client id` and `Client secret` for the next step.
+
+**Obtain Discovery URI**
+
+On the same page, you should see an `Okta Domain`. Your OIDC discovery URI will be formatted as follows:
+
+```
+https://your-okta-domain.com/.well-known/openid-configuration
+```
+
+For example, `https://dev-33231928.okta.com/.well-known/openid-configuration`.
+
+At this point, you should be looking at a screen like the following:
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sso/okta-setup-1.png"/>
+</p>
 
+</TabItem>
+<TabItem value="azure" label="Azure">
 
-### 2. Configure DataHub Frontend Server
+**Obtain Application (Client) ID**
 
-The second step to enabling OIDC involves configuring `datahub-frontend` to enable OIDC authentication with your Identity Provider.
+On the left-side navigation bar, go back to the **Overview** tab. You should see the `Application (client) ID`. Save its value for the next step.
+
+**Obtain Discovery URI**
+
+On the same page, you should see a `Directory (tenant) ID`. Your OIDC discovery URI will be formatted as follows:
+
+```
+https://login.microsoftonline.com/{tenant ID}/v2.0/.well-known/openid-configuration
+```
+
+</TabItem>
+</Tabs>
+
+## 3. Configure DataHub Frontend Server
+
+### Docker
+
+The next step to enabling OIDC involves configuring `datahub-frontend` to enable OIDC authentication with your Identity Provider.
 
 To do so, you must update the `datahub-frontend` [docker.env](../../../../docker/datahub-frontend/env/docker.env) file with the
 values received from your identity provider:
@@ -67,22 +209,29 @@ AUTH_OIDC_DISCOVERY_URI=your-provider-discovery-url
 AUTH_OIDC_BASE_URL=your-datahub-url
 ```
 
-- `AUTH_OIDC_ENABLED`: Enable delegating authentication to OIDC identity provider
-- `AUTH_OIDC_CLIENT_ID`: Unique client id received from identity provider
-- `AUTH_OIDC_CLIENT_SECRET`: Unique client secret received from identity provider
-- `AUTH_OIDC_DISCOVERY_URI`: Location of the identity provider OIDC discovery API. Suffixed with `.well-known/openid-configuration`
-- `AUTH_OIDC_BASE_URL`: The base URL of your DataHub deployment, e.g. https://yourorgdatahub.com (prod) or http://localhost:9002 (testing)
-- `AUTH_SESSION_TTL_HOURS`: The length of time in hours before a user will be prompted to login again. Controls the actor cookie expiration time in the browser. Numeric value converted to hours, default 24.
-- `MAX_SESSION_TOKEN_AGE`: Determines the expiration time of a session token. Session tokens are stateless so this determines at what time a session token may no longer be used and a valid session token can be used until this time has passed. Accepts a valid relative Java date style String, default 24h.
+| Configuration           | Description                                                                                                                                                                                                                                                              | Default |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- |
+| AUTH_OIDC_ENABLED       | Enable delegating authentication to OIDC identity provider                                                                                                                                                                                                               |         |
+| AUTH_OIDC_CLIENT_ID     | Unique client id received from identity provider                                                                                                                                                                                                                         |         |
+| AUTH_OIDC_CLIENT_SECRET | Unique client secret received from identity provider                                                                                                                                                                                                                     |         |
+| AUTH_OIDC_DISCOVERY_URI | Location of the identity provider OIDC discovery API. Suffixed with `.well-known/openid-configuration`                                                                                                                                                                   |         |
+| AUTH_OIDC_BASE_URL      | The base URL of your DataHub deployment, e.g. https://yourorgdatahub.com (prod) or http://localhost:9002 (testing)                                                                                                                                                       |         |
+| AUTH_SESSION_TTL_HOURS  | The length of time in hours before a user will be prompted to login again. Controls the actor cookie expiration time in the browser. Numeric value converted to hours.                                                                                                   | 24      |
+| MAX_SESSION_TOKEN_AGE   | Determines the expiration time of a session token. Session tokens are stateless so this determines at what time a session token may no longer be used and a valid session token can be used until this time has passed. Accepts a valid relative Java date style String. | 24h     |
 
 Providing these configs will cause DataHub to delegate authentication to your identity
 provider, requesting the "oidc email profile" scopes and parsing the "preferred_username" claim from
 the authenticated profile as the DataHub CorpUser identity.
 
+:::note
+
+By default, the login callback endpoint exposed by DataHub will be located at `${AUTH_OIDC_BASE_URL}/callback/oidc`. This must **exactly** match the login redirect URL you've registered with your identity provider in step 1.
+
+:::
 
-> By default, the login callback endpoint exposed by DataHub will be located at `${AUTH_OIDC_BASE_URL}/callback/oidc`. This must **exactly** match the login redirect URL you've registered with your identity provider in step 1.
+### Kubernetes
 
-In kubernetes, you can add the above env variables in the values.yaml as follows.
+In Kubernetes, you can add the above env variables in the `values.yaml` as follows.
 
 ```yaml
 datahub-frontend:
@@ -102,20 +251,21 @@ datahub-frontend:
 
 You can also package OIDC client secrets into a k8s secret by running
 
-```kubectl create secret generic datahub-oidc-secret --from-literal=secret=<<OIDC SECRET>>```
+```
+kubectl create secret generic datahub-oidc-secret --from-literal=secret=<<OIDC SECRET>>
+```
 
 Then set the secret env as follows.
 
 ```yaml
-    - name: AUTH_OIDC_CLIENT_SECRET
-      valueFrom:
-        secretKeyRef:
-          name: datahub-oidc-secret
-          key: secret
+- name: AUTH_OIDC_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: datahub-oidc-secret
+      key: secret
 ```
 
-
-#### Advanced
+### Advanced OIDC Configurations
 
 You can optionally customize the flow further using advanced configurations. These allow
 you to specify the OIDC scopes requested, how the DataHub username is parsed from the claims returned by the identity provider, and how users and groups are extracted and provisioned from the OIDC claim set.
@@ -128,23 +278,15 @@ AUTH_OIDC_SCOPE=your-custom-scope
 AUTH_OIDC_CLIENT_AUTHENTICATION_METHOD=authentication-method
 ```
 
-- `AUTH_OIDC_USER_NAME_CLAIM`: The attribute that will contain the username used on the DataHub platform. By default, this is "email" provided
-  as part of the standard `email` scope.
-- `AUTH_OIDC_USER_NAME_CLAIM_REGEX`: A regex string used for extracting the username from the userNameClaim attribute. For example, if
-  the userNameClaim field will contain an email address, and we want to omit the domain name suffix of the email, we can specify a custom
-  regex to do so. (e.g. `([^@]+)`)
-- `AUTH_OIDC_SCOPE`: a string representing the scopes to be requested from the identity provider, granted by the end user. For more info,
-  see [OpenID Connect Scopes](https://auth0.com/docs/scopes/openid-connect-scopes).
-- `AUTH_OIDC_CLIENT_AUTHENTICATION_METHOD`: a string representing the token authentication method to use with the identity provider. Default value
-  is `client_secret_basic`, which uses HTTP Basic authentication. Another option is `client_secret_post`, which includes the client_id and secret_id
-  as form parameters in the HTTP POST request. For more info, see [OAuth 2.0 Client Authentication](https://darutk.medium.com/oauth-2-0-client-authentication-4b5f929305d4)
-
-Additional OIDC Options:
+| Configuration                          | Description                                                                                                                                                                                                                                                                                                                                                                                                                         | Default             |
+| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- |
+| AUTH_OIDC_USER_NAME_CLAIM              | The attribute that will contain the username used on the DataHub platform. By default, this is "email" providedas part of the standard `email` scope.                                                                                                                                                                                                                                                                               |                     |
+| AUTH_OIDC_USER_NAME_CLAIM_REGEX        | A regex string used for extracting the username from the userNameClaim attribute. For example, if the userNameClaim field will contain an email address, and we want to omit the domain name suffix of the email, we can specify a customregex to do so. (e.g. `([^@]+)`)                                                                                                                                                           |                     |
+| AUTH_OIDC_SCOPE                        | A string representing the scopes to be requested from the identity provider, granted by the end user. For more info, see [OpenID Connect Scopes](https://auth0.com/docs/scopes/openid-connect-scopes).                                                                                                                                                                                                                              |                     |
+| AUTH_OIDC_CLIENT_AUTHENTICATION_METHOD | a string representing the token authentication method to use with the identity provider. Default value is `client_secret_basic`, which uses HTTP Basic authentication. Another option is `client_secret_post`, which includes the client_id and secret_id as form parameters in the HTTP POST request. For more info, see [OAuth 2.0 Client Authentication](https://darutk.medium.com/oauth-2-0-client-authentication-4b5f929305d4) | client_secret_basic |
+| AUTH_OIDC_PREFERRED_JWS_ALGORITHM      | Can be used to select a preferred signing algorithm for id tokens. Examples include: `RS256` or `HS256`. If your IdP includes `none` before `RS256`/`HS256` in the list of signing algorithms, then this value **MUST** be set.                                                                                                                                                                                                     |                     |
 
-- `AUTH_OIDC_PREFERRED_JWS_ALGORITHM` - Can be used to select a preferred signing algorithm for id tokens. Examples include: `RS256` or `HS256`. If
-your IdP includes `none` before `RS256`/`HS256` in the list of signing algorithms, then this value **MUST** be set.
-
-##### User & Group Provisioning (JIT Provisioning)
+### User & Group Provisioning (JIT Provisioning)
 
 By default, DataHub will optimistically attempt to provision users and groups that do not already exist at the time of login.
 For users, we extract information like first name, last name, display name, & email to construct a basic user profile. If a groups claim is present,
@@ -160,26 +302,30 @@ AUTH_OIDC_EXTRACT_GROUPS_ENABLED=false
 AUTH_OIDC_GROUPS_CLAIM=<your-groups-claim-name>
 ```
 
-- `AUTH_OIDC_JIT_PROVISIONING_ENABLED`: Whether DataHub users & groups should be provisioned on login if they do not exist. Defaults to true.
-- `AUTH_OIDC_PRE_PROVISIONING_REQUIRED`: Whether the user should already exist in DataHub when they login, failing login if they are not. This is appropriate for situations in which users and groups are batch ingested and tightly controlled inside your environment. Defaults to false.
-- `AUTH_OIDC_EXTRACT_GROUPS_ENABLED`: Only applies if `AUTH_OIDC_JIT_PROVISIONING_ENABLED` is set to true. This determines whether we should attempt to extract a list of group names from a particular claim in the OIDC attributes. Note that if this is enabled, each login will re-sync group membership with the groups in your Identity Provider, clearing the group membership that has been assigned through the DataHub UI. Enable with care! Defaults to false.
-- `AUTH_OIDC_GROUPS_CLAIM`: Only applies if `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` is set to true. This determines which OIDC claims will contain a list of string group names. Accepts multiple claim names with comma-separated values. I.e: `groups, teams, departments`. Defaults to 'groups'.
+| Configuration                       | Description                                                                                                                                                                                                                                                                                                                                                                                                      | Default |
+| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| AUTH_OIDC_JIT_PROVISIONING_ENABLED  | Whether DataHub users & groups should be provisioned on login if they do not exist.                                                                                                                                                                                                                                                                                                                              | true    |
+| AUTH_OIDC_PRE_PROVISIONING_REQUIRED | Whether the user should already exist in DataHub when they login, failing login if they are not. This is appropriate for situations in which users and groups are batch ingested and tightly controlled inside your environment.                                                                                                                                                                                 | false   |
+| AUTH_OIDC_EXTRACT_GROUPS_ENABLED    | Only applies if `AUTH_OIDC_JIT_PROVISIONING_ENABLED` is set to true. This determines whether we should attempt to extract a list of group names from a particular claim in the OIDC attributes. Note that if this is enabled, each login will re-sync group membership with the groups in your Identity Provider, clearing the group membership that has been assigned through the DataHub UI. Enable with care! | false   |
+| AUTH_OIDC_GROUPS_CLAIM              | Only applies if `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` is set to true. This determines which OIDC claims will contain a list of string group names. Accepts multiple claim names with comma-separated values. I.e: `groups, teams, departments`.                                                                                                                                                                     | groups  |
 
+## 4. Restart datahub-frontend-react
 
-Once configuration has been updated, `datahub-frontend-react` will need to be restarted to pick up the new environment variables:
+Once configured, restarting the `datahub-frontend-react` container will enable an indirect authentication flow in which DataHub delegates authentication to the specified identity provider.
 
 ```
 docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml  up datahub-frontend-react
 ```
 
->Note that by default, enabling OIDC will *not* disable the dummy JAAS authentication path, which can be reached at the `/login`
-route of the React app. To disable this authentication path, additionally specify the following config:
-> `AUTH_JAAS_ENABLED=false`
+Navigate to your DataHub domain to see SSO in action.
 
-### Summary
+:::caution
+By default, enabling OIDC will _not_ disable the dummy JAAS authentication path, which can be reached at the `/login`
+route of the React app. To disable this authentication path, additionally specify the following config:
+`AUTH_JAAS_ENABLED=false`
+:::
 
-Once configured, deploying the `datahub-frontend-react` container will enable an indirect authentication flow in which DataHub delegates
-authentication to the specified identity provider.
+## Summary
 
 Once a user is authenticated by the identity provider, DataHub will extract a username from the provided claims
 and grant DataHub access to the user by setting a pair of session cookies.
@@ -196,44 +342,45 @@ A brief summary of the steps that occur when the user navigates to the React app
 7. DataHub sets session cookies for the newly authenticated user
 8. DataHub redirects the user to the homepage ("/")
 
-## FAQ
+## Troubleshooting
 
-**No users can log in. Instead, I get redirected to the login page with an error. What do I do?**
+<details>
+<summary>No users can log in. Instead, I get redirected to the login page with an error. What do I do?</summary>
 
 This can occur for a variety of reasons, but most often it is due to misconfiguration of Single-Sign On, either on the DataHub
-side or on the Identity Provider side. 
-
-First, verify that all values are consistent across them (e.g. the host URL where DataHub is deployed), and that no values
-are misspelled (client id, client secret). 
+side or on the Identity Provider side.
 
-Next, verify that the scopes requested are supported by your Identity Provider
-and that the claim (i.e. attribute) DataHub uses for uniquely identifying the user is supported by your Identity Provider (refer to Identity Provider OpenID Connect documentation). By default, this claim is `email`. 
+- Verify that all values are consistent across them (e.g. the host URL where DataHub is deployed), and that no values are misspelled (client id, client secret).
+- Verify that the scopes requested are supported by your Identity Provider and that the claim (i.e. attribute) DataHub uses for uniquely identifying the user is supported by your Identity Provider (refer to Identity Provider OpenID Connect documentation). By default, this claim is `email`.
+- Make sure the Discovery URI you've configured (`AUTH_OIDC_DISCOVERY_URI`) is accessible where the datahub-frontend container is running. You can do this by issuing a basic CURL to the address (**Pro-Tip**: you may also visit the address in your browser to check more specific details about your Identity Provider).
+- Check the container logs for the `datahub-frontend` container. This should hopefully provide some additional context around why exactly the login handoff is not working.
 
-Then, make sure the Discovery URI you've configured (`AUTH_OIDC_DISCOVERY_URI`) is accessible where the datahub-frontend container is running. You
-can do this by issuing a basic CURL to the address (**Pro-Tip**: you may also visit the address in your browser to check more specific details about your Identity Provider). 
+If all else fails, feel free to reach out to the DataHub Community on Slack for real-time support.
 
-Finally, check the container logs for the `datahub-frontend` container. This should hopefully provide some additional context
-around why exactly the login handoff is not working. 
+</details>
 
-If all else fails, feel free to reach out to the DataHub Community on Slack for
-real-time support 
-
-
-
-**I'm seeing an error in the `datahub-frontend` logs when a user tries to login**
-```shell
-Caused by: java.lang.RuntimeException: Failed to resolve user name claim from profile provided by Identity Provider. Missing attribute. Attribute: 'email', Regex: '(.*)', Profile: { ...
-```
-**what do I do?**
+<details>
+<summary>
+I'm seeing an error in the `datahub-frontend` logs when a user tries to login: Caused by: java.lang.RuntimeException: Failed to resolve user name claim from profile provided by Identity Provider. Missing attribute. Attribute: 'email', Regex: '(.*)', Profile: &#123; ....
+</summary>
 
 This indicates that your Identity Provider does not provide the claim with name 'email', which DataHub
 uses by default to uniquely identify users within your organization.
 
-To fix this, you may need to 
+To fix this, you may need to
 
-1. Change the claim that is used as the unique user identifier to something else by changing the `AUTH_OIDC_USER_NAME_CLAIM` (e.g. to "name" or "preferred_username") _OR_
+1. Change the claim that is used as the unique user identifier to something else by changing the `AUTH_OIDC_USER_NAME_CLAIM` (e.g. to "name" or "preferred*username") \_OR*
 2. Change the environment variable `AUTH_OIDC_SCOPE` to include the scope required to retrieve the claim with name "email"
 
-For the `datahub-frontend` container / pod. 
+For the `datahub-frontend` container / pod.
+
+</details>
+
+## Reference
 
-**Pro-Tip**: Check the documentation for your Identity Provider to learn more about the scope claims supported.
+Check the documentation for your Identity Provider to learn more about the scope claims supported.
+
+- [Registering an App in Okta](https://developer.okta.com/docs/guides/add-an-external-idp/openidconnect/main/)
+- [OpenID Connect in Google Identity](https://developers.google.com/identity/protocols/oauth2/openid-connect)
+- [OpenID Connect authentication with Azure Active Directory](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/auth-oidc)
+- [Keycloak - Securing Applications and Services Guide](https://www.keycloak.org/docs/latest/securing_apps/)

From ec9725026dca7b89d6a6464ea9b5c547debf42e5 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 2 Nov 2023 09:39:08 -0700
Subject: [PATCH 152/259] chore(ingest): remove legacy memory_leak_detector
 (#9158)

---
 .../src/datahub/cli/ingest_cli.py             |   4 -
 metadata-ingestion/src/datahub/entrypoints.py |  15 ---
 .../ingestion/source/looker/looker_config.py  |   6 +-
 .../datahub/utilities/memory_leak_detector.py | 106 ------------------
 .../tests/integration/snowflake/common.py     |   3 +-
 .../tests/unit/test_snowflake_source.py       |  15 +--
 6 files changed, 10 insertions(+), 139 deletions(-)
 delete mode 100644 metadata-ingestion/src/datahub/utilities/memory_leak_detector.py

diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py
index 9b5716408f3e4..dd0287004a368 100644
--- a/metadata-ingestion/src/datahub/cli/ingest_cli.py
+++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py
@@ -27,7 +27,6 @@
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
-from datahub.utilities import memory_leak_detector
 
 logger = logging.getLogger(__name__)
 
@@ -98,7 +97,6 @@ def ingest() -> None:
 @click.option(
     "--no-spinner", type=bool, is_flag=True, default=False, help="Turn off spinner"
 )
-@click.pass_context
 @telemetry.with_telemetry(
     capture_kwargs=[
         "dry_run",
@@ -109,9 +107,7 @@ def ingest() -> None:
         "no_spinner",
     ]
 )
-@memory_leak_detector.with_leak_detection
 def run(
-    ctx: click.Context,
     config: str,
     dry_run: bool,
     preview: bool,
diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py
index 5bfab3b841fa3..0cd37cc939854 100644
--- a/metadata-ingestion/src/datahub/entrypoints.py
+++ b/metadata-ingestion/src/datahub/entrypoints.py
@@ -70,21 +70,10 @@
     version=datahub_package.nice_version_name(),
     prog_name=datahub_package.__package_name__,
 )
-@click.option(
-    "-dl",
-    "--detect-memory-leaks",
-    type=bool,
-    is_flag=True,
-    default=False,
-    help="Run memory leak detection.",
-)
-@click.pass_context
 def datahub(
-    ctx: click.Context,
     debug: bool,
     log_file: Optional[str],
     debug_vars: bool,
-    detect_memory_leaks: bool,
 ) -> None:
     if debug_vars:
         # debug_vars implies debug. This option isn't actually used here, but instead
@@ -109,10 +98,6 @@ def datahub(
     _logging_configured = configure_logging(debug=debug, log_file=log_file)
     _logging_configured.__enter__()
 
-    # Setup the context for the memory_leak_detector decorator.
-    ctx.ensure_object(dict)
-    ctx.obj["detect_memory_leaks"] = detect_memory_leaks
-
 
 @datahub.command()
 @telemetry.with_telemetry()
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
index 96c405f7257d0..98d58c9fc9d87 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
@@ -121,7 +121,10 @@ class LookerCommonConfig(DatasetSourceConfigMixin):
         "discoverable. When disabled, adds this information to the description of the column.",
     )
     platform_name: str = Field(
-        "looker", description="Default platform name. Don't change."
+        # TODO: This shouldn't be part of the config.
+        "looker",
+        description="Default platform name.",
+        hidden_from_docs=True,
     )
     extract_column_level_lineage: bool = Field(
         True,
@@ -213,7 +216,6 @@ def external_url_defaults_to_api_config_base_url(
     def stateful_ingestion_should_be_enabled(
         cls, v: Optional[bool], *, values: Dict[str, Any], **kwargs: Dict[str, Any]
     ) -> Optional[bool]:
-
         stateful_ingestion: StatefulStaleMetadataRemovalConfig = cast(
             StatefulStaleMetadataRemovalConfig, values.get("stateful_ingestion")
         )
diff --git a/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py b/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py
deleted file mode 100644
index 85ad0fb4938eb..0000000000000
--- a/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py
+++ /dev/null
@@ -1,106 +0,0 @@
-import fnmatch
-import gc
-import logging
-import sys
-import tracemalloc
-from collections import defaultdict
-from functools import wraps
-from typing import Any, Callable, Dict, List, TypeVar, Union, cast
-
-import click
-from typing_extensions import Concatenate, ParamSpec
-
-logger = logging.getLogger(__name__)
-T = TypeVar("T")
-P = ParamSpec("P")
-
-
-def _trace_has_file(trace: tracemalloc.Traceback, file_pattern: str) -> bool:
-    for frame_index in range(len(trace)):
-        cur_frame = trace[frame_index]
-        if fnmatch.fnmatch(cur_frame.filename, file_pattern):
-            return True
-    return False
-
-
-def _init_leak_detection() -> None:
-    # Initialize trace malloc to track up to 25 stack frames.
-    tracemalloc.start(25)
-    if sys.version_info >= (3, 9):
-        # Nice to reset peak to 0. Available for versions >= 3.9.
-        tracemalloc.reset_peak()
-    # Enable leak debugging in the garbage collector.
-    gc.set_debug(gc.DEBUG_LEAK)
-
-
-def _perform_leak_detection() -> None:
-    # Log potentially useful memory usage metrics
-    logger.info(f"GC count before collect {gc.get_count()}")
-    traced_memory_size, traced_memory_peak = tracemalloc.get_traced_memory()
-    logger.info(f"Traced Memory: size={traced_memory_size}, peak={traced_memory_peak}")
-    num_unreacheable_objects = gc.collect()
-    logger.info(f"Number of unreachable objects = {num_unreacheable_objects}")
-    logger.info(f"GC count after collect {gc.get_count()}")
-
-    # Collect unique traces of all live objects in the garbage - these have potential leaks.
-    unique_traces_to_objects: Dict[
-        Union[tracemalloc.Traceback, int], List[object]
-    ] = defaultdict(list)
-    for obj in gc.garbage:
-        obj_trace = tracemalloc.get_object_traceback(obj)
-        if obj_trace is not None:
-            if _trace_has_file(obj_trace, "*datahub/*.py"):
-                # Leaking object
-                unique_traces_to_objects[obj_trace].append(obj)
-            else:
-                unique_traces_to_objects[id(obj)].append(obj)
-    logger.info("Potentially leaking objects start")
-    for key, obj_list in sorted(
-        unique_traces_to_objects.items(),
-        key=lambda item: sum(
-            [sys.getsizeof(o) for o in item[1]]
-        ),  # TODO: add support for deep sizeof
-        reverse=True,
-    ):
-        if isinstance(key, tracemalloc.Traceback):
-            obj_traceback: tracemalloc.Traceback = cast(tracemalloc.Traceback, key)
-            logger.info(
-                f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};"
-                + " Allocation Trace:\n\t"
-                + "\n\t".join(obj_traceback.format(limit=25))
-            )
-        else:
-            logger.info(
-                f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};"
-                + " No Allocation Trace available!"
-            )
-    logger.info("Potentially leaking objects end")
-
-    tracemalloc.stop()
-
-
-def with_leak_detection(
-    func: Callable[Concatenate[click.Context, P], T]
-) -> Callable[Concatenate[click.Context, P], T]:
-    @wraps(func)
-    def wrapper(ctx: click.Context, *args: P.args, **kwargs: P.kwargs) -> Any:
-        detect_leaks: bool = ctx.obj.get("detect_memory_leaks", False)
-        if detect_leaks:
-            logger.info(
-                f"Initializing memory leak detection on command: {func.__module__}.{func.__name__}"
-            )
-            _init_leak_detection()
-
-        try:
-            return func(ctx, *args, **kwargs)
-        finally:
-            if detect_leaks:
-                logger.info(
-                    f"Starting memory leak detection on command: {func.__module__}.{func.__name__}"
-                )
-                _perform_leak_detection()
-                logger.info(
-                    f"Finished memory leak detection on command: {func.__module__}.{func.__name__}"
-                )
-
-    return wrapper
diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py
index ff448eca01071..78e5499697311 100644
--- a/metadata-ingestion/tests/integration/snowflake/common.py
+++ b/metadata-ingestion/tests/integration/snowflake/common.py
@@ -565,5 +565,4 @@ def default_query_results(  # noqa: C901
                 "DOMAIN": "DATABASE",
             },
         ]
-    # Unreachable code
-    raise Exception(f"Unknown query {query}")
+    raise ValueError(f"Unexpected query: {query}")
diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py
index 888a7c0441554..aaff878b81eee 100644
--- a/metadata-ingestion/tests/unit/test_snowflake_source.py
+++ b/metadata-ingestion/tests/unit/test_snowflake_source.py
@@ -368,8 +368,7 @@ def default_query_results(query):
             return [('{"roles":"","value":""}',)]
         elif query == "select current_warehouse()":
             return [("TEST_WAREHOUSE")]
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     connection_mock = MagicMock()
     cursor_mock = MagicMock()
@@ -397,8 +396,7 @@ def query_results(query):
             ]
         elif query == 'show grants to role "PUBLIC"':
             return []
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     config = {
         "username": "user",
@@ -441,8 +439,7 @@ def query_results(query):
             return [("", "USAGE", "DATABASE", "DB1")]
         elif query == 'show grants to role "PUBLIC"':
             return []
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     setup_mock_connect(mock_connect, query_results)
 
@@ -485,8 +482,7 @@ def query_results(query):
             ]
         elif query == 'show grants to role "PUBLIC"':
             return []
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     setup_mock_connect(mock_connect, query_results)
 
@@ -536,8 +532,7 @@ def query_results(query):
                 ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY"],
                 ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES"],
             ]
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     setup_mock_connect(mock_connect, query_results)
 

From 148ad1ad9f00d6eb43d6acb270b9a90a745c8af3 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 2 Nov 2023 09:44:35 -0700
Subject: [PATCH 153/259] feat(ingest/looker): support emitting unused explores
 (#9159)

---
 .../ingestion/source/looker/looker_common.py  |  2 +-
 .../ingestion/source/looker/looker_config.py  |  4 ++
 .../source/looker/looker_lib_wrapper.py       |  7 +++
 .../ingestion/source/looker/looker_source.py  | 46 +++++++++++++------
 4 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
index 30c38720dd96c..7ca5ce49019ab 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -388,7 +388,7 @@ def _get_field_type(
 
         # if still not found, log and continue
         if type_class is None:
-            logger.info(
+            logger.debug(
                 f"The type '{native_type}' is not recognized for field type, setting as NullTypeClass.",
             )
             type_class = NullTypeClass
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
index 98d58c9fc9d87..e6ddea9a30489 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
@@ -205,6 +205,10 @@ class LookerDashboardSourceConfig(
         False,
         description="Extract looks which are not part of any Dashboard. To enable this flag the stateful_ingestion should also be enabled.",
     )
+    emit_used_explores_only: bool = Field(
+        True,
+        description="When enabled, only explores that are used by a Dashboard/Look will be ingested.",
+    )
 
     @validator("external_base_url", pre=True, always=True)
     def external_url_defaults_to_api_config_base_url(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
index b00f74b71e792..988caba1c0d74 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
@@ -59,6 +59,7 @@ class LookerAPIStats(BaseModel):
     lookml_model_calls: int = 0
     all_dashboards_calls: int = 0
     all_looks_calls: int = 0
+    all_models_calls: int = 0
     get_query_calls: int = 0
     search_looks_calls: int = 0
     search_dashboards_calls: int = 0
@@ -155,6 +156,12 @@ def dashboard(self, dashboard_id: str, fields: Union[str, List[str]]) -> Dashboa
             transport_options=self.transport_options,
         )
 
+    def all_lookml_models(self) -> Sequence[LookmlModel]:
+        self.client_stats.all_models_calls += 1
+        return self.client.all_lookml_models(
+            transport_options=self.transport_options,
+        )
+
     def lookml_model_explore(self, model: str, explore_name: str) -> LookmlModelExplore:
         self.client_stats.explore_calls += 1
         return self.client.lookml_model_explore(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
index 09683d790c14c..4a98e8874bca0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -147,9 +147,12 @@ def __init__(self, config: LookerDashboardSourceConfig, ctx: PipelineContext):
         )
         self.reporter._looker_explore_registry = self.explore_registry
         self.reporter._looker_api = self.looker_api
+
         self.reachable_look_registry = set()
 
-        self.explores_to_fetch_set: Dict[Tuple[str, str], List[str]] = {}
+        # (model, explore) -> list of charts/looks/dashboards that reference this explore
+        # The list values are used purely for debugging purposes.
+        self.reachable_explores: Dict[Tuple[str, str], List[str]] = {}
 
         # Keep stat generators to generate entity stat aspect later
         stat_generator_config: looker_usage.StatGeneratorConfig = (
@@ -378,11 +381,11 @@ def _get_input_fields_from_query(
 
         return result
 
-    def add_explore_to_fetch(self, model: str, explore: str, via: str) -> None:
-        if (model, explore) not in self.explores_to_fetch_set:
-            self.explores_to_fetch_set[(model, explore)] = []
+    def add_reachable_explore(self, model: str, explore: str, via: str) -> None:
+        if (model, explore) not in self.reachable_explores:
+            self.reachable_explores[(model, explore)] = []
 
-        self.explores_to_fetch_set[(model, explore)].append(via)
+        self.reachable_explores[(model, explore)].append(via)
 
     def _get_looker_dashboard_element(  # noqa: C901
         self, element: DashboardElement
@@ -403,7 +406,7 @@ def _get_looker_dashboard_element(  # noqa: C901
                 f"Element {element.title}: Explores added via query: {explores}"
             )
             for exp in explores:
-                self.add_explore_to_fetch(
+                self.add_reachable_explore(
                     model=element.query.model,
                     explore=exp,
                     via=f"look:{element.look_id}:query:{element.dashboard_id}",
@@ -439,7 +442,7 @@ def _get_looker_dashboard_element(  # noqa: C901
                     explores = [element.look.query.view]
                 logger.debug(f"Element {title}: Explores added via look: {explores}")
                 for exp in explores:
-                    self.add_explore_to_fetch(
+                    self.add_reachable_explore(
                         model=element.look.query.model,
                         explore=exp,
                         via=f"Look:{element.look_id}:query:{element.dashboard_id}",
@@ -483,7 +486,7 @@ def _get_looker_dashboard_element(  # noqa: C901
                 )
 
                 for exp in explores:
-                    self.add_explore_to_fetch(
+                    self.add_reachable_explore(
                         model=element.result_maker.query.model,
                         explore=exp,
                         via=f"Look:{element.look_id}:resultmaker:query",
@@ -495,7 +498,7 @@ def _get_looker_dashboard_element(  # noqa: C901
                 if filterable.view is not None and filterable.model is not None:
                     model = filterable.model
                     explores.append(filterable.view)
-                    self.add_explore_to_fetch(
+                    self.add_reachable_explore(
                         model=filterable.model,
                         explore=filterable.view,
                         via=f"Look:{element.look_id}:resultmaker:filterable",
@@ -694,20 +697,26 @@ def _make_dashboard_metadata_events(
     def _make_explore_metadata_events(
         self,
     ) -> Iterable[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]]:
+        if self.source_config.emit_used_explores_only:
+            explores_to_fetch = list(self.reachable_explores.keys())
+        else:
+            explores_to_fetch = list(self.list_all_explores())
+        explores_to_fetch.sort()
+
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=self.source_config.max_threads
         ) as async_executor:
-            self.reporter.total_explores = len(self.explores_to_fetch_set)
+            self.reporter.total_explores = len(explores_to_fetch)
 
             explore_futures = {
                 async_executor.submit(self.fetch_one_explore, model, explore): (
                     model,
                     explore,
                 )
-                for (model, explore) in self.explores_to_fetch_set
+                for (model, explore) in explores_to_fetch
             }
 
-            for future in concurrent.futures.as_completed(explore_futures):
+            for future in concurrent.futures.wait(explore_futures).done:
                 events, explore_id, start_time, end_time = future.result()
                 del explore_futures[future]
                 self.reporter.explores_scanned += 1
@@ -717,6 +726,17 @@ def _make_explore_metadata_events(
                     f"Running time of fetch_one_explore for {explore_id}: {(end_time - start_time).total_seconds()}"
                 )
 
+    def list_all_explores(self) -> Iterable[Tuple[str, str]]:
+        # returns a list of (model, explore) tuples
+
+        for model in self.looker_api.all_lookml_models():
+            if model.name is None or model.explores is None:
+                continue
+            for explore in model.explores:
+                if explore.name is None:
+                    continue
+                yield (model.name, explore.name)
+
     def fetch_one_explore(
         self, model: str, explore: str
     ) -> Tuple[
@@ -954,7 +974,7 @@ def _input_fields_from_dashboard_element(
                 )
                 if explore is not None:
                     # add this to the list of explores to finally generate metadata for
-                    self.add_explore_to_fetch(
+                    self.add_reachable_explore(
                         input_field.model, input_field.explore, entity_urn
                     )
                     entity_urn = explore.get_explore_urn(self.source_config)

From 7ff48b37aaea165ba3c3cb6f9f9f742ea2e37654 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 3 Nov 2023 10:23:37 -0500
Subject: [PATCH 154/259] refactor(policy): refactor policy locking, no
 functional difference (#9163)

---
 .../authorization/DataHubAuthorizer.java      | 111 +++++++++---------
 1 file changed, 55 insertions(+), 56 deletions(-)

diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
index e30fb93109915..f8b28f6c182a7 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
@@ -19,6 +19,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import javax.annotation.Nonnull;
@@ -55,7 +56,8 @@ public enum AuthorizationMode {
   // Maps privilege name to the associated set of policies for fast access.
   // Not concurrent data structure because writes are always against the entire thing.
   private final Map<String, List<DataHubPolicyInfo>> _policyCache = new HashMap<>(); // Shared Policy Cache.
-  private final ReadWriteLock _lockPolicyCache = new ReentrantReadWriteLock();
+  private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+  private final Lock readLock = readWriteLock.readLock();
 
   private final ScheduledExecutorService _refreshExecutorService = Executors.newScheduledThreadPool(1);
   private final PolicyRefreshRunnable _policyRefreshRunnable;
@@ -74,7 +76,7 @@ public DataHubAuthorizer(
     _systemAuthentication = Objects.requireNonNull(systemAuthentication);
     _mode = Objects.requireNonNull(mode);
     _policyEngine = new PolicyEngine(systemAuthentication, Objects.requireNonNull(entityClient));
-    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache, _lockPolicyCache);
+    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache, readWriteLock.writeLock());
     _refreshExecutorService.scheduleAtFixedRate(_policyRefreshRunnable, delayIntervalSeconds, refreshIntervalSeconds, TimeUnit.SECONDS);
   }
 
@@ -93,41 +95,30 @@ public AuthorizationResult authorize(@Nonnull final AuthorizationRequest request
 
     Optional<ResolvedEntitySpec> resolvedResourceSpec = request.getResourceSpec().map(_entitySpecResolver::resolve);
 
-    _lockPolicyCache.readLock().lock();
-    try {
-      // 1. Fetch the policies relevant to the requested privilege.
-      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>());
-
-      // 2. Evaluate each policy.
-      for (DataHubPolicyInfo policy : policiesToEvaluate) {
-        if (isRequestGranted(policy, request, resolvedResourceSpec)) {
-          // Short circuit if policy has granted privileges to this actor.
-          return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW,
-              String.format("Granted by policy with type: %s", policy.getType()));
-        }
+    // 1. Fetch the policies relevant to the requested privilege.
+    final List<DataHubPolicyInfo> policiesToEvaluate = getOrDefault(request.getPrivilege(), new ArrayList<>());
+
+    // 2. Evaluate each policy.
+    for (DataHubPolicyInfo policy : policiesToEvaluate) {
+      if (isRequestGranted(policy, request, resolvedResourceSpec)) {
+        // Short circuit if policy has granted privileges to this actor.
+        return new AuthorizationResult(request, AuthorizationResult.Type.ALLOW,
+            String.format("Granted by policy with type: %s", policy.getType()));
       }
-      return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
-    } finally {
-      _lockPolicyCache.readLock().unlock();
     }
+    return new AuthorizationResult(request, AuthorizationResult.Type.DENY,  null);
   }
 
   public List<String> getGrantedPrivileges(final String actor, final Optional<EntitySpec> resourceSpec) {
+    // 1. Fetch all policies
+    final List<DataHubPolicyInfo> policiesToEvaluate = getOrDefault(ALL, new ArrayList<>());
 
-    _lockPolicyCache.readLock().lock();
-    try {
-      // 1. Fetch all policies
-      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>());
-
-      Urn actorUrn = UrnUtils.getUrn(actor);
-      final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
+    Urn actorUrn = UrnUtils.getUrn(actor);
+    final ResolvedEntitySpec resolvedActorSpec = _entitySpecResolver.resolve(new EntitySpec(actorUrn.getEntityType(), actor));
 
-      Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
+    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-       return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
-    } finally {
-      _lockPolicyCache.readLock().unlock();
-    }
+    return _policyEngine.getGrantedPrivileges(policiesToEvaluate, resolvedActorSpec, resolvedResourceSpec);
   }
 
   /**
@@ -143,36 +134,31 @@ public AuthorizedActors authorizedActors(
     boolean allUsers = false;
     boolean allGroups = false;
 
-    _lockPolicyCache.readLock().lock();
-    try {
-      // Step 1: Find policies granting the privilege.
-      final List<DataHubPolicyInfo> policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>());
-
-      Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
+    // Step 1: Find policies granting the privilege.
+    final List<DataHubPolicyInfo> policiesToEvaluate = getOrDefault(privilege, new ArrayList<>());
 
+    Optional<ResolvedEntitySpec> resolvedResourceSpec = resourceSpec.map(_entitySpecResolver::resolve);
 
-      // Step 2: For each policy, determine whether the resource is a match.
-      for (DataHubPolicyInfo policy : policiesToEvaluate) {
-        if (!PoliciesConfig.ACTIVE_POLICY_STATE.equals(policy.getState())) {
-          // Policy is not active, skip.
-          continue;
-        }
+    // Step 2: For each policy, determine whether the resource is a match.
+    for (DataHubPolicyInfo policy : policiesToEvaluate) {
+      if (!PoliciesConfig.ACTIVE_POLICY_STATE.equals(policy.getState())) {
+        // Policy is not active, skip.
+        continue;
+      }
 
-        final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec);
+      final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec);
 
-        // Step 3: For each matching policy, add actors that are authorized.
-        authorizedUsers.addAll(matchingActors.getUsers());
-        authorizedGroups.addAll(matchingActors.getGroups());
-        if (matchingActors.allUsers()) {
-          allUsers = true;
-        }
-        if (matchingActors.allGroups()) {
-          allGroups = true;
-        }
+      // Step 3: For each matching policy, add actors that are authorized.
+      authorizedUsers.addAll(matchingActors.getUsers());
+      authorizedGroups.addAll(matchingActors.getGroups());
+      if (matchingActors.allUsers()) {
+        allUsers = true;
+      }
+      if (matchingActors.allGroups()) {
+        allGroups = true;
       }
-    } finally {
-      _lockPolicyCache.readLock().unlock();
     }
+
     // Step 4: Return all authorized users and groups.
     return new AuthorizedActors(privilege, authorizedUsers, authorizedGroups, allUsers, allGroups);
   }
@@ -234,6 +220,16 @@ private Optional<Urn> getUrnFromRequestActor(String actor) {
     }
   }
 
+  private List<DataHubPolicyInfo> getOrDefault(String key, List<DataHubPolicyInfo> defaultValue) {
+    readLock.lock();
+    try {
+      return _policyCache.getOrDefault(key, defaultValue);
+    } finally {
+      // To unlock the acquired read thread
+      readLock.unlock();
+    }
+  }
+
   /**
    * A {@link Runnable} used to periodically fetch a new instance of the policies Cache.
    *
@@ -247,7 +243,7 @@ static class PolicyRefreshRunnable implements Runnable {
     private final Authentication _systemAuthentication;
     private final PolicyFetcher _policyFetcher;
     private final Map<String, List<DataHubPolicyInfo>> _policyCache;
-    private final ReadWriteLock _lockPolicyCache;
+    private final Lock writeLock;
 
     @Override
     public void run() {
@@ -274,13 +270,16 @@ public void run() {
             return;
           }
         }
-        _lockPolicyCache.writeLock().lock();
+
+        writeLock.lock();
         try {
           _policyCache.clear();
           _policyCache.putAll(newCache);
         } finally {
-          _lockPolicyCache.writeLock().unlock();
+          // To unlock the acquired write thread
+          writeLock.unlock();
         }
+
         log.debug(String.format("Successfully fetched %s policies.", total));
       } catch (Exception e) {
         log.error("Caught exception while loading Policy cache. Will retry on next scheduled attempt.", e);

From 07311115c5ca436f64fad9c685cfc586cc5d4180 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Fri, 3 Nov 2023 13:00:15 -0400
Subject: [PATCH 155/259] API test for managing access token privilege (#9167)

---
 .../tests/privileges/test_privileges.py       | 155 ++++++++++++++----
 1 file changed, 127 insertions(+), 28 deletions(-)

diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py
index 13d6b6cf3415a..740311754678e 100644
--- a/smoke-test/tests/privileges/test_privileges.py
+++ b/smoke-test/tests/privileges/test_privileges.py
@@ -52,6 +52,20 @@ def privileges_and_test_user_setup(admin_session):
     wait_for_writes_to_sync()
 
 
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
+)
+def _ensure_cant_perform_action(session, json,assertion_key):
+    action_response = session.post(
+        f"{get_frontend_url()}/api/v2/graphql", json=json)
+    action_response.raise_for_status()
+    action_data = action_response.json()
+
+    assert action_data["errors"][0]["extensions"]["code"] == 403
+    assert action_data["errors"][0]["extensions"]["type"] == "UNAUTHORIZED"
+    assert action_data["data"][assertion_key] == None
+
+
 @tenacity.retry(
     stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(sleep_sec)
 )
@@ -67,20 +81,6 @@ def _ensure_can_create_secret(session, json, urn):
     assert secret_data["data"]["createSecret"] == urn
     
 
-@tenacity.retry(
-    stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
-)
-def _ensure_cant_create_secret(session, json):
-    create_secret_response = session.post(
-        f"{get_frontend_url()}/api/v2/graphql", json=json)
-    create_secret_response.raise_for_status()
-    create_secret_data = create_secret_response.json()
-
-    assert create_secret_data["errors"][0]["extensions"]["code"] == 403
-    assert create_secret_data["errors"][0]["extensions"]["type"] == "UNAUTHORIZED"
-    assert create_secret_data["data"]["createSecret"] == None
-
-
 @tenacity.retry(
     stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(sleep_sec)
 )
@@ -99,17 +99,19 @@ def _ensure_can_create_ingestion_source(session, json):
     
 
 @tenacity.retry(
-    stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec)
+    stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(sleep_sec)
 )
-def _ensure_cant_create_ingestion_source(session, json):
-    create_source_response = session.post(
+def _ensure_can_create_access_token(session, json):
+    create_access_token_success = session.post(
         f"{get_frontend_url()}/api/v2/graphql", json=json)
-    create_source_response.raise_for_status()
-    create_source_data = create_source_response.json()
+    create_access_token_success.raise_for_status()
+    ingestion_data = create_access_token_success.json()
 
-    assert create_source_data["errors"][0]["extensions"]["code"] == 403
-    assert create_source_data["errors"][0]["extensions"]["type"] == "UNAUTHORIZED"
-    assert create_source_data["data"]["createIngestionSource"] == None
+    assert ingestion_data
+    assert ingestion_data["data"]
+    assert ingestion_data["data"]["createAccessToken"]
+    assert ingestion_data["data"]["createAccessToken"]["accessToken"] is not None
+    assert ingestion_data["data"]["createAccessToken"]["__typename"] == "AccessToken"
 
 
 @pytest.mark.dependency(depends=["test_healthchecks"])
@@ -132,7 +134,7 @@ def test_privilege_to_create_and_manage_secrets():
                 }
         },
     }
-    _ensure_cant_create_secret(user_session, create_secret)
+    _ensure_cant_perform_action(user_session, create_secret,"createSecret")
 
 
     # Assign privileges to the new user to manage secrets
@@ -166,7 +168,7 @@ def test_privilege_to_create_and_manage_secrets():
     remove_policy(policy_urn, admin_session)
 
     # Ensure user can't create secret after policy is removed
-    _ensure_cant_create_secret(user_session, create_secret)
+    _ensure_cant_perform_action(user_session, create_secret,"createSecret")
 
 
 @pytest.mark.dependency(depends=["test_healthchecks"])
@@ -182,11 +184,18 @@ def test_privilege_to_create_and_manage_ingestion_source():
             createIngestionSource(input: $input)\n}""",
         "variables": {"input":{"type":"snowflake","name":"test","config":
             {"recipe":
-                "{\"source\":{\"type\":\"snowflake\",\"config\":{\"account_id\":null,\"include_table_lineage\":true,\"include_view_lineage\":true,\"include_tables\":true,\"include_views\":true,\"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},\"stateful_ingestion\":{\"enabled\":true}}}}",
+                """{\"source\":{\"type\":\"snowflake\",\"config\":{
+                    \"account_id\":null,
+                    \"include_table_lineage\":true,
+                    \"include_view_lineage\":true,
+                    \"include_tables\":true,
+                    \"include_views\":true,
+                    \"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},
+                    \"stateful_ingestion\":{\"enabled\":true}}}}""",
                 "executorId":"default","debugMode":False,"extraArgs":[]}}},
     }
 
-    _ensure_cant_create_ingestion_source(user_session, create_ingestion_source)
+    _ensure_cant_perform_action(user_session, create_ingestion_source, "createIngestionSource")
 
 
      # Assign privileges to the new user to manage ingestion source
@@ -201,7 +210,14 @@ def test_privilege_to_create_and_manage_ingestion_source():
             updateIngestionSource(urn: $urn, input: $input)\n}""",
         "variables": {"urn":ingestion_source_urn,
               "input":{"type":"snowflake","name":"test updated",
-                       "config":{"recipe":"{\"source\":{\"type\":\"snowflake\",\"config\":{\"account_id\":null,\"include_table_lineage\":true,\"include_view_lineage\":true,\"include_tables\":true,\"include_views\":true,\"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},\"stateful_ingestion\":{\"enabled\":true}}}}",
+                       "config":{"recipe":"""{\"source\":{\"type\":\"snowflake\",\"config\":{
+                                 \"account_id\":null,
+                                 \"include_table_lineage\":true,
+                                 \"include_view_lineage\":true,
+                                 \"include_tables\":true,
+                                 \"include_views\":true,
+                                 \"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},
+                                 \"stateful_ingestion\":{\"enabled\":true}}}}""",
                         "executorId":"default","debugMode":False,"extraArgs":[]}}}
     }
 
@@ -238,4 +254,87 @@ def test_privilege_to_create_and_manage_ingestion_source():
     remove_policy(policy_urn, admin_session)
 
     # Ensure that user can't create ingestion source after policy is removed
-    _ensure_cant_create_ingestion_source(user_session, create_ingestion_source)
\ No newline at end of file
+    _ensure_cant_perform_action(user_session, create_ingestion_source, "createIngestionSource")
+
+
+@pytest.mark.dependency(depends=["test_healthchecks"])
+def test_privilege_to_create_and_manage_access_tokens():
+
+    (admin_user, admin_pass) = get_admin_credentials()
+    admin_session = login_as(admin_user, admin_pass)
+    user_session = login_as("user", "user")
+  
+
+    # Verify new user can't create access token
+    create_access_token = { 
+        "query": """mutation createAccessToken($input: CreateAccessTokenInput!) {\n 
+          createAccessToken(input: $input) {\n    accessToken\n    __typename\n  }\n}\n""",
+        "variables": {"input":{"actorUrn":"urn:li:corpuser:user",
+                               "type":"PERSONAL",
+                               "duration":"ONE_MONTH",
+                               "name":"test",
+                               "description":"test"}}
+    }
+
+    _ensure_cant_perform_action(user_session, create_access_token,"createAccessToken")
+
+
+    # Assign privileges to the new user to create and manage access tokens
+    policy_urn = create_user_policy("urn:li:corpuser:user", ["MANAGE_ACCESS_TOKENS"], admin_session)
+   
+
+    # Verify new user can create and manage access token(create, revoke)
+    # Create a access token
+    _ensure_can_create_access_token(user_session, create_access_token)
+
+
+    # List access tokens first to get token id
+    list_access_tokens = { 
+        "query": """query listAccessTokens($input: ListAccessTokenInput!) {\n 
+          listAccessTokens(input: $input) {\n
+                start\n count\n total\n tokens {\n urn\n type\n
+                id\n name\n description\n actorUrn\n ownerUrn\n  
+                createdAt\n expiresAt\n __typename\n }\n __typename\n  }\n}\n""",
+        "variables": {
+            "input":{
+                "start":0,"count":10,"filters":[{
+                    "field":"ownerUrn",
+                    "values":["urn:li:corpuser:user"]}]}
+        }
+    }
+
+    list_tokens_response = user_session.post(f"{get_frontend_url()}/api/v2/graphql", json=list_access_tokens)
+    list_tokens_response.raise_for_status()
+    list_tokens_data = list_tokens_response.json()
+
+    assert list_tokens_data
+    assert list_tokens_data["data"]
+    assert list_tokens_data["data"]["listAccessTokens"]["tokens"][0]["id"] is not None
+    
+    access_token_id = list_tokens_data["data"]["listAccessTokens"]["tokens"][0]["id"]
+
+
+    # Revoke access token
+    revoke_access_token = { 
+        "query": "mutation revokeAccessToken($tokenId: String!) {\n  revokeAccessToken(tokenId: $tokenId)\n}\n",
+        "variables": {
+            "tokenId": access_token_id
+        },
+    }
+
+    revoke_token_response = user_session.post(f"{get_frontend_url()}/api/v2/graphql", json=revoke_access_token)
+    revoke_token_response.raise_for_status()
+    revoke_token_data = revoke_token_response.json()
+
+    assert revoke_token_data
+    assert revoke_token_data["data"]
+    assert revoke_token_data["data"]["revokeAccessToken"]
+    assert revoke_token_data["data"]["revokeAccessToken"] is True
+
+
+    # Remove the policy
+    remove_policy(policy_urn, admin_session)
+
+
+    # Ensure that user can't create access token after policy is removed
+    _ensure_cant_perform_action(user_session, create_access_token,"createAccessToken")
\ No newline at end of file

From ddb4e1b5ffa01763d7d3353a506d4329faf11e25 Mon Sep 17 00:00:00 2001
From: Davi Arnaut <davi@verdesmares.com>
Date: Fri, 3 Nov 2023 10:26:11 -0700
Subject: [PATCH 156/259] fix(mysql-setup): quote database name (#9169)

---
 docker/mysql-setup/init.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/mysql-setup/init.sql b/docker/mysql-setup/init.sql
index b789329ddfd17..b6a1d47fb2a02 100644
--- a/docker/mysql-setup/init.sql
+++ b/docker/mysql-setup/init.sql
@@ -1,6 +1,6 @@
 -- create datahub database
-CREATE DATABASE IF NOT EXISTS DATAHUB_DB_NAME CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
-USE DATAHUB_DB_NAME;
+CREATE DATABASE IF NOT EXISTS `DATAHUB_DB_NAME` CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+USE `DATAHUB_DB_NAME`;
 
 -- create metadata aspect table
 create table if not exists metadata_aspect_v2 (

From c2bc41d15eed31f89076913f641298ded5219a4f Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 3 Nov 2023 12:29:31 -0500
Subject: [PATCH 157/259] fix(health): fix health check url authentication
 (#9117)

---
 .../authentication/AuthenticationRequest.java | 12 ++++
 .../filter/AuthenticationFilter.java          | 13 ++++-
 .../HealthStatusAuthenticator.java            | 55 +++++++++++++++++++
 .../src/main/resources/application.yml        |  2 +
 metadata-service/health-servlet/build.gradle  | 22 --------
 .../openapi/config/SpringWebConfig.java       |  2 -
 .../health}/HealthCheckController.java        | 30 ++++++----
 metadata-service/war/build.gradle             |  1 -
 .../webapp/WEB-INF/openapiServlet-servlet.xml |  2 +-
 settings.gradle                               |  1 -
 10 files changed, 101 insertions(+), 39 deletions(-)
 create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/HealthStatusAuthenticator.java
 delete mode 100644 metadata-service/health-servlet/build.gradle
 rename metadata-service/{health-servlet/src/main/java/com/datahub/health/controller => openapi-servlet/src/main/java/io/datahubproject/openapi/health}/HealthCheckController.java (79%)

diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authentication/AuthenticationRequest.java b/metadata-auth/auth-api/src/main/java/com/datahub/authentication/AuthenticationRequest.java
index 91f15f9d5ae61..5673bac5442b2 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authentication/AuthenticationRequest.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authentication/AuthenticationRequest.java
@@ -1,6 +1,8 @@
 package com.datahub.authentication;
 
 import com.datahub.plugins.auth.authentication.Authenticator;
+import lombok.Getter;
+
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
@@ -13,14 +15,24 @@
  * Currently, this class only hold the inbound request's headers, but could certainly be extended
  * to contain additional information like the request parameters, body, ip, etc as needed.
  */
+@Getter
 public class AuthenticationRequest {
 
   private final Map<String, String> caseInsensitiveHeaders;
 
+  private final String servletInfo;
+  private final String pathInfo;
+
   public AuthenticationRequest(@Nonnull final Map<String, String> requestHeaders) {
+    this("", "", requestHeaders);
+  }
+
+  public AuthenticationRequest(@Nonnull String servletInfo, @Nonnull String pathInfo, @Nonnull final Map<String, String> requestHeaders) {
     Objects.requireNonNull(requestHeaders);
     caseInsensitiveHeaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
     caseInsensitiveHeaders.putAll(requestHeaders);
+    this.servletInfo = servletInfo;
+    this.pathInfo = pathInfo;
   }
 
   /**
diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java
index e15918a813158..8c7b3ac8b98f0 100644
--- a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java
+++ b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java
@@ -2,6 +2,7 @@
 
 import com.datahub.authentication.authenticator.AuthenticatorChain;
 import com.datahub.authentication.authenticator.DataHubSystemAuthenticator;
+import com.datahub.authentication.authenticator.HealthStatusAuthenticator;
 import com.datahub.authentication.authenticator.NoOpAuthenticator;
 import com.datahub.authentication.token.StatefulTokenService;
 import com.datahub.plugins.PluginConstant;
@@ -29,6 +30,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -148,7 +150,7 @@ private void buildAuthenticatorChain() {
   }
 
   private AuthenticationRequest buildAuthContext(HttpServletRequest request) {
-    return new AuthenticationRequest(Collections.list(request.getHeaderNames())
+    return new AuthenticationRequest(request.getServletPath(), request.getPathInfo(), Collections.list(request.getHeaderNames())
         .stream()
         .collect(Collectors.toMap(headerName -> headerName, request::getHeader)));
   }
@@ -242,7 +244,14 @@ private void registerNativeAuthenticator(AuthenticatorChain authenticatorChain,
         final Authenticator authenticator = clazz.newInstance();
         // Successfully created authenticator. Now init and register it.
         log.debug(String.format("Initializing Authenticator with name %s", type));
-        authenticator.init(configs, authenticatorContext);
+        if (authenticator instanceof HealthStatusAuthenticator) {
+          Map<String, Object> authenticatorConfig = new HashMap<>(Map.of(SYSTEM_CLIENT_ID_CONFIG,
+                  this.configurationProvider.getAuthentication().getSystemClientId()));
+          authenticatorConfig.putAll(Optional.ofNullable(internalAuthenticatorConfig.getConfigs()).orElse(Collections.emptyMap()));
+          authenticator.init(authenticatorConfig, authenticatorContext);
+        } else {
+          authenticator.init(configs, authenticatorContext);
+        }
         log.info(String.format("Registering Authenticator with name %s", type));
         authenticatorChain.register(authenticator);
       } catch (Exception e) {
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/HealthStatusAuthenticator.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/HealthStatusAuthenticator.java
new file mode 100644
index 0000000000000..5749eacf5d25d
--- /dev/null
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/HealthStatusAuthenticator.java
@@ -0,0 +1,55 @@
+package com.datahub.authentication.authenticator;
+
+import com.datahub.authentication.Actor;
+import com.datahub.authentication.ActorType;
+import com.datahub.authentication.Authentication;
+import com.datahub.authentication.AuthenticationException;
+import com.datahub.authentication.AuthenticationRequest;
+import com.datahub.authentication.AuthenticatorContext;
+import com.datahub.plugins.auth.authentication.Authenticator;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static com.datahub.authentication.AuthenticationConstants.SYSTEM_CLIENT_ID_CONFIG;
+
+
+/**
+ * This Authenticator is used for allowing access for unauthenticated health check endpoints
+ *
+ * It exists to support load balancers, liveness/readiness checks
+ *
+ */
+@Slf4j
+public class HealthStatusAuthenticator implements Authenticator {
+  private static final Set<String> HEALTH_ENDPOINTS = Set.of(
+          "/openapi/check/",
+          "/openapi/up/"
+  );
+  private String systemClientId;
+
+  @Override
+  public void init(@Nonnull final Map<String, Object> config, @Nullable final AuthenticatorContext context) {
+    Objects.requireNonNull(config, "Config parameter cannot be null");
+    this.systemClientId = Objects.requireNonNull((String) config.get(SYSTEM_CLIENT_ID_CONFIG),
+            String.format("Missing required config %s", SYSTEM_CLIENT_ID_CONFIG));
+  }
+
+  @Override
+  public Authentication authenticate(@Nonnull AuthenticationRequest context) throws AuthenticationException {
+    Objects.requireNonNull(context);
+    if (HEALTH_ENDPOINTS.stream().anyMatch(prefix -> String.join("", context.getServletInfo(), context.getPathInfo()).startsWith(prefix))) {
+      return new Authentication(
+              new Actor(ActorType.USER, systemClientId),
+              "",
+              Collections.emptyMap()
+      );
+    }
+    throw new AuthenticationException("Authorization not allowed. Non-health check endpoint.");
+  }
+}
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index b817208672e08..91b10a75c922e 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -11,6 +11,8 @@ authentication:
         # Key used to validate incoming tokens. Should typically be the same as authentication.tokenService.signingKey
         signingKey: ${DATAHUB_TOKEN_SERVICE_SIGNING_KEY:WnEdIeTG/VVCLQqGwC/BAkqyY0k+H8NEAtWGejrBI94=}
         salt: ${DATAHUB_TOKEN_SERVICE_SALT:ohDVbJBvHHVJh9S/UA4BYF9COuNnqqVhr9MLKEGXk1O=}
+    # Required for unauthenticated health check endpoints - best not to remove.
+    - type: com.datahub.authentication.authenticator.HealthStatusAuthenticator
 
   # Normally failures are only warnings, enable this to throw them.
   logAuthenticatorExceptions: ${METADATA_SERVICE_AUTHENTICATOR_EXCEPTIONS_ENABLED:false}
diff --git a/metadata-service/health-servlet/build.gradle b/metadata-service/health-servlet/build.gradle
deleted file mode 100644
index 6095f724b3cd4..0000000000000
--- a/metadata-service/health-servlet/build.gradle
+++ /dev/null
@@ -1,22 +0,0 @@
-apply plugin: 'java'
-
-dependencies {
-
-  implementation project(':metadata-service:factories')
-
-  implementation externalDependency.guava
-  implementation externalDependency.reflections
-  implementation externalDependency.springBoot
-  implementation externalDependency.springCore
-  implementation externalDependency.springDocUI
-  implementation externalDependency.springWeb
-  implementation externalDependency.springWebMVC
-  implementation externalDependency.springBeans
-  implementation externalDependency.springContext
-  implementation externalDependency.slf4jApi
-  compileOnly externalDependency.lombok
-  implementation externalDependency.antlr4Runtime
-  implementation externalDependency.antlr4
-
-  annotationProcessor externalDependency.lombok
-}
\ No newline at end of file
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java
index 71e8c79a2275a..e4f49df90c392 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java
@@ -44,7 +44,6 @@ public GroupedOpenApi defaultOpenApiGroup() {
             .group("default")
             .packagesToExclude(
                     "io.datahubproject.openapi.operations",
-                    "com.datahub.health",
                     "io.datahubproject.openapi.health"
             ).build();
   }
@@ -55,7 +54,6 @@ public GroupedOpenApi operationsOpenApiGroup() {
             .group("operations")
             .packagesToScan(
                     "io.datahubproject.openapi.operations",
-                    "com.datahub.health",
                     "io.datahubproject.openapi.health"
             ).build();
   }
diff --git a/metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/health/HealthCheckController.java
similarity index 79%
rename from metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java
rename to metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/health/HealthCheckController.java
index c200e63e0d497..c90603bf88c31 100644
--- a/metadata-service/health-servlet/src/main/java/com/datahub/health/controller/HealthCheckController.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/health/HealthCheckController.java
@@ -1,5 +1,6 @@
-package com.datahub.health.controller;
+package io.datahubproject.openapi.health;
 
+import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import io.swagger.v3.oas.annotations.tags.Tag;
@@ -9,7 +10,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
 
 import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -27,7 +27,7 @@
 
 
 @RestController
-@RequestMapping("/check")
+@RequestMapping("/")
 @Tag(name = "HealthCheck", description = "An API for checking health of GMS and its clients.")
 public class HealthCheckController {
   @Autowired
@@ -41,6 +41,12 @@ public HealthCheckController(ConfigurationProvider config) {
         this::getElasticHealth, config.getHealthCheck().getCacheDurationSeconds(), TimeUnit.SECONDS);
   }
 
+  @GetMapping(path = "/check/ready", produces = MediaType.APPLICATION_JSON_VALUE)
+  public ResponseEntity<Boolean> getCombinedHealthCheck(String... checks) {
+    return ResponseEntity.status(getCombinedDebug(checks).getStatusCode())
+            .body(getCombinedDebug(checks).getStatusCode().is2xxSuccessful());
+  }
+
   /**
    * Combined health check endpoint for checking GMS clients.
    * For now, just checks the health of the ElasticSearch client
@@ -48,11 +54,10 @@ public HealthCheckController(ConfigurationProvider config) {
    * that component). The status code will be 200 if all components are okay, and 500 if one or more components are not
    * healthy.
    */
-  @GetMapping(path = "/ready", produces = MediaType.APPLICATION_JSON_VALUE)
-  public ResponseEntity<Map<String, ResponseEntity<String>>> getCombinedHealthCheck(String... checks) {
-
+  @GetMapping(path = "/debug/ready", produces = MediaType.APPLICATION_JSON_VALUE)
+  public ResponseEntity<Map<String, ResponseEntity<String>>> getCombinedDebug(String... checks) {
     Map<String, Supplier<ResponseEntity<String>>> healthChecks = new HashMap<>();
-    healthChecks.put("elasticsearch", this::getElasticHealthWithCache);
+    healthChecks.put("elasticsearch", this::getElasticDebugWithCache);
     // Add new components here
 
     List<String> componentsToCheck = checks != null && checks.length > 0
@@ -67,7 +72,6 @@ public ResponseEntity<Map<String, ResponseEntity<String>>> getCombinedHealthChec
               .get());
     }
 
-
     boolean isHealthy = componentHealth.values().stream().allMatch(resp -> resp.getStatusCode() == HttpStatus.OK);
     if (isHealthy) {
       return ResponseEntity.ok(componentHealth);
@@ -75,12 +79,18 @@ public ResponseEntity<Map<String, ResponseEntity<String>>> getCombinedHealthChec
     return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body(componentHealth);
   }
 
+  @GetMapping(path = "/check/elastic", produces = MediaType.APPLICATION_JSON_VALUE)
+  public ResponseEntity<Boolean> getElasticHealthWithCache() {
+    return ResponseEntity.status(getElasticDebugWithCache().getStatusCode())
+            .body(getElasticDebugWithCache().getStatusCode().is2xxSuccessful());
+  }
+
   /**
    * Checks the memoized cache for the latest elastic health check result
    * @return The ResponseEntity containing the health check result
    */
-  @GetMapping(path = "/elastic", produces = MediaType.APPLICATION_JSON_VALUE)
-  public ResponseEntity<String> getElasticHealthWithCache() {
+  @GetMapping(path = "/debug/elastic", produces = MediaType.APPLICATION_JSON_VALUE)
+  public ResponseEntity<String> getElasticDebugWithCache() {
     return this.memoizedSupplier.get();
   }
 
diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle
index 122c2b9d5357b..54e95fdcfe579 100644
--- a/metadata-service/war/build.gradle
+++ b/metadata-service/war/build.gradle
@@ -17,7 +17,6 @@ dependencies {
   runtimeOnly project(':metadata-service:servlet')
   runtimeOnly project(':metadata-service:auth-servlet-impl')
   runtimeOnly project(':metadata-service:graphql-servlet-impl')
-  runtimeOnly project(':metadata-service:health-servlet')
   runtimeOnly project(':metadata-service:openapi-servlet')
   runtimeOnly project(':metadata-service:openapi-entity-servlet')
   runtimeOnly project(':metadata-service:openapi-analytics-servlet')
diff --git a/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml b/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml
index 3077cfb062638..fb2bc6c0336cd 100644
--- a/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml
+++ b/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml
@@ -3,7 +3,7 @@
        xmlns:context="http://www.springframework.org/schema/context"
        xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd">
 
-  <context:component-scan base-package="io.datahubproject.openapi,com.datahub.health"/>
+  <context:component-scan base-package="io.datahubproject.openapi"/>
   <context:component-scan base-package="org.springdoc.webmvc.ui,org.springdoc.core,org.springdoc.webmvc.core,org.springframework.boot.autoconfigure.jackson"/>
 
 <bean id="yamlProperties" class="org.springframework.beans.factory.config.YamlPropertiesFactoryBean">
diff --git a/settings.gradle b/settings.gradle
index 52de461383b5e..d2844fe00cdbc 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -8,7 +8,6 @@ include 'metadata-service:auth-config'
 include 'metadata-service:auth-impl'
 include 'metadata-service:auth-filter'
 include 'metadata-service:auth-servlet-impl'
-include 'metadata-service:health-servlet'
 include 'metadata-service:restli-api'
 include 'metadata-service:restli-client'
 include 'metadata-service:restli-servlet-impl'

From efd73a5f5766872ebd4997bbb261d2f95d295dd6 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 3 Nov 2023 16:19:39 -0500
Subject: [PATCH 158/259] fix(elasticsearch): fix elasticsearch-setup for
 dropped 000001 index (#9074)

---
 docker/elasticsearch-setup/create-indices.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/elasticsearch-setup/create-indices.sh b/docker/elasticsearch-setup/create-indices.sh
index 343013402394f..5c4eb3ce3851e 100755
--- a/docker/elasticsearch-setup/create-indices.sh
+++ b/docker/elasticsearch-setup/create-indices.sh
@@ -129,7 +129,7 @@ function create_datahub_usage_event_aws_elasticsearch() {
   if [ $USAGE_EVENT_STATUS -eq 200 ]; then
     USAGE_EVENT_DEFINITION=$(curl "${CURL_ARGS[@]}" "$ELASTICSEARCH_URL/${PREFIX}datahub_usage_event")
     # the definition is expected to contain "datahub_usage_event-000001" string
-    if [[ $USAGE_EVENT_DEFINITION != *"datahub_usage_event-$INDEX_SUFFIX"* ]]; then
+    if [[ $USAGE_EVENT_DEFINITION != *"datahub_usage_event-"* ]]; then
       # ... if it doesn't, we need to drop it
       echo -e "\n>>> deleting invalid datahub_usage_event ..."
       curl "${CURL_ARGS[@]}" -XDELETE "$ELASTICSEARCH_URL/${PREFIX}datahub_usage_event"

From ac1a5a6d184d7991d0006bfe33d31b4471b64729 Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Sat, 4 Nov 2023 15:06:09 -0400
Subject: [PATCH 159/259] fix(tests): Origin/fix flaky glossary navigation
 cypress test (#9175)

---
 .../tests/cypress/cypress/e2e/glossary/glossary_navigation.js | 4 ++--
 .../tests/cypress/cypress/e2e/lineage/impact_analysis.js      | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
index aeceaf99be889..c6e9d93f71b8c 100644
--- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
+++ b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
@@ -27,7 +27,7 @@ describe("glossary sidebar navigation test", () => {
         cy.waitTextVisible("Moved Glossary Term!");
 
         // Ensure the new term is under the parent term group in the navigation sidebar
-        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryTermGroup).click();
+        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryTermGroup).click().wait(3000);
         cy.get('*[class^="GlossaryEntitiesList"]').contains(glossaryTerm).should("be.visible");
 
         // Move a term group from the root level to be under a parent term group
@@ -41,7 +41,7 @@ describe("glossary sidebar navigation test", () => {
         cy.waitTextVisible("Moved Term Group!");
 
         // Ensure it is no longer on the sidebar navigator at the top level but shows up under the new parent
-        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryParentGroup).click();
+        cy.get('[data-testid="glossary-browser-sidebar"]').contains(glossaryParentGroup).click().wait(3000);
         cy.get('*[class^="GlossaryEntitiesList"]').contains(glossaryTermGroup).should("be.visible");
 
         // Delete a term group
diff --git a/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js b/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js
index defb786d1fa5d..784ccf8f0f87d 100644
--- a/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js
+++ b/smoke-test/tests/cypress/cypress/e2e/lineage/impact_analysis.js
@@ -21,6 +21,10 @@ const startAtDataSetLineage = () => {
 }
 
 describe("impact analysis", () => {
+  beforeEach(() => {
+    cy.on('uncaught:exception', (err, runnable) => { return false; });
+  });
+
   it("can see 1 hop of lineage by default", () => {
     startAtDataSetLineage()
 

From 60131a85438efc3c5d75fe6d4ed4cff634792325 Mon Sep 17 00:00:00 2001
From: Alex Waldron <51122673+walter9388@users.noreply.github.com>
Date: Sat, 4 Nov 2023 19:07:00 +0000
Subject: [PATCH 160/259] fix: bad lineage link in
 `LineageGraphOnboardingConfig.tsx` (#9162)

---
 .../src/app/onboarding/config/LineageGraphOnboardingConfig.tsx  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datahub-web-react/src/app/onboarding/config/LineageGraphOnboardingConfig.tsx b/datahub-web-react/src/app/onboarding/config/LineageGraphOnboardingConfig.tsx
index 54bae6978a4a9..89a01ab3bd241 100644
--- a/datahub-web-react/src/app/onboarding/config/LineageGraphOnboardingConfig.tsx
+++ b/datahub-web-react/src/app/onboarding/config/LineageGraphOnboardingConfig.tsx
@@ -23,7 +23,7 @@ export const LineageGraphOnboardingConfig: OnboardingStep[] = [
                     <a
                         target="_blank"
                         rel="noreferrer noopener"
-                        href="https://datahubproject.io/docs/lineage/lineage-feature-guide/"
+                        href="https://datahubproject.io/docs/generated/lineage/lineage-feature-guide/"
                     >
                         here.
                     </a>

From 7cfe3c79794e2c5660c405cdb447086ed32b52f4 Mon Sep 17 00:00:00 2001
From: Sumit Patil <91715217+sumitappt@users.noreply.github.com>
Date: Sun, 5 Nov 2023 01:02:23 +0530
Subject: [PATCH 161/259] =?UTF-8?q?OBS-191=20|=20Viewing=20domains=20page?=
 =?UTF-8?q?=20should=20not=20require=20Manage=20Domains=20priv=E2=80=A6=20?=
 =?UTF-8?q?(#9156)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 datahub-web-react/src/app/shared/admin/HeaderLinks.tsx | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
index ce1ad93565ba4..3f46f35889fd1 100644
--- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
+++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
@@ -73,7 +73,6 @@ export function HeaderLinks(props: Props) {
     const showSettings = true;
     const showIngestion =
         isIngestionEnabled && me && me.platformPrivileges?.manageIngestion && me.platformPrivileges?.manageSecrets;
-    const showDomains = me?.platformPrivileges?.createDomains || me?.platformPrivileges?.manageDomains;
 
     useUpdateEducationStepIdsAllowlist(!!showIngestion, HOME_PAGE_INGESTION_ID);
 
@@ -106,7 +105,6 @@ export function HeaderLinks(props: Props) {
                                 <NavTitleDescription>View and modify your data dictionary</NavTitleDescription>
                             </Link>
                         </MenuItem>
-                        {showDomains && (
                             <MenuItem key="1">
                                 <Link to="/domains">
                                     <NavTitleContainer>
@@ -121,7 +119,6 @@ export function HeaderLinks(props: Props) {
                                     <NavTitleDescription>Manage related groups of data assets</NavTitleDescription>
                                 </Link>
                             </MenuItem>
-                        )}
                     </Menu>
                 }
             >

From 81daae815af4498192f487418941379b2170762c Mon Sep 17 00:00:00 2001
From: gaurav2733 <77378510+gaurav2733@users.noreply.github.com>
Date: Sun, 5 Nov 2023 01:16:39 +0530
Subject: [PATCH 162/259] fix: expand the stats row in search preview cards
 (#9140)

---
 .../app/entity/dashboard/shared/DashboardStatsSummary.tsx    | 3 +++
 .../src/app/entity/dataset/shared/DatasetStatsSummary.tsx    | 3 +++
 .../src/app/entity/dataset/shared/ExpandingStat.tsx          | 5 ++---
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx
index e8fb4c16aca9c..fb6364cffac8b 100644
--- a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx
+++ b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx
@@ -11,6 +11,9 @@ import ExpandingStat from '../../dataset/shared/ExpandingStat';
 
 const StatText = styled.span`
     color: ${ANTD_GRAY[8]};
+    @media (min-width: 1024px) {
+        width: 100%;
+        white-space: nowrap;
 `;
 
 const HelpIcon = styled(QuestionCircleOutlined)`
diff --git a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
index 14f550de25be7..3dcd41a3f8a41 100644
--- a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
+++ b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
@@ -12,6 +12,9 @@ import ExpandingStat from './ExpandingStat';
 
 const StatText = styled.span<{ color: string }>`
     color: ${(props) => props.color};
+    @media (min-width: 1160px) {
+        width: 100%;
+        white-space: nowrap;
 `;
 
 const PopoverContent = styled.div`
diff --git a/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx b/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx
index 8101a696bf274..4e223b6e54058 100644
--- a/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx
+++ b/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx
@@ -2,9 +2,7 @@ import React, { ReactNode, useEffect, useRef, useState } from 'react';
 import styled from 'styled-components';
 
 const ExpandingStatContainer = styled.span<{ disabled: boolean; expanded: boolean; width: string }>`
-    overflow: hidden;
-    white-space: nowrap;
-    width: ${(props) => props.width};
+    max-width: 100%;
     transition: width 250ms ease;
 `;
 
@@ -13,6 +11,7 @@ const ExpandingStat = ({
     render,
 }: {
     disabled?: boolean;
+
     render: (isExpanded: boolean) => ReactNode;
 }) => {
     const contentRef = useRef<HTMLSpanElement>(null);

From 02156662b5e7f24f3db908d4d19f8d1bb94a32b5 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 6 Nov 2023 12:47:07 -0800
Subject: [PATCH 163/259] docs(ingest): clarify adding source guide (#9161)

---
 metadata-ingestion/adding-source.md | 32 ++++++++++++++++-------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/metadata-ingestion/adding-source.md b/metadata-ingestion/adding-source.md
index a0930102c6827..6baddf6b2010d 100644
--- a/metadata-ingestion/adding-source.md
+++ b/metadata-ingestion/adding-source.md
@@ -6,7 +6,7 @@ There are two ways of adding a metadata ingestion source.
 2. You are writing the custom source for yourself and are not going to contribute back (yet).
 
 If you are going for case (1) just follow the steps 1 to 9 below. In case you are building it for yourself you can skip
-steps 4-9 (but maybe write tests and docs for yourself as well) and follow the documentation
+steps 4-8 (but maybe write tests and docs for yourself as well) and follow the documentation
 on [how to use custom ingestion sources](../docs/how/add-custom-ingestion-source.md)
 without forking Datahub.
 
@@ -27,6 +27,7 @@ from `ConfigModel`. The [file source](./src/datahub/ingestion/source/file.py) is
 We use [pydantic](https://pydantic-docs.helpmanual.io) conventions for documenting configuration flags. Use the `description` attribute to write rich documentation for your configuration field.
 
 For example, the following code:
+
 ```python
 from pydantic import Field
 from datahub.api.configuration.common import ConfigModel
@@ -49,12 +50,10 @@ generates the following documentation:
   <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/metadata-ingestion/generated_config_docs.png"/>
 </p>
 
-
 :::note
 Inline markdown or code snippets are not yet supported for field level documentation.
 :::
 
-
 ### 2. Set up the reporter
 
 The reporter interface enables the source to report statistics, warnings, failures, and other information about the run.
@@ -71,6 +70,8 @@ some [convenience methods](./src/datahub/emitter/mce_builder.py) for commonly us
 
 ### 4. Set up the dependencies
 
+Note: Steps 4-8 are only required if you intend to contribute the source back to the Datahub project.
+
 Declare the source's pip dependencies in the `plugins` variable of the [setup script](./setup.py).
 
 ### 5. Enable discoverability
@@ -119,37 +120,38 @@ from datahub.ingestion.api.decorators import (
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class FileSource(Source):
    """
-   
-   The File Source can be used to produce all kinds of metadata from a generic metadata events file. 
+
+   The File Source can be used to produce all kinds of metadata from a generic metadata events file.
    :::note
    Events in this file can be in MCE form or MCP form.
    :::
-   
+
    """
 
    ... source code goes here
 
 ```
 
-
 #### 7.2 Write custom documentation
 
-- Create a copy of [`source-docs-template.md`](./source-docs-template.md) and edit all relevant components. 
+- Create a copy of [`source-docs-template.md`](./source-docs-template.md) and edit all relevant components.
 - Name the document as `<plugin.md>` and move it to `metadata-ingestion/docs/sources/<platform>/<plugin>.md`. For example for the Kafka platform, under the `kafka` plugin, move the document to `metadata-ingestion/docs/sources/kafka/kafka.md`.
 - Add a quickstart recipe corresponding to the plugin under `metadata-ingestion/docs/sources/<platform>/<plugin>_recipe.yml`. For example, for the Kafka platform, under the `kafka` plugin, there is a quickstart recipe located at `metadata-ingestion/docs/sources/kafka/kafka_recipe.yml`.
 - To write platform-specific documentation (that is cross-plugin), write the documentation under `metadata-ingestion/docs/sources/<platform>/README.md`. For example, cross-plugin documentation for the BigQuery platform is located under `metadata-ingestion/docs/sources/bigquery/README.md`.
 
 #### 7.3 Viewing the Documentation
 
-Documentation for the source can be viewed by running the documentation generator from the `docs-website` module. 
+Documentation for the source can be viewed by running the documentation generator from the `docs-website` module.
 
 ##### Step 1: Build the Ingestion docs
+
 ```console
 # From the root of DataHub repo
 ./gradlew :metadata-ingestion:docGen
 ```
 
 If this finishes successfully, you will see output messages like:
+
 ```console
 Ingestion Documentation Generation Complete
 ############################################
@@ -170,7 +172,8 @@ Ingestion Documentation Generation Complete
 You can also find documentation files generated at `./docs/generated/ingestion/sources` relative to the root of the DataHub repo. You should be able to locate your specific source's markdown file here and investigate it to make sure things look as expected.
 
 #### Step 2: Build the Entire Documentation
-To view how this documentation looks in the browser, there is one more step. Just build the entire docusaurus page from the `docs-website` module. 
+
+To view how this documentation looks in the browser, there is one more step. Just build the entire docusaurus page from the `docs-website` module.
 
 ```console
 # From the root of DataHub repo
@@ -178,6 +181,7 @@ To view how this documentation looks in the browser, there is one more step. Jus
 ```
 
 This will generate messages like:
+
 ```console
 ...
 > Task :docs-website:yarnGenerate
@@ -219,15 +223,15 @@ BUILD SUCCESSFUL in 35s
 36 actionable tasks: 16 executed, 20 up-to-date
 ```
 
-After this you need to run the following script from the `docs-website` module. 
+After this you need to run the following script from the `docs-website` module.
+
 ```console
 cd docs-website
 npm run serve
 ```
 
-Now, browse to http://localhost:3000 or whichever port npm is running on, to browse the docs. 
-Your source should show up on the left sidebar under `Metadata Ingestion / Sources`. 
-
+Now, browse to http://localhost:3000 or whichever port npm is running on, to browse the docs.
+Your source should show up on the left sidebar under `Metadata Ingestion / Sources`.
 
 ### 8. Add SQL Alchemy mapping (if applicable)
 

From 4a4c29030c0cfd2da9eab01798bc74a94fbb8c1d Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 6 Nov 2023 12:47:24 -0800
Subject: [PATCH 164/259] chore: stop ingestion-smoke CI errors on forks
 (#9160)

---
 .github/workflows/docker-ingestion-smoke.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
index 8d52c23792857..82b57d23609a5 100644
--- a/.github/workflows/docker-ingestion-smoke.yml
+++ b/.github/workflows/docker-ingestion-smoke.yml
@@ -47,6 +47,7 @@ jobs:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

From 86d2b08d2bbecc90e9adffd250c894abe54667e7 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 6 Nov 2023 12:58:07 -0800
Subject: [PATCH 165/259] docs(ingest): inherit capabilities from superclasses
 (#9174)

---
 metadata-ingestion-modules/airflow-plugin/setup.py   |  4 ++++
 .../src/datahub/ingestion/api/decorators.py          | 12 +++++++++++-
 .../source/state/stateful_ingestion_base.py          |  8 +++++++-
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py
index a5af881022d8c..e88fc870cb333 100644
--- a/metadata-ingestion-modules/airflow-plugin/setup.py
+++ b/metadata-ingestion-modules/airflow-plugin/setup.py
@@ -101,6 +101,10 @@ def get_long_description():
     f"acryl-datahub[testing-utils]{_self_pin}",
     # Extra requirements for loading our test dags.
     "apache-airflow[snowflake]>=2.0.2",
+    # Connexion's new version breaks Airflow:
+    # See https://github.com/apache/airflow/issues/35234.
+    # TODO: We should transition to using Airflow's constraints file.
+    "connexion<3",
     # https://github.com/snowflakedb/snowflake-sqlalchemy/issues/350
     # Eventually we want to set this to "snowflake-sqlalchemy>=1.4.3".
     # However, that doesn't work with older versions of Airflow. Instead
diff --git a/metadata-ingestion/src/datahub/ingestion/api/decorators.py b/metadata-ingestion/src/datahub/ingestion/api/decorators.py
index 5e4427047104f..b390ffb9dd036 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/decorators.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/decorators.py
@@ -93,10 +93,20 @@ def capability(
     """
 
     def wrapper(cls: Type) -> Type:
-        if not hasattr(cls, "__capabilities"):
+        if not hasattr(cls, "__capabilities") or any(
+            # It's from this class and not a superclass.
+            cls.__capabilities is getattr(base, "__capabilities", None)
+            for base in cls.__bases__
+        ):
             cls.__capabilities = {}
             cls.get_capabilities = lambda: cls.__capabilities.values()
 
+            # If the superclasses have capability annotations, copy those over.
+            for base in cls.__bases__:
+                base_caps = getattr(base, "__capabilities", None)
+                if base_caps:
+                    cls.__capabilities.update(base_caps)
+
         cls.__capabilities[capability_name] = CapabilitySetting(
             capability=capability_name, description=description, supported=supported
         )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
index 7fb2cf9813cab..d11b1f9ad6a53 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
@@ -15,11 +15,12 @@
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import capability
 from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import (
     IngestionCheckpointingProviderBase,
     JobId,
 )
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceCapability, SourceReport
 from datahub.ingestion.source.state.checkpoint import Checkpoint, StateType
 from datahub.ingestion.source.state.use_case_handler import (
     StatefulIngestionUsecaseHandlerBase,
@@ -177,6 +178,11 @@ class StatefulIngestionReport(SourceReport):
     pass
 
 
+@capability(
+    SourceCapability.DELETION_DETECTION,
+    "Optionally enabled via `stateful_ingestion.remove_stale_metadata`",
+    supported=True,
+)
 class StatefulIngestionSourceBase(Source):
     """
     Defines the base class for all stateful sources.

From 2c58c63780970606e50ba95b382dc9ffbde17bfc Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Mon, 6 Nov 2023 15:58:57 -0500
Subject: [PATCH 166/259] fix(ingest/datahub-source): Order by version in
 memory (#9185)

---
 .../source/datahub/datahub_database_reader.py | 100 ++++++++++++++----
 .../tests/unit/test_datahub_source.py         |  51 +++++++++
 2 files changed, 133 insertions(+), 18 deletions(-)
 create mode 100644 metadata-ingestion/tests/unit/test_datahub_source.py

diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
index 96184d8d445e4..e4f1bb275487e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
@@ -1,9 +1,11 @@
 import json
 import logging
 from datetime import datetime
-from typing import Dict, Iterable, Optional, Tuple
+from typing import Any, Generic, Iterable, List, Optional, Tuple, TypeVar
 
 from sqlalchemy import create_engine
+from sqlalchemy.engine import Row
+from typing_extensions import Protocol
 
 from datahub.emitter.aspect import ASPECT_MAP
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -20,6 +22,62 @@
 DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
 
 
+class VersionOrderable(Protocol):
+    createdon: Any  # Should restrict to only orderable types
+    version: int
+
+
+ROW = TypeVar("ROW", bound=VersionOrderable)
+
+
+class VersionOrderer(Generic[ROW]):
+    """Orders rows by (createdon, version == 0).
+
+    That is, orders rows first by createdon, and for equal timestamps, puts version 0 rows last.
+    """
+
+    def __init__(self, enabled: bool):
+        # Stores all version 0 aspects for a given createdon timestamp
+        # Once we have emitted all aspects for a given timestamp, we can emit the version 0 aspects
+        # Guaranteeing that, for a given timestamp, we always ingest version 0 aspects last
+        self.queue: Optional[Tuple[datetime, List[ROW]]] = None
+        self.enabled = enabled
+
+    def __call__(self, rows: Iterable[ROW]) -> Iterable[ROW]:
+        for row in rows:
+            yield from self._process_row(row)
+        yield from self._flush_queue()
+
+    def _process_row(self, row: ROW) -> Iterable[ROW]:
+        if not self.enabled:
+            yield row
+            return
+
+        yield from self._attempt_queue_flush(row)
+        if row.version == 0:
+            self._add_to_queue(row)
+        else:
+            yield row
+
+    def _add_to_queue(self, row: ROW) -> None:
+        if self.queue is None:
+            self.queue = (row.createdon, [row])
+        else:
+            self.queue[1].append(row)
+
+    def _attempt_queue_flush(self, row: ROW) -> Iterable[ROW]:
+        if self.queue is None:
+            return
+
+        if row.createdon > self.queue[0]:
+            yield from self._flush_queue()
+
+    def _flush_queue(self) -> Iterable[ROW]:
+        if self.queue is not None:
+            yield from self.queue[1]
+            self.queue = None
+
+
 class DataHubDatabaseReader:
     def __init__(
         self,
@@ -40,13 +98,14 @@ def query(self) -> str:
         # Offset is generally 0, unless we repeat the same createdon twice
 
         # Ensures stable order, chronological per (urn, aspect)
-        # Version 0 last, only when createdon is the same. Otherwise relies on createdon order
+        # Relies on createdon order to reflect version order
+        # Ordering of entries with the same createdon is handled by VersionOrderer
         return f"""
-            SELECT urn, aspect, metadata, systemmetadata, createdon
+            SELECT urn, aspect, metadata, systemmetadata, createdon, version
             FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
             WHERE createdon >= %(since_createdon)s
             {"" if self.config.include_all_versions else "AND version = 0"}
-            ORDER BY createdon, urn, aspect, CASE WHEN version = 0 THEN 1 ELSE 0 END, version
+            ORDER BY createdon, urn, aspect, version
             LIMIT %(limit)s
             OFFSET %(offset)s
         """
@@ -54,6 +113,14 @@ def query(self) -> str:
     def get_aspects(
         self, from_createdon: datetime, stop_time: datetime
     ) -> Iterable[Tuple[MetadataChangeProposalWrapper, datetime]]:
+        orderer = VersionOrderer[Row](enabled=self.config.include_all_versions)
+        rows = self._get_rows(from_createdon=from_createdon, stop_time=stop_time)
+        for row in orderer(rows):
+            mcp = self._parse_row(row)
+            if mcp:
+                yield mcp, row.createdon
+
+    def _get_rows(self, from_createdon: datetime, stop_time: datetime) -> Iterable[Row]:
         with self.engine.connect() as conn:
             ts = from_createdon
             offset = 0
@@ -69,34 +136,31 @@ def get_aspects(
                     return
 
                 for i, row in enumerate(rows):
-                    row_dict = row._asdict()
-                    mcp = self._parse_row(row_dict)
-                    if mcp:
-                        yield mcp, row_dict["createdon"]
+                    yield row
 
-                if ts == row_dict["createdon"]:
-                    offset += i
+                if ts == row.createdon:
+                    offset += i + 1
                 else:
-                    ts = row_dict["createdon"]
+                    ts = row.createdon
                     offset = 0
 
-    def _parse_row(self, d: Dict) -> Optional[MetadataChangeProposalWrapper]:
+    def _parse_row(self, row: Row) -> Optional[MetadataChangeProposalWrapper]:
         try:
-            json_aspect = post_json_transform(json.loads(d["metadata"]))
-            json_metadata = post_json_transform(json.loads(d["systemmetadata"] or "{}"))
+            json_aspect = post_json_transform(json.loads(row.metadata))
+            json_metadata = post_json_transform(json.loads(row.systemmetadata or "{}"))
             system_metadata = SystemMetadataClass.from_obj(json_metadata)
             return MetadataChangeProposalWrapper(
-                entityUrn=d["urn"],
-                aspect=ASPECT_MAP[d["aspect"]].from_obj(json_aspect),
+                entityUrn=row.urn,
+                aspect=ASPECT_MAP[row.aspect].from_obj(json_aspect),
                 systemMetadata=system_metadata,
                 changeType=ChangeTypeClass.UPSERT,
             )
         except Exception as e:
             logger.warning(
-                f"Failed to parse metadata for {d['urn']}: {e}", exc_info=True
+                f"Failed to parse metadata for {row.urn}: {e}", exc_info=True
             )
             self.report.num_database_parse_errors += 1
             self.report.database_parse_errors.setdefault(
                 str(e), LossyDict()
-            ).setdefault(d["aspect"], LossyList()).append(d["urn"])
+            ).setdefault(row.aspect, LossyList()).append(row.urn)
             return None
diff --git a/metadata-ingestion/tests/unit/test_datahub_source.py b/metadata-ingestion/tests/unit/test_datahub_source.py
new file mode 100644
index 0000000000000..adc131362b326
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_datahub_source.py
@@ -0,0 +1,51 @@
+from dataclasses import dataclass
+
+import pytest
+
+from datahub.ingestion.source.datahub.datahub_database_reader import (
+    VersionOrderable,
+    VersionOrderer,
+)
+
+
+@dataclass
+class MockRow(VersionOrderable):
+    createdon: int
+    version: int
+    urn: str
+
+
+@pytest.fixture
+def rows():
+    return [
+        MockRow(0, 0, "one"),
+        MockRow(0, 1, "one"),
+        MockRow(0, 0, "two"),
+        MockRow(0, 0, "three"),
+        MockRow(0, 1, "three"),
+        MockRow(0, 2, "three"),
+        MockRow(0, 1, "two"),
+        MockRow(0, 4, "three"),
+        MockRow(0, 5, "three"),
+        MockRow(1, 6, "three"),
+        MockRow(1, 0, "four"),
+        MockRow(2, 0, "five"),
+        MockRow(2, 1, "six"),
+        MockRow(2, 0, "six"),
+        MockRow(3, 0, "seven"),
+        MockRow(3, 0, "eight"),
+    ]
+
+
+def test_version_orderer(rows):
+    orderer = VersionOrderer[MockRow](enabled=True)
+    ordered_rows = list(orderer(rows))
+    assert ordered_rows == sorted(
+        ordered_rows, key=lambda x: (x.createdon, x.version == 0)
+    )
+
+
+def test_version_orderer_disabled(rows):
+    orderer = VersionOrderer[MockRow](enabled=False)
+    ordered_rows = list(orderer(rows))
+    assert ordered_rows == rows

From f2ce3ab62cc29bd0d4d4cade2577a50a39fa0f32 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Mon, 6 Nov 2023 15:19:55 -0600
Subject: [PATCH 167/259] lint(frontend): fix HeaderLinks lint error (#9189)

---
 .../src/app/shared/admin/HeaderLinks.tsx      | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
index 3f46f35889fd1..4a7a4938ea970 100644
--- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
+++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx
@@ -105,20 +105,20 @@ export function HeaderLinks(props: Props) {
                                 <NavTitleDescription>View and modify your data dictionary</NavTitleDescription>
                             </Link>
                         </MenuItem>
-                            <MenuItem key="1">
-                                <Link to="/domains">
-                                    <NavTitleContainer>
-                                        <DomainIcon
-                                            style={{
-                                                fontSize: 14,
-                                                fontWeight: 'bold',
-                                            }}
-                                        />
-                                        <NavTitleText>Domains</NavTitleText>
-                                    </NavTitleContainer>
-                                    <NavTitleDescription>Manage related groups of data assets</NavTitleDescription>
-                                </Link>
-                            </MenuItem>
+                        <MenuItem key="1">
+                            <Link to="/domains">
+                                <NavTitleContainer>
+                                    <DomainIcon
+                                        style={{
+                                            fontSize: 14,
+                                            fontWeight: 'bold',
+                                        }}
+                                    />
+                                    <NavTitleText>Domains</NavTitleText>
+                                </NavTitleContainer>
+                                <NavTitleDescription>Manage related groups of data assets</NavTitleDescription>
+                            </Link>
+                        </MenuItem>
                     </Menu>
                 }
             >

From 34aa08b7f38d733adcfe31ca97131e1ea52b49e6 Mon Sep 17 00:00:00 2001
From: John Joyce <john@acryl.io>
Date: Mon, 6 Nov 2023 16:51:05 -0800
Subject: [PATCH 168/259] refactor(ui): Refactor entity page loading indicators
  (#9195)

unrelated smoke test failing.
---
 .../src/app/entity/EntityPage.tsx             |  4 +-
 .../containers/profile/EntityProfile.tsx      |  3 --
 .../profile/header/EntityHeader.tsx           | 46 +++++++++++--------
 .../header/EntityHeaderLoadingSection.tsx     | 29 ++++++++++++
 .../src/app/lineage/LineageExplorer.tsx       |  7 +--
 .../src/app/lineage/LineageLoadingSection.tsx | 27 +++++++++++
 6 files changed, 86 insertions(+), 30 deletions(-)
 create mode 100644 datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeaderLoadingSection.tsx
 create mode 100644 datahub-web-react/src/app/lineage/LineageLoadingSection.tsx

diff --git a/datahub-web-react/src/app/entity/EntityPage.tsx b/datahub-web-react/src/app/entity/EntityPage.tsx
index 09233dbd89f69..916fa41795412 100644
--- a/datahub-web-react/src/app/entity/EntityPage.tsx
+++ b/datahub-web-react/src/app/entity/EntityPage.tsx
@@ -8,7 +8,6 @@ import { useEntityRegistry } from '../useEntityRegistry';
 import analytics, { EventType } from '../analytics';
 import { decodeUrn } from './shared/utils';
 import { useGetGrantedPrivilegesQuery } from '../../graphql/policy.generated';
-import { Message } from '../shared/Message';
 import { UnauthorizedPage } from '../authorization/UnauthorizedPage';
 import { ErrorSection } from '../shared/error/ErrorSection';
 import { VIEW_ENTITY_PAGE } from './shared/constants';
@@ -34,7 +33,7 @@ export const EntityPage = ({ entityType }: Props) => {
     const isLineageSupported = entity.isLineageEnabled();
     const isLineageMode = useIsLineageMode();
     const authenticatedUserUrn = useUserContext()?.user?.urn;
-    const { loading, error, data } = useGetGrantedPrivilegesQuery({
+    const { error, data } = useGetGrantedPrivilegesQuery({
         variables: {
             input: {
                 actorUrn: authenticatedUserUrn as string,
@@ -71,7 +70,6 @@ export const EntityPage = ({ entityType }: Props) => {
 
     return (
         <>
-            {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
             {error && <ErrorSection />}
             {data && !canViewEntityPage && <UnauthorizedPage />}
             {canViewEntityPage &&
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
index 5384eb94429ed..74c127cb05dd9 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
@@ -4,7 +4,6 @@ import { MutationHookOptions, MutationTuple, QueryHookOptions, QueryResult } fro
 import styled from 'styled-components/macro';
 import { useHistory } from 'react-router';
 import { EntityType, Exact } from '../../../../../types.generated';
-import { Message } from '../../../../shared/Message';
 import {
     getEntityPath,
     getOnboardingStepIdsForEntityType,
@@ -274,7 +273,6 @@ export const EntityProfile = <T, U>({
                 }}
             >
                 <>
-                    {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
                     {(error && <ErrorSection />) ||
                         (!loading && (
                             <CompactProfile>
@@ -323,7 +321,6 @@ export const EntityProfile = <T, U>({
                         banner
                     />
                 )}
-                {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
                 {(error && <ErrorSection />) || (
                     <ContentContainer>
                         {isLineageMode ? (
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
index 97595a515b34d..69389f5dcf6fc 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx
@@ -16,6 +16,7 @@ import ShareButton from '../../../../../shared/share/ShareButton';
 import { capitalizeFirstLetterOnly } from '../../../../../shared/textUtil';
 import { useUserContext } from '../../../../../context/useUserContext';
 import { useEntityRegistry } from '../../../../../useEntityRegistry';
+import EntityHeaderLoadingSection from './EntityHeaderLoadingSection';
 
 const TitleWrapper = styled.div`
     display: flex;
@@ -81,7 +82,7 @@ type Props = {
 };
 
 export const EntityHeader = ({ headerDropdownItems, headerActionItems, isNameEditable, subHeader }: Props) => {
-    const { urn, entityType, entityData } = useEntityData();
+    const { urn, entityType, entityData, loading } = useEntityData();
     const refetch = useRefetch();
     const me = useUserContext();
     const platformName = getPlatformName(entityData);
@@ -99,25 +100,32 @@ export const EntityHeader = ({ headerDropdownItems, headerActionItems, isNameEdi
         <>
             <HeaderContainer data-testid="entity-header-test-id">
                 <MainHeaderContent>
-                    <PlatformContent />
-                    <TitleWrapper>
-                        <EntityName isNameEditable={canEditName} />
-                        {entityData?.deprecation?.deprecated && (
-                            <DeprecationPill
-                                urn={urn}
-                                deprecation={entityData?.deprecation}
-                                showUndeprecate
-                                refetch={refetch}
-                            />
-                        )}
-                        {entityData?.health && (
-                            <EntityHealth
-                                health={entityData.health}
-                                baseUrl={entityRegistry.getEntityUrl(entityType, urn)}
+                    {(loading && <EntityHeaderLoadingSection />) || (
+                        <>
+                            <PlatformContent />
+                            <TitleWrapper>
+                                <EntityName isNameEditable={canEditName} />
+                                {entityData?.deprecation?.deprecated && (
+                                    <DeprecationPill
+                                        urn={urn}
+                                        deprecation={entityData?.deprecation}
+                                        showUndeprecate
+                                        refetch={refetch}
+                                    />
+                                )}
+                                {entityData?.health && (
+                                    <EntityHealth
+                                        health={entityData.health}
+                                        baseUrl={entityRegistry.getEntityUrl(entityType, urn)}
+                                    />
+                                )}
+                            </TitleWrapper>
+                            <EntityCount
+                                entityCount={entityCount}
+                                displayAssetsText={entityType === EntityType.DataProduct}
                             />
-                        )}
-                    </TitleWrapper>
-                    <EntityCount entityCount={entityCount} displayAssetsText={entityType === EntityType.DataProduct} />
+                        </>
+                    )}
                 </MainHeaderContent>
                 <SideHeaderContent>
                     <TopButtonsWrapper>
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeaderLoadingSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeaderLoadingSection.tsx
new file mode 100644
index 0000000000000..bbf813804edd4
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeaderLoadingSection.tsx
@@ -0,0 +1,29 @@
+import * as React from 'react';
+import { Skeleton, Space } from 'antd';
+import styled from 'styled-components';
+import { ANTD_GRAY } from '../../../constants';
+
+const ContextSkeleton = styled(Skeleton.Input)`
+    && {
+        width: 320px;
+        border-radius: 4px;
+        background-color: ${ANTD_GRAY[3]};
+    }
+`;
+
+const NameSkeleton = styled(Skeleton.Input)`
+    && {
+        width: 240px;
+        border-radius: 4px;
+        background-color: ${ANTD_GRAY[3]};
+    }
+`;
+
+export default function EntityHeaderLoadingSection() {
+    return (
+        <Space direction="vertical">
+            <ContextSkeleton active size="small" />
+            <NameSkeleton active size="default" />
+        </Space>
+    );
+}
diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
index ed0b26bde11ef..f59d1843b8a99 100644
--- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx
+++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
@@ -3,7 +3,6 @@ import { useHistory } from 'react-router';
 import { Button, Drawer } from 'antd';
 import { InfoCircleOutlined } from '@ant-design/icons';
 import styled from 'styled-components';
-import { Message } from '../shared/Message';
 import { useEntityRegistry } from '../useEntityRegistry';
 import CompactContext from '../shared/CompactContext';
 import { EntityAndType, EntitySelectParams, FetchedEntities } from './types';
@@ -18,12 +17,10 @@ import { ErrorSection } from '../shared/error/ErrorSection';
 import usePrevious from '../shared/usePrevious';
 import { useGetLineageTimeParams } from './utils/useGetLineageTimeParams';
 import analytics, { EventType } from '../analytics';
+import LineageLoadingSection from './LineageLoadingSection';
 
 const DEFAULT_DISTANCE_FROM_TOP = 106;
 
-const LoadingMessage = styled(Message)`
-    margin-top: 10%;
-`;
 const FooterButtonGroup = styled.div`
     display: flex;
     justify-content: space-between;
@@ -167,7 +164,7 @@ export default function LineageExplorer({ urn, type }: Props) {
     return (
         <>
             {error && <ErrorSection />}
-            {loading && <LoadingMessage type="loading" content="Loading..." />}
+            {loading && <LineageLoadingSection />}
             {!!data && (
                 <div>
                     <LineageViz
diff --git a/datahub-web-react/src/app/lineage/LineageLoadingSection.tsx b/datahub-web-react/src/app/lineage/LineageLoadingSection.tsx
new file mode 100644
index 0000000000000..9d84de0c21172
--- /dev/null
+++ b/datahub-web-react/src/app/lineage/LineageLoadingSection.tsx
@@ -0,0 +1,27 @@
+import * as React from 'react';
+import styled from 'styled-components';
+import { LoadingOutlined } from '@ant-design/icons';
+import { ANTD_GRAY } from '../entity/shared/constants';
+
+const Container = styled.div`
+    height: auto;
+    width: 100%;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    background-color: rgb(250, 250, 250);
+`;
+
+const StyledLoading = styled(LoadingOutlined)`
+    font-size: 36px;
+    color: ${ANTD_GRAY[7]};
+    padding-bottom: 18px;
+]`;
+
+export default function LineageLoadingSection() {
+    return (
+        <Container>
+            <StyledLoading />
+        </Container>
+    );
+}

From 279fdd50d7870cc404a58a5c9afbf6b3c7c432ec Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Mon, 6 Nov 2023 19:51:20 -0600
Subject: [PATCH 169/259] fix(security): fix for zookeeper CVE-2023-44981
 (#9190)

---
 build.gradle                             | 4 ++--
 metadata-service/restli-api/build.gradle | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index bd282535fa13c..31e005e001cf0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,7 +1,7 @@
 buildscript {
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.45.0'
+  ext.pegasusVersion = '29.46.8'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.29'
   ext.springBootVersion = '2.7.14'
@@ -212,7 +212,7 @@ project.ext.externalDependency = [
     'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0',
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
-    'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14',
+    'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2',
     'wire': 'com.squareup.wire:wire-compiler:3.7.1',
     'charle':  'com.charleskorn.kaml:kaml:0.53.0',
     'common': 'commons-io:commons-io:2.7',
diff --git a/metadata-service/restli-api/build.gradle b/metadata-service/restli-api/build.gradle
index ed4f4118dba30..f182d11b6baeb 100644
--- a/metadata-service/restli-api/build.gradle
+++ b/metadata-service/restli-api/build.gradle
@@ -8,4 +8,10 @@ dependencies {
 
   restClientCompile spec.product.pegasus.d2
   restClientCompile spec.product.pegasus.restliClient
+
+  constraints {
+    restClientCompile(externalDependency.zookeeper) {
+      because("CVE-2023-44981")
+    }
+  }
 }
\ No newline at end of file

From ac9a0140570b3ada060ce716304f33ff62a1348a Mon Sep 17 00:00:00 2001
From: John Joyce <john@acryl.io>
Date: Mon, 6 Nov 2023 18:33:02 -0800
Subject: [PATCH 170/259] refactor(ui): Rename "dataset details" button text to
 "view details" on lineage sidebar profile (#9196)

---
 datahub-web-react/src/app/lineage/LineageExplorer.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
index f59d1843b8a99..28cd7025f51f4 100644
--- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx
+++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
@@ -217,7 +217,7 @@ export default function LineageExplorer({ urn, type }: Props) {
                                 Close
                             </Button>
                             <Button href={entityRegistry.getEntityUrl(selectedEntity.type, selectedEntity.urn)}>
-                                <InfoCircleOutlined /> {entityRegistry.getEntityName(selectedEntity.type)} Details
+                                <InfoCircleOutlined /> View details
                             </Button>
                         </FooterButtonGroup>
                     )

From 45770013c9bdaadfb49950c67a838aef879a8e8a Mon Sep 17 00:00:00 2001
From: John Joyce <john@acryl.io>
Date: Mon, 6 Nov 2023 18:33:13 -0800
Subject: [PATCH 171/259] feat(ui): Add command-k icons to search bar  (#9194)

---
 .../src/app/home/HomePageHeader.tsx           |  1 +
 datahub-web-react/src/app/search/CommandK.tsx | 29 +++++++++++++++
 .../src/app/search/SearchBar.tsx              | 37 ++++++++++++-------
 .../src/app/search/SearchHeader.tsx           |  1 +
 4 files changed, 55 insertions(+), 13 deletions(-)
 create mode 100644 datahub-web-react/src/app/search/CommandK.tsx

diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
index e5c01252a865b..0052d54f562eb 100644
--- a/datahub-web-react/src/app/home/HomePageHeader.tsx
+++ b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -276,6 +276,7 @@ export const HomePageHeader = () => {
                         combineSiblings
                         showQuickFilters
                         showViewAllResults
+                        showCommandK
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>
diff --git a/datahub-web-react/src/app/search/CommandK.tsx b/datahub-web-react/src/app/search/CommandK.tsx
new file mode 100644
index 0000000000000..13e55a0e3f266
--- /dev/null
+++ b/datahub-web-react/src/app/search/CommandK.tsx
@@ -0,0 +1,29 @@
+import React from 'react';
+import styled from 'styled-components';
+import { ANTD_GRAY } from '../entity/shared/constants';
+
+const Container = styled.div`
+    color: ${ANTD_GRAY[6]};
+    background-color: #ffffff;
+    opacity: 0.9;
+    border-color: black;
+    border-radius: 6px;
+    border: 1px solid ${ANTD_GRAY[6]};
+    padding-right: 6px;
+    padding-left: 6px;
+    margin-right: 4px;
+    margin-left: 4px;
+`;
+
+const Letter = styled.span`
+    padding: 2px;
+`;
+
+export const CommandK = () => {
+    return (
+        <Container>
+            <Letter>⌘</Letter>
+            <Letter>K</Letter>
+        </Container>
+    );
+};
diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
index 5f797e68fe0e8..a23ead83caf54 100644
--- a/datahub-web-react/src/app/search/SearchBar.tsx
+++ b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -23,6 +23,7 @@ import { navigateToSearchUrl } from './utils/navigateToSearchUrl';
 import ViewAllSearchItem from './ViewAllSearchItem';
 import { ViewSelect } from '../entity/view/select/ViewSelect';
 import { combineSiblingsInAutoComplete } from './utils/combineSiblingsInAutoComplete';
+import { CommandK } from './CommandK';
 
 const StyledAutoComplete = styled(AutoComplete)`
     width: 100%;
@@ -114,6 +115,7 @@ interface Props {
     fixAutoComplete?: boolean;
     hideRecommendations?: boolean;
     showQuickFilters?: boolean;
+    showCommandK?: boolean;
     viewsEnabled?: boolean;
     combineSiblings?: boolean;
     setIsSearchBarFocused?: (isSearchBarFocused: boolean) => void;
@@ -142,6 +144,7 @@ export const SearchBar = ({
     fixAutoComplete,
     hideRecommendations,
     showQuickFilters,
+    showCommandK = false,
     viewsEnabled = false,
     combineSiblings = false,
     setIsSearchBarFocused,
@@ -153,6 +156,8 @@ export const SearchBar = ({
     const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
     const [selected, setSelected] = useState<string>();
     const [isDropdownVisible, setIsDropdownVisible] = useState(false);
+    const [isFocused, setIsFocused] = useState(false);
+
     useEffect(() => setSelected(initialQuery), [initialQuery]);
 
     const searchEntityTypes = entityRegistry.getSearchEntityTypes();
@@ -277,11 +282,13 @@ export const SearchBar = ({
     function handleFocus() {
         if (onFocus) onFocus();
         handleSearchBarClick(true);
+        setIsFocused(true);
     }
 
     function handleBlur() {
         if (onBlur) onBlur();
         handleSearchBarClick(false);
+        setIsFocused(false);
     }
 
     function handleSearch(query: string, type?: EntityType, appliedQuickFilters?: FacetFilterInput[]) {
@@ -294,18 +301,21 @@ export const SearchBar = ({
     const searchInputRef = useRef(null);
 
     useEffect(() => {
-        const handleKeyDown = (event) => {
-            // Support command-k to select the search bar.
-            // 75 is the keyCode for 'k'
-            if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) {
-                (searchInputRef?.current as any)?.focus();
-            }
-        };
-        document.addEventListener('keydown', handleKeyDown);
-        return () => {
-            document.removeEventListener('keydown', handleKeyDown);
-        };
-    }, []);
+        if (showCommandK) {
+            const handleKeyDown = (event) => {
+                // Support command-k to select the search bar.
+                // 75 is the keyCode for 'k'
+                if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) {
+                    (searchInputRef?.current as any)?.focus();
+                }
+            };
+            document.addEventListener('keydown', handleKeyDown);
+            return () => {
+                document.removeEventListener('keydown', handleKeyDown);
+            };
+        }
+        return () => null;
+    }, [showCommandK]);
 
     return (
         <AutoCompleteContainer style={style} ref={searchBarWrapperRef}>
@@ -377,7 +387,7 @@ export const SearchBar = ({
                     data-testid="search-input"
                     onFocus={handleFocus}
                     onBlur={handleBlur}
-                    allowClear={{ clearIcon: <ClearIcon /> }}
+                    allowClear={(isFocused && { clearIcon: <ClearIcon /> }) || false}
                     prefix={
                         <>
                             {viewsEnabled && (
@@ -411,6 +421,7 @@ export const SearchBar = ({
                         </>
                     }
                     ref={searchInputRef}
+                    suffix={(showCommandK && !isFocused && <CommandK />) || null}
                 />
             </StyledAutoComplete>
         </AutoCompleteContainer>
diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx
index 91f9753a3d601..76e78a11d3e9d 100644
--- a/datahub-web-react/src/app/search/SearchHeader.tsx
+++ b/datahub-web-react/src/app/search/SearchHeader.tsx
@@ -108,6 +108,7 @@ export const SearchHeader = ({
                     fixAutoComplete
                     showQuickFilters
                     showViewAllResults
+                    showCommandK
                 />
             </LogoSearchContainer>
             <NavGroup>

From 88cde08d060041bfb6f585ed7a486f6ba5886733 Mon Sep 17 00:00:00 2001
From: Chris Collins <chriscollins3456@gmail.com>
Date: Mon, 6 Nov 2023 21:34:17 -0500
Subject: [PATCH 172/259] feat(ui): Update Apollo cache to work with union
 types (#9193)

---
 datahub-web-react/codegen.yml  |  3 ++
 datahub-web-react/package.json |  1 +
 datahub-web-react/src/App.tsx  |  3 ++
 datahub-web-react/yarn.lock    | 73 ++++++++++++++++++++++++++++++++++
 4 files changed, 80 insertions(+)

diff --git a/datahub-web-react/codegen.yml b/datahub-web-react/codegen.yml
index 96a2bd6137920..35728e8aeb7d4 100644
--- a/datahub-web-react/codegen.yml
+++ b/datahub-web-react/codegen.yml
@@ -20,6 +20,9 @@ generates:
     src/types.generated.ts:
         plugins:
             - 'typescript'
+    src/possibleTypes.generated.ts:
+        plugins:
+            - 'fragment-matcher'
     src/:
         preset: near-operation-file
         presetConfig:
diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index 2d9329919fdc1..019295f3e6ffe 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -11,6 +11,7 @@
         "@apollo/client": "^3.3.19",
         "@craco/craco": "^6.1.1",
         "@data-ui/xy-chart": "^0.0.84",
+        "@graphql-codegen/fragment-matcher": "^5.0.0",
         "@miragejs/graphql": "^0.1.11",
         "@monaco-editor/react": "^4.3.1",
         "@react-hook/window-size": "^3.0.7",
diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx
index b6bc608dccbbb..342a89f350429 100644
--- a/datahub-web-react/src/App.tsx
+++ b/datahub-web-react/src/App.tsx
@@ -36,6 +36,7 @@ import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity
 import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity';
 import { DataPlatformInstanceEntity } from './app/entity/dataPlatformInstance/DataPlatformInstanceEntity';
 import { RoleEntity } from './app/entity/Access/RoleEntity';
+import possibleTypesResult from './possibleTypes.generated';
 
 /*
     Construct Apollo Client
@@ -77,6 +78,8 @@ const client = new ApolloClient({
                 },
             },
         },
+        // need to define possibleTypes to allow us to use Apollo cache with union types
+        possibleTypes: possibleTypesResult.possibleTypes,
     }),
     credentials: 'include',
     defaultOptions: {
diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
index 590f3ebcef8c3..ce0f2f514dad1 100644
--- a/datahub-web-react/yarn.lock
+++ b/datahub-web-react/yarn.lock
@@ -2298,6 +2298,14 @@
     "@graphql-tools/utils" "^6"
     tslib "~2.0.1"
 
+"@graphql-codegen/fragment-matcher@^5.0.0":
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/@graphql-codegen/fragment-matcher/-/fragment-matcher-5.0.0.tgz#2a016715e42e8f21aa08830f34a4d0a930e660fe"
+  integrity sha512-mbash9E8eY6RSMSNrrO+C9JJEn8rdr8ORaxMpgdWL2qe2q/TlLUCE3ZvQvHkSc7GjBnMEk36LncA8ApwHR2BHg==
+  dependencies:
+    "@graphql-codegen/plugin-helpers" "^5.0.0"
+    tslib "~2.5.0"
+
 "@graphql-codegen/near-operation-file-preset@^1.17.13":
   version "1.18.6"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/near-operation-file-preset/-/near-operation-file-preset-1.18.6.tgz#2378ac75feaeaa1cfd2146bd84bf839b1fe20d9d"
@@ -2331,6 +2339,18 @@
     lodash "~4.17.0"
     tslib "~2.3.0"
 
+"@graphql-codegen/plugin-helpers@^5.0.0":
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/@graphql-codegen/plugin-helpers/-/plugin-helpers-5.0.1.tgz#e2429fcfba3f078d5aa18aa062d46c922bbb0d55"
+  integrity sha512-6L5sb9D8wptZhnhLLBcheSPU7Tg//DGWgc5tQBWX46KYTOTQHGqDpv50FxAJJOyFVJrveN9otWk9UT9/yfY4ww==
+  dependencies:
+    "@graphql-tools/utils" "^10.0.0"
+    change-case-all "1.0.15"
+    common-tags "1.8.2"
+    import-from "4.0.0"
+    lodash "~4.17.0"
+    tslib "~2.5.0"
+
 "@graphql-codegen/typescript-operations@1.17.13":
   version "1.17.13"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/typescript-operations/-/typescript-operations-1.17.13.tgz#a5b08c1573b9507ca5a9e66e795aecc40ddc5305"
@@ -2584,6 +2604,16 @@
   dependencies:
     tslib "^2.4.0"
 
+"@graphql-tools/utils@^10.0.0":
+  version "10.0.8"
+  resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-10.0.8.tgz#c7b84275ec83dc42ad9f3d4ffc424ff682075759"
+  integrity sha512-yjyA8ycSa1WRlJqyX/aLqXeE5DvF/H02+zXMUFnCzIDrj0UvLMUrxhmVFnMK0Q2n3bh4uuTeY3621m5za9ovXw==
+  dependencies:
+    "@graphql-typed-document-node/core" "^3.1.1"
+    cross-inspect "1.0.0"
+    dset "^3.1.2"
+    tslib "^2.4.0"
+
 "@graphql-tools/utils@^6":
   version "6.2.4"
   resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-6.2.4.tgz#38a2314d2e5e229ad4f78cca44e1199e18d55856"
@@ -2618,6 +2648,11 @@
   resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.1.0.tgz#0eee6373e11418bfe0b5638f654df7a4ca6a3950"
   integrity sha512-wYn6r8zVZyQJ6rQaALBEln5B1pzxb9shV5Ef97kTvn6yVGrqyXVnDqnU24MXnFubR+rZjBY9NWuxX3FB2sTsjg==
 
+"@graphql-typed-document-node/core@^3.1.1":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.2.0.tgz#5f3d96ec6b2354ad6d8a28bf216a1d97b5426861"
+  integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==
+
 "@hapi/address@2.x.x":
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5"
@@ -7001,6 +7036,22 @@ change-case-all@1.0.14:
     upper-case "^2.0.2"
     upper-case-first "^2.0.2"
 
+change-case-all@1.0.15:
+  version "1.0.15"
+  resolved "https://registry.yarnpkg.com/change-case-all/-/change-case-all-1.0.15.tgz#de29393167fc101d646cd76b0ef23e27d09756ad"
+  integrity sha512-3+GIFhk3sNuvFAJKU46o26OdzudQlPNBCu1ZQi3cMeMHhty1bhDxu2WrEilVNYaGvqUtR1VSigFcJOiS13dRhQ==
+  dependencies:
+    change-case "^4.1.2"
+    is-lower-case "^2.0.2"
+    is-upper-case "^2.0.2"
+    lower-case "^2.0.2"
+    lower-case-first "^2.0.2"
+    sponge-case "^1.0.1"
+    swap-case "^2.0.2"
+    title-case "^3.0.3"
+    upper-case "^2.0.2"
+    upper-case-first "^2.0.2"
+
 change-case@^4.1.2:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/change-case/-/change-case-4.1.2.tgz#fedfc5f136045e2398c0410ee441f95704641e12"
@@ -7357,6 +7408,11 @@ common-tags@1.8.0, common-tags@^1.8.0:
   resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937"
   integrity sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==
 
+common-tags@1.8.2:
+  version "1.8.2"
+  resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6"
+  integrity sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==
+
 commondir@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
@@ -7698,6 +7754,13 @@ cross-fetch@^3.1.5:
   dependencies:
     node-fetch "2.6.7"
 
+cross-inspect@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/cross-inspect/-/cross-inspect-1.0.0.tgz#5fda1af759a148594d2d58394a9e21364f6849af"
+  integrity sha512-4PFfn4b5ZN6FMNGSZlyb7wUhuN8wvj8t/VQHZdM4JsDcruGJ8L2kf9zao98QIrBPFCpdk27qst/AGTl7pL3ypQ==
+  dependencies:
+    tslib "^2.4.0"
+
 cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -8595,6 +8658,11 @@ dotenv@^8.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
   integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==
 
+dset@^3.1.2:
+  version "3.1.3"
+  resolved "https://registry.yarnpkg.com/dset/-/dset-3.1.3.tgz#c194147f159841148e8e34ca41f638556d9542d2"
+  integrity sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -18712,6 +18780,11 @@ tslib@~2.3.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
   integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==
 
+tslib@~2.5.0:
+  version "2.5.3"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.3.tgz#24944ba2d990940e6e982c4bea147aba80209913"
+  integrity sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==
+
 tsutils@^3.17.1:
   version "3.21.0"
   resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"

From 23c98ecf7a88d11e3b195d457ab42c763818df47 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:40:48 -0600
Subject: [PATCH 173/259] feat(policy): enable support for 10k+ policies
 (#9177)

Co-authored-by: Pedro Silva <pedro@acryl.io>
---
 .../policy/ListPoliciesResolver.java          | 26 ++----
 .../metadata/client/JavaEntityClient.java     |  2 +-
 .../metadata/search/SearchService.java        | 18 ++--
 .../authorization/DataHubAuthorizer.java      | 21 ++---
 .../datahub/authorization/PolicyFetcher.java  | 62 +++++++++++---
 .../authorization/DataHubAuthorizerTest.java  | 82 +++++++++++++------
 .../src/main/resources/application.yml        |  1 +
 .../auth/DataHubAuthorizerFactory.java        |  5 +-
 .../linkedin/entity/client/EntityClient.java  |  2 +-
 .../entity/client/RestliEntityClient.java     |  7 +-
 .../cypress/e2e/settings/managing_groups.js   |  2 +-
 11 files changed, 153 insertions(+), 75 deletions(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java
index 516d6fa2d3137..b44da1c2f832c 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java
@@ -40,23 +40,15 @@ public CompletableFuture<ListPoliciesResult> get(final DataFetchingEnvironment e
       final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount();
       final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery();
 
-      return CompletableFuture.supplyAsync(() -> {
-        try {
-          // First, get all policy Urns.
-          final PolicyFetcher.PolicyFetchResult policyFetchResult =
-              _policyFetcher.fetchPolicies(start, count, query, context.getAuthentication());
-
-          // Now that we have entities we can bind this to a result.
-          final ListPoliciesResult result = new ListPoliciesResult();
-          result.setStart(start);
-          result.setCount(count);
-          result.setTotal(policyFetchResult.getTotal());
-          result.setPolicies(mapEntities(policyFetchResult.getPolicies()));
-          return result;
-        } catch (Exception e) {
-          throw new RuntimeException("Failed to list policies", e);
-        }
-      });
+      return _policyFetcher.fetchPolicies(start, query, count, context.getAuthentication())
+              .thenApply(policyFetchResult -> {
+                final ListPoliciesResult result = new ListPoliciesResult();
+                result.setStart(start);
+                result.setCount(count);
+                result.setTotal(policyFetchResult.getTotal());
+                result.setPolicies(mapEntities(policyFetchResult.getPolicies()));
+                return result;
+              });
     }
     throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
   }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
index a69c6008fea47..dff9a22de8efd 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java
@@ -381,7 +381,7 @@ public SearchResult searchAcrossEntities(
     @Nonnull
     @Override
     public ScrollResult scrollAcrossEntities(@Nonnull List<String> entities, @Nonnull String input,
-        @Nullable Filter filter, @Nullable String scrollId, @Nonnull String keepAlive, int count,
+        @Nullable Filter filter, @Nullable String scrollId, @Nullable String keepAlive, int count,
         @Nullable SearchFlags searchFlags, @Nonnull Authentication authentication)
         throws RemoteInvocationException {
         final SearchFlags finalFlags = searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true);
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
index 94b8d57efcc16..c99e4a94feb29 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java
@@ -147,15 +147,23 @@ public SearchResult searchAcrossEntities(@Nonnull List<String> entities, @Nonnul
     return result;
   }
 
+  /**
+   * If no entities are provided, fallback to the list of non-empty entities
+   * @param inputEntities the requested entities
+   * @return some entities to search
+   */
   private List<String> getEntitiesToSearch(@Nonnull List<String> inputEntities) {
     List<String> nonEmptyEntities;
     List<String> lowercaseEntities = inputEntities.stream().map(String::toLowerCase).collect(Collectors.toList());
-    try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "getNonEmptyEntities").time()) {
-      nonEmptyEntities = _entityDocCountCache.getNonEmptyEntities();
-    }
-    if (!inputEntities.isEmpty()) {
-      nonEmptyEntities = nonEmptyEntities.stream().filter(lowercaseEntities::contains).collect(Collectors.toList());
+
+    if (lowercaseEntities.isEmpty()) {
+      try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "getNonEmptyEntities").time()) {
+        nonEmptyEntities = _entityDocCountCache.getNonEmptyEntities();
+      }
+    } else {
+      nonEmptyEntities = lowercaseEntities;
     }
+
     return nonEmptyEntities;
   }
 
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
index f8b28f6c182a7..f8f99475de23e 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
@@ -72,11 +72,13 @@ public DataHubAuthorizer(
       final EntityClient entityClient,
       final int delayIntervalSeconds,
       final int refreshIntervalSeconds,
-      final AuthorizationMode mode) {
+      final AuthorizationMode mode,
+      final int policyFetchSize) {
     _systemAuthentication = Objects.requireNonNull(systemAuthentication);
     _mode = Objects.requireNonNull(mode);
     _policyEngine = new PolicyEngine(systemAuthentication, Objects.requireNonNull(entityClient));
-    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache, readWriteLock.writeLock());
+    _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, new PolicyFetcher(entityClient), _policyCache,
+            readWriteLock.writeLock(), policyFetchSize);
     _refreshExecutorService.scheduleAtFixedRate(_policyRefreshRunnable, delayIntervalSeconds, refreshIntervalSeconds, TimeUnit.SECONDS);
   }
 
@@ -244,29 +246,28 @@ static class PolicyRefreshRunnable implements Runnable {
     private final PolicyFetcher _policyFetcher;
     private final Map<String, List<DataHubPolicyInfo>> _policyCache;
     private final Lock writeLock;
+    private final int count;
 
     @Override
     public void run() {
       try {
         // Populate new cache and swap.
         Map<String, List<DataHubPolicyInfo>> newCache = new HashMap<>();
+        Integer total = null;
+        String scrollId = null;
 
-        int start = 0;
-        int count = 30;
-        int total = 30;
-
-        while (start < total) {
+        while (total == null || scrollId != null) {
           try {
             final PolicyFetcher.PolicyFetchResult
-                policyFetchResult = _policyFetcher.fetchPolicies(start, count, _systemAuthentication);
+                    policyFetchResult = _policyFetcher.fetchPolicies(count, scrollId, _systemAuthentication);
 
             addPoliciesToCache(newCache, policyFetchResult.getPolicies());
 
             total = policyFetchResult.getTotal();
-            start = start + count;
+            scrollId = policyFetchResult.getScrollId();
           } catch (Exception e) {
             log.error(
-                "Failed to retrieve policy urns! Skipping updating policy cache until next refresh. start: {}, count: {}", start, count, e);
+                    "Failed to retrieve policy urns! Skipping updating policy cache until next refresh. count: {}, scrollId: {}", count, scrollId, e);
             return;
           }
         }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java
index 92d12bad41c9f..c06da4d245f91 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java
@@ -8,8 +8,8 @@
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.SortCriterion;
 import com.linkedin.metadata.query.filter.SortOrder;
+import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntity;
-import com.linkedin.metadata.search.SearchResult;
 import com.linkedin.policy.DataHubPolicyInfo;
 import com.linkedin.r2.RemoteInvocationException;
 import java.net.URISyntaxException;
@@ -18,11 +18,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
 
+import javax.annotation.Nullable;
+
 import static com.linkedin.metadata.Constants.DATAHUB_POLICY_INFO_ASPECT_NAME;
 import static com.linkedin.metadata.Constants.POLICY_ENTITY_NAME;
 
@@ -38,22 +41,53 @@ public class PolicyFetcher {
   private static final SortCriterion POLICY_SORT_CRITERION =
       new SortCriterion().setField("lastUpdatedTimestamp").setOrder(SortOrder.DESCENDING);
 
-  public PolicyFetchResult fetchPolicies(int start, int count, Authentication authentication)
-      throws RemoteInvocationException, URISyntaxException {
-    return fetchPolicies(start, count, "", authentication);
+  /**
+   * This is to provide a scroll implementation using the start/count api. It is not efficient
+   * and the scroll native functions should be used instead. This does fix a failure to fetch
+   * policies when deep pagination happens where there are >10k policies.
+   * Exists primarily to prevent breaking change to the graphql api.
+   */
+  @Deprecated
+  public CompletableFuture<PolicyFetchResult> fetchPolicies(int start, String query, int count, Authentication authentication) {
+    return CompletableFuture.supplyAsync(() -> {
+      try {
+        PolicyFetchResult result = PolicyFetchResult.EMPTY;
+        String scrollId = "";
+        int fetchedResults = 0;
+
+        while (PolicyFetchResult.EMPTY.equals(result) && scrollId != null) {
+          PolicyFetchResult tmpResult = fetchPolicies(query, count, scrollId.isEmpty() ? null : scrollId, authentication);
+          fetchedResults += tmpResult.getPolicies().size();
+          scrollId = tmpResult.getScrollId();
+          if (fetchedResults > start) {
+            result = tmpResult;
+          }
+        }
+
+        return result;
+      } catch (Exception e) {
+        throw new RuntimeException("Failed to list policies", e);
+      }
+    });
   }
 
-  public PolicyFetchResult fetchPolicies(int start, int count, String query, Authentication authentication)
+  public PolicyFetchResult fetchPolicies(int count, @Nullable String scrollId, Authentication authentication)
+          throws RemoteInvocationException, URISyntaxException {
+    return fetchPolicies("", count, scrollId, authentication);
+  }
+
+  public PolicyFetchResult fetchPolicies(String query, int count, @Nullable String scrollId, Authentication authentication)
       throws RemoteInvocationException, URISyntaxException {
-    log.debug(String.format("Batch fetching policies. start: %s, count: %s ", start, count));
-    // First fetch all policy urns from start - start + count
-    SearchResult result =
-        _entityClient.search(POLICY_ENTITY_NAME, query, null, POLICY_SORT_CRITERION, start, count, authentication,
-            new SearchFlags().setFulltext(true));
+    log.debug(String.format("Batch fetching policies. count: %s, scroll: %s", count, scrollId));
+
+    // First fetch all policy urns
+    ScrollResult result = _entityClient.scrollAcrossEntities(List.of(POLICY_ENTITY_NAME), query, null, scrollId,
+            null, count, new SearchFlags().setSkipCache(true).setSkipAggregates(true)
+                    .setSkipHighlighting(true).setFulltext(true), authentication);
     List<Urn> policyUrns = result.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList());
 
     if (policyUrns.isEmpty()) {
-      return new PolicyFetchResult(Collections.emptyList(), 0);
+      return PolicyFetchResult.EMPTY;
     }
 
     // Fetch DataHubPolicyInfo aspects for each urn
@@ -64,7 +98,7 @@ public PolicyFetchResult fetchPolicies(int start, int count, String query, Authe
         .filter(Objects::nonNull)
         .map(this::extractPolicy)
         .filter(Objects::nonNull)
-        .collect(Collectors.toList()), result.getNumEntities());
+        .collect(Collectors.toList()), result.getNumEntities(), result.getScrollId());
   }
 
   private Policy extractPolicy(EntityResponse entityResponse) {
@@ -82,6 +116,10 @@ private Policy extractPolicy(EntityResponse entityResponse) {
   public static class PolicyFetchResult {
     List<Policy> policies;
     int total;
+    @Nullable
+    String scrollId;
+
+    public static final PolicyFetchResult EMPTY = new PolicyFetchResult(Collections.emptyList(), 0, null);
   }
 
   @Value
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
index 24ecfa6fefc85..babb1c5d00ee8 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
@@ -22,6 +22,7 @@
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.query.SearchFlags;
+import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.search.SearchResult;
@@ -35,6 +36,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -89,30 +92,58 @@ public void setupTest() throws Exception {
     final EnvelopedAspectMap childDomainPolicyAspectMap = new EnvelopedAspectMap();
     childDomainPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(childDomainPolicy.data())));
 
-    final SearchResult policySearchResult = new SearchResult();
-    policySearchResult.setNumEntities(3);
-    policySearchResult.setEntities(
-        new SearchEntityArray(
-            ImmutableList.of(
-                new SearchEntity().setEntity(activePolicyUrn),
-                new SearchEntity().setEntity(inactivePolicyUrn),
-                new SearchEntity().setEntity(parentDomainPolicyUrn),
-                new SearchEntity().setEntity(childDomainPolicyUrn)
-            )
-        )
-    );
-
-    when(_entityClient.search(eq("dataHubPolicy"), eq(""), isNull(), any(), anyInt(), anyInt(), any(),
-        eq(new SearchFlags().setFulltext(true)))).thenReturn(policySearchResult);
-    when(_entityClient.batchGetV2(eq(POLICY_ENTITY_NAME),
-        eq(ImmutableSet.of(activePolicyUrn, inactivePolicyUrn, parentDomainPolicyUrn, childDomainPolicyUrn)), eq(null), any())).thenReturn(
-        ImmutableMap.of(
-            activePolicyUrn, new EntityResponse().setUrn(activePolicyUrn).setAspects(activeAspectMap),
-            inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(inactiveAspectMap),
-            parentDomainPolicyUrn, new EntityResponse().setUrn(parentDomainPolicyUrn).setAspects(parentDomainPolicyAspectMap),
-            childDomainPolicyUrn, new EntityResponse().setUrn(childDomainPolicyUrn).setAspects(childDomainPolicyAspectMap)
-        )
-    );
+    final ScrollResult policySearchResult1 = new ScrollResult()
+            .setScrollId("1")
+            .setNumEntities(4)
+            .setEntities(
+                    new SearchEntityArray(
+                            ImmutableList.of(new SearchEntity().setEntity(activePolicyUrn))));
+
+    final ScrollResult policySearchResult2 = new ScrollResult()
+            .setScrollId("2")
+            .setNumEntities(4)
+            .setEntities(
+                    new SearchEntityArray(
+                            ImmutableList.of(new SearchEntity().setEntity(inactivePolicyUrn))));
+
+    final ScrollResult policySearchResult3 = new ScrollResult()
+            .setScrollId("3")
+            .setNumEntities(4)
+            .setEntities(
+                    new SearchEntityArray(
+                            ImmutableList.of(new SearchEntity().setEntity(parentDomainPolicyUrn))));
+
+    final ScrollResult policySearchResult4 = new ScrollResult()
+            .setNumEntities(4)
+            .setEntities(
+                    new SearchEntityArray(
+                            ImmutableList.of(
+                                    new SearchEntity().setEntity(childDomainPolicyUrn))));
+
+    when(_entityClient.scrollAcrossEntities(eq(List.of("dataHubPolicy")), eq(""), isNull(), any(), isNull(),
+            anyInt(), eq(new SearchFlags().setFulltext(true).setSkipAggregates(true).setSkipHighlighting(true).setSkipCache(true)), any()))
+            .thenReturn(policySearchResult1)
+            .thenReturn(policySearchResult2)
+            .thenReturn(policySearchResult3)
+            .thenReturn(policySearchResult4);
+
+    when(_entityClient.batchGetV2(eq(POLICY_ENTITY_NAME), any(), eq(null), any())).thenAnswer(args -> {
+      Set<Urn> inputUrns = args.getArgument(1);
+      Urn urn = inputUrns.stream().findFirst().get();
+
+      switch (urn.toString()) {
+        case "urn:li:dataHubPolicy:0":
+          return Map.of(activePolicyUrn, new EntityResponse().setUrn(activePolicyUrn).setAspects(activeAspectMap));
+        case "urn:li:dataHubPolicy:1":
+          return Map.of(inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(inactiveAspectMap));
+        case "urn:li:dataHubPolicy:2":
+          return Map.of(parentDomainPolicyUrn, new EntityResponse().setUrn(parentDomainPolicyUrn).setAspects(parentDomainPolicyAspectMap));
+        case "urn:li:dataHubPolicy:3":
+          return Map.of(childDomainPolicyUrn, new EntityResponse().setUrn(childDomainPolicyUrn).setAspects(childDomainPolicyAspectMap));
+        default:
+          throw new IllegalStateException();
+      }
+    });
 
     final List<Urn> userUrns = ImmutableList.of(Urn.createFromString("urn:li:corpuser:user3"), Urn.createFromString("urn:li:corpuser:user4"));
     final List<Urn> groupUrns = ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group3"), Urn.createFromString("urn:li:corpGroup:group4"));
@@ -146,7 +177,8 @@ childDomainPolicyUrn, new EntityResponse().setUrn(childDomainPolicyUrn).setAspec
         _entityClient,
         10,
         10,
-        DataHubAuthorizer.AuthorizationMode.DEFAULT
+        DataHubAuthorizer.AuthorizationMode.DEFAULT,
+        1 // force pagination logic
     );
     _dataHubAuthorizer.init(Collections.emptyMap(), createAuthorizerContext(systemAuthentication, _entityClient));
     _dataHubAuthorizer.invalidateCache();
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 91b10a75c922e..e9113d339e81d 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -39,6 +39,7 @@ authorization:
   defaultAuthorizer:
     enabled: ${AUTH_POLICIES_ENABLED:true}
     cacheRefreshIntervalSecs: ${POLICY_CACHE_REFRESH_INTERVAL_SECONDS:120}
+    cachePolicyFetchSize: ${POLICY_CACHE_FETCH_SIZE:1000}
   # Enables authorization of reads, writes, and deletes on REST APIs. Defaults to false for backwards compatibility, but should become true down the road
   restApiAuthorization: ${REST_API_AUTHORIZATION_ENABLED:false}
 
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java
index 5b298a453547a..663234e2519fa 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java
@@ -32,6 +32,9 @@ public class DataHubAuthorizerFactory {
   @Value("${authorization.defaultAuthorizer.cacheRefreshIntervalSecs}")
   private Integer policyCacheRefreshIntervalSeconds;
 
+  @Value("${authorization.defaultAuthorizer.cachePolicyFetchSize}")
+  private Integer policyCacheFetchSize;
+
   @Value("${authorization.defaultAuthorizer.enabled:true}")
   private Boolean policiesEnabled;
 
@@ -44,6 +47,6 @@ protected DataHubAuthorizer getInstance() {
         : DataHubAuthorizer.AuthorizationMode.ALLOW_ALL;
 
     return new DataHubAuthorizer(systemAuthentication, entityClient, 10,
-        policyCacheRefreshIntervalSeconds, mode);
+        policyCacheRefreshIntervalSeconds, mode, policyCacheFetchSize);
   }
 }
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java
index b9661ec75e1b1..84d0ed6b9594d 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java
@@ -241,7 +241,7 @@ public SearchResult searchAcrossEntities(@Nonnull List<String> entities, @Nonnul
    */
   @Nonnull
   ScrollResult scrollAcrossEntities(@Nonnull List<String> entities, @Nonnull String input,
-      @Nullable Filter filter, @Nullable String scrollId, @Nonnull String keepAlive, int count, @Nullable SearchFlags searchFlags,
+      @Nullable Filter filter, @Nullable String scrollId, @Nullable String keepAlive, int count, @Nullable SearchFlags searchFlags,
       @Nonnull Authentication authentication)
       throws RemoteInvocationException;
 
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
index 47a00e711a935..2716e27518fcc 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java
@@ -482,11 +482,11 @@ public SearchResult searchAcrossEntities(@Nonnull List<String> entities, @Nonnul
   @Nonnull
   @Override
   public ScrollResult scrollAcrossEntities(@Nonnull List<String> entities, @Nonnull String input,
-      @Nullable Filter filter, @Nullable String scrollId, @Nonnull String keepAlive, int count,
+      @Nullable Filter filter, @Nullable String scrollId, @Nullable String keepAlive, int count,
       @Nullable SearchFlags searchFlags, @Nonnull Authentication authentication)
       throws RemoteInvocationException {
     final EntitiesDoScrollAcrossEntitiesRequestBuilder requestBuilder =
-        ENTITIES_REQUEST_BUILDERS.actionScrollAcrossEntities().inputParam(input).countParam(count).keepAliveParam(keepAlive);
+        ENTITIES_REQUEST_BUILDERS.actionScrollAcrossEntities().inputParam(input).countParam(count);
 
     if (entities != null) {
       requestBuilder.entitiesParam(new StringArray(entities));
@@ -500,6 +500,9 @@ public ScrollResult scrollAcrossEntities(@Nonnull List<String> entities, @Nonnul
     if (searchFlags != null) {
       requestBuilder.searchFlagsParam(searchFlags);
     }
+    if (keepAlive != null) {
+      requestBuilder.keepAliveParam(keepAlive);
+    }
 
     return sendClientRequest(requestBuilder, authentication).getEntity();
   }
diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js
index 9559435ff01c8..8d689c7e2303c 100644
--- a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js
+++ b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js
@@ -81,7 +81,7 @@ describe("create and manage group", () => {
         cy.focused().type(expected_name);
         cy.get(".ant-select-item-option").contains(expected_name, { matchCase: false }).click();
         cy.focused().blur();
-        cy.contains(expected_name).should("have.length", 1);
+        cy.contains(expected_name, { matchCase: false }).should("have.length", 1);
         cy.get('[role="dialog"] button').contains("Done").click();
         cy.waitTextVisible("Owners Added");
         cy.contains(expected_name, { matchCase: false }).should("be.visible");

From 353584c10acbee7554c2eb255512173f24e86785 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Tue, 7 Nov 2023 18:22:18 -0600
Subject: [PATCH 174/259] feat(browsepathv2): Allow system-update to reprocess
 browse paths v2 (#9200)

---
 .../steps/BackfillBrowsePathsV2Step.java      | 86 ++++++++++++++-----
 .../env/docker-without-neo4j.env              |  1 +
 docker/datahub-upgrade/env/docker.env         |  1 +
 docker/docker-compose.dev.yml                 |  4 +
 .../docker-compose-m1.quickstart.yml          |  1 +
 ...er-compose-without-neo4j-m1.quickstart.yml |  1 +
 ...ocker-compose-without-neo4j.quickstart.yml |  1 +
 .../quickstart/docker-compose.quickstart.yml  |  1 +
 .../client/CachingEntitySearchService.java    | 16 ++--
 .../elasticsearch/query/ESSearchDAO.java      |  4 +-
 .../query/request/SearchRequestHandler.java   |  8 +-
 .../src/main/resources/application.yml        |  2 +
 .../metadata/search/EntitySearchService.java  |  4 +-
 13 files changed, 94 insertions(+), 36 deletions(-)

diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java
index 7547186ccfb23..08a752d9597f4 100644
--- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java
+++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java
@@ -6,6 +6,7 @@
 import com.linkedin.common.BrowsePathsV2;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.upgrade.UpgradeContext;
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
@@ -13,6 +14,7 @@
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
@@ -37,6 +39,8 @@
 public class BackfillBrowsePathsV2Step implements UpgradeStep {
 
   public static final String BACKFILL_BROWSE_PATHS_V2 = "BACKFILL_BROWSE_PATHS_V2";
+  public static final String REPROCESS_DEFAULT_BROWSE_PATHS_V2 = "REPROCESS_DEFAULT_BROWSE_PATHS_V2";
+  public static final String DEFAULT_BROWSE_PATH_V2 = "␟Default";
 
   private static final Set<String> ENTITY_TYPES_TO_MIGRATE = ImmutableSet.of(
       Constants.DATASET_ENTITY_NAME,
@@ -81,27 +85,14 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
 
   private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId) {
 
-    // Condition: has `browsePaths` AND does NOT have `browsePathV2`
-    Criterion missingBrowsePathV2 = new Criterion();
-    missingBrowsePathV2.setCondition(Condition.IS_NULL);
-    missingBrowsePathV2.setField("browsePathV2");
-    // Excludes entities without browsePaths
-    Criterion hasBrowsePathV1 = new Criterion();
-    hasBrowsePathV1.setCondition(Condition.EXISTS);
-    hasBrowsePathV1.setField("browsePaths");
-
-    CriterionArray criterionArray = new CriterionArray();
-    criterionArray.add(missingBrowsePathV2);
-    criterionArray.add(hasBrowsePathV1);
-
-    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
-    conjunctiveCriterion.setAnd(criterionArray);
+    final Filter filter;
 
-    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
-    conjunctiveCriterionArray.add(conjunctiveCriterion);
-
-    Filter filter = new Filter();
-    filter.setOr(conjunctiveCriterionArray);
+    if (System.getenv().containsKey(REPROCESS_DEFAULT_BROWSE_PATHS_V2)
+            && Boolean.parseBoolean(System.getenv(REPROCESS_DEFAULT_BROWSE_PATHS_V2))) {
+      filter = backfillDefaultBrowsePathsV2Filter();
+    } else  {
+      filter = backfillBrowsePathsV2Filter();
+    }
 
     final ScrollResult scrollResult = _searchService.scrollAcrossEntities(
         ImmutableList.of(entityType),
@@ -109,9 +100,9 @@ private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, S
         filter,
         null,
         scrollId,
-        "5m",
+        null,
         BATCH_SIZE,
-        null
+        new SearchFlags().setFulltext(true).setSkipCache(true).setSkipHighlighting(true).setSkipAggregates(true)
     );
     if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) {
       return null;
@@ -129,6 +120,55 @@ private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, S
     return scrollResult.getScrollId();
   }
 
+  private Filter backfillBrowsePathsV2Filter() {
+    // Condition: has `browsePaths` AND does NOT have `browsePathV2`
+    Criterion missingBrowsePathV2 = new Criterion();
+    missingBrowsePathV2.setCondition(Condition.IS_NULL);
+    missingBrowsePathV2.setField("browsePathV2");
+    // Excludes entities without browsePaths
+    Criterion hasBrowsePathV1 = new Criterion();
+    hasBrowsePathV1.setCondition(Condition.EXISTS);
+    hasBrowsePathV1.setField("browsePaths");
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(missingBrowsePathV2);
+    criterionArray.add(hasBrowsePathV1);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+    return filter;
+  }
+
+  private Filter backfillDefaultBrowsePathsV2Filter() {
+    // Condition: has default `browsePathV2`
+    Criterion hasDefaultBrowsePathV2 = new Criterion();
+    hasDefaultBrowsePathV2.setCondition(Condition.EQUAL);
+    hasDefaultBrowsePathV2.setField("browsePathV2");
+    StringArray values = new StringArray();
+    values.add(DEFAULT_BROWSE_PATH_V2);
+    hasDefaultBrowsePathV2.setValues(values);
+    hasDefaultBrowsePathV2.setValue(DEFAULT_BROWSE_PATH_V2); // not used, but required field?
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(hasDefaultBrowsePathV2);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+    return filter;
+  }
+
   private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exception {
     BrowsePathsV2 browsePathsV2 = _entityService.buildDefaultBrowsePathV2(urn, true);
     log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2));
@@ -142,7 +182,7 @@ private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exceptio
     _entityService.ingestProposal(
         proposal,
         auditStamp,
-        false
+        true
     );
   }
 
diff --git a/docker/datahub-upgrade/env/docker-without-neo4j.env b/docker/datahub-upgrade/env/docker-without-neo4j.env
index c399f71b7b15c..04d888f076cd6 100644
--- a/docker/datahub-upgrade/env/docker-without-neo4j.env
+++ b/docker/datahub-upgrade/env/docker-without-neo4j.env
@@ -21,6 +21,7 @@ DATAHUB_GMS_PORT=8080
 
 ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
 BACKFILL_BROWSE_PATHS_V2=true
+REPROCESS_DEFAULT_BROWSE_PATHS_V2=${REPROCESS_DEFAULT_BROWSE_PATHS_V2:-false}
 
 # Uncomment and set these to support SSL connection to Elasticsearch
 # ELASTICSEARCH_USE_SSL=
diff --git a/docker/datahub-upgrade/env/docker.env b/docker/datahub-upgrade/env/docker.env
index 491470406153b..b2a0d01e5d4ae 100644
--- a/docker/datahub-upgrade/env/docker.env
+++ b/docker/datahub-upgrade/env/docker.env
@@ -25,6 +25,7 @@ DATAHUB_GMS_PORT=8080
 
 ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
 BACKFILL_BROWSE_PATHS_V2=true
+REPROCESS_DEFAULT_BROWSE_PATHS_V2=${REPROCESS_DEFAULT_BROWSE_PATHS_V2:-false}
 
 # Uncomment and set these to support SSL connection to Elasticsearch
 # ELASTICSEARCH_USE_SSL=
diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
index c4e5ee7fa0cae..774c4e17bee21 100644
--- a/docker/docker-compose.dev.yml
+++ b/docker/docker-compose.dev.yml
@@ -54,6 +54,8 @@ services:
     - ${HOME}/.datahub/plugins:/etc/datahub/plugins
   datahub-upgrade:
     image: acryldata/datahub-upgrade:debug
+    ports:
+      - ${DATAHUB_MAPPED_UPGRADE_DEBUG_PORT:-5003}:5003
     build:
       context: datahub-upgrade
       dockerfile: Dockerfile
@@ -63,6 +65,8 @@ services:
     - SKIP_ELASTICSEARCH_CHECK=false
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-dev}
     - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
+    - REPROCESS_DEFAULT_BROWSE_PATHS_V2=${REPROCESS_DEFAULT_BROWSE_PATHS_V2:-false}
+    - JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5003
     volumes:
     - ../datahub-upgrade/build/libs/:/datahub/datahub-upgrade/bin/
     - ../metadata-models/src/main/resources/:/datahub/datahub-gms/resources
diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml
index 3b6d02c83d0f0..c96baf37551b2 100644
--- a/docker/quickstart/docker-compose-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-m1.quickstart.yml
@@ -151,6 +151,7 @@ services:
     - DATAHUB_GMS_PORT=8080
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - BACKFILL_BROWSE_PATHS_V2=true
+    - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
     hostname: datahub-upgrade
     image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
     labels:
diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
index e45bafc3da480..b1cb6c208a42d 100644
--- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
@@ -144,6 +144,7 @@ services:
     - DATAHUB_GMS_PORT=8080
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - BACKFILL_BROWSE_PATHS_V2=true
+    - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
     hostname: datahub-upgrade
     image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
     labels:
diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
index 020ef5e9a97b9..ab5182bf98ae5 100644
--- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -144,6 +144,7 @@ services:
     - DATAHUB_GMS_PORT=8080
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - BACKFILL_BROWSE_PATHS_V2=true
+    - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
     hostname: datahub-upgrade
     image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
     labels:
diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
index 8adc2b9063b84..8a66521cbb522 100644
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@@ -151,6 +151,7 @@ services:
     - DATAHUB_GMS_PORT=8080
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - BACKFILL_BROWSE_PATHS_V2=true
+    - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
     hostname: datahub-upgrade
     image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
     labels:
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
index ceaf37a1289d9..db414d70603dc 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java
@@ -16,7 +16,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.RequiredArgsConstructor;
-import org.javatuples.Quintet;
+import org.javatuples.Septet;
 import org.javatuples.Sextet;
 import org.springframework.cache.Cache;
 import org.springframework.cache.CacheManager;
@@ -154,8 +154,9 @@ public SearchResult getCachedSearchResults(
         batchSize,
         querySize -> getRawSearchResults(entityNames, query, filters, sortCriterion, querySize.getFrom(),
                 querySize.getSize(), flags, facets),
-        querySize -> Sextet.with(entityNames, query, filters != null ? toJsonString(filters) : null,
-            sortCriterion != null ? toJsonString(sortCriterion) : null, facets, querySize), flags, enableCache).getSearchResults(from, size);
+        querySize -> Septet.with(entityNames, query, filters != null ? toJsonString(filters) : null,
+                sortCriterion != null ? toJsonString(sortCriterion) : null, flags != null ? toJsonString(flags) : null,
+                facets, querySize), flags, enableCache).getSearchResults(from, size);
   }
 
 
@@ -175,7 +176,8 @@ public AutoCompleteResult getCachedAutoCompleteResults(
       if (enableCache(flags)) {
         try (Timer.Context ignored2 = MetricUtils.timer(this.getClass(), "getCachedAutoCompleteResults_cache").time()) {
           Timer.Context cacheAccess = MetricUtils.timer(this.getClass(), "autocomplete_cache_access").time();
-          Object cacheKey = Quintet.with(entityName, input, field, filters != null ? toJsonString(filters) : null, limit);
+          Object cacheKey = Sextet.with(entityName, input, field, filters != null ? toJsonString(filters) : null,
+                  flags != null ?  toJsonString(flags) : null, limit);
           String json = cache.get(cacheKey, String.class);
           result = json != null ? toRecordTemplate(AutoCompleteResult.class, json) : null;
           cacheAccess.stop();
@@ -210,7 +212,8 @@ public BrowseResult getCachedBrowseResults(
       if (enableCache(flags)) {
         try (Timer.Context ignored2 = MetricUtils.timer(this.getClass(), "getCachedBrowseResults_cache").time()) {
           Timer.Context cacheAccess = MetricUtils.timer(this.getClass(), "browse_cache_access").time();
-          Object cacheKey = Quintet.with(entityName, path, filters != null ? toJsonString(filters) : null, from, size);
+          Object cacheKey = Sextet.with(entityName, path, filters != null ? toJsonString(filters) : null,
+                  flags != null ? toJsonString(flags) : null, from, size);
           String json = cache.get(cacheKey, String.class);
           result = json != null ? toRecordTemplate(BrowseResult.class, json) : null;
           cacheAccess.stop();
@@ -247,9 +250,10 @@ public ScrollResult getCachedScrollResults(
       ScrollResult result;
       if (enableCache(flags)) {
         Timer.Context cacheAccess = MetricUtils.timer(this.getClass(), "scroll_cache_access").time();
-        Object cacheKey = Sextet.with(entities, query,
+        Object cacheKey = Septet.with(entities, query,
             filters != null ? toJsonString(filters) : null,
             sortCriterion != null ? toJsonString(sortCriterion) : null,
+            flags != null ?  toJsonString(flags) : null,
             scrollId, size);
         String json = cache.get(cacheKey, String.class);
         result = json != null ? toRecordTemplate(ScrollResult.class, json) : null;
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
index cbaf70ca22617..290e8c60deb00 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
@@ -157,7 +157,7 @@ private AggregationMetadataArray transformIndexIntoEntityName(AggregationMetadat
   @Nonnull
   @WithSpan
   private ScrollResult executeAndExtract(@Nonnull List<EntitySpec> entitySpecs, @Nonnull SearchRequest searchRequest, @Nullable Filter filter,
-      @Nullable String scrollId, @Nonnull String keepAlive, int size) {
+      @Nullable String scrollId, @Nullable String keepAlive, int size) {
     try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "executeAndExtract_scroll").time()) {
       final SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
       // extract results, validated against document model as well
@@ -166,7 +166,7 @@ private ScrollResult executeAndExtract(@Nonnull List<EntitySpec> entitySpecs, @N
               .extractScrollResult(searchResponse,
               filter, scrollId, keepAlive, size, supportsPointInTime()));
     } catch (Exception e) {
-      log.error("Search query failed", e);
+      log.error("Search query failed: {}", searchRequest, e);
       throw new ESQueryException("Search query failed:", e);
     }
   }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
index 49571a60d5f21..0df6afd49c373 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
@@ -241,7 +241,9 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
 
     BoolQueryBuilder filterQuery = getFilterQuery(filter);
     searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery));
-    _aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
+    if (!finalSearchFlags.isSkipAggregates()) {
+      _aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
+    }
     if (!finalSearchFlags.isSkipHighlighting()) {
       searchSourceBuilder.highlighter(_highlights);
     }
@@ -366,7 +368,7 @@ public SearchResult extractResult(@Nonnull SearchResponse searchResponse, Filter
 
   @WithSpan
   public ScrollResult extractScrollResult(@Nonnull SearchResponse searchResponse, Filter filter, @Nullable String scrollId,
-      @Nonnull String keepAlive, int size, boolean supportsPointInTime) {
+      @Nullable String keepAlive, int size, boolean supportsPointInTime) {
     int totalCount = (int) searchResponse.getHits().getTotalHits().value;
     List<SearchEntity> resultList = getResults(searchResponse);
     SearchResultMetadata searchResultMetadata = extractSearchResultMetadata(searchResponse, filter);
@@ -376,7 +378,7 @@ public ScrollResult extractScrollResult(@Nonnull SearchResponse searchResponse,
     if (searchHits.length == size) {
       Object[] sort = searchHits[searchHits.length - 1].getSortValues();
       long expirationTimeMs = 0L;
-      if (supportsPointInTime) {
+      if (keepAlive != null && supportsPointInTime) {
         expirationTimeMs = TimeValue.parseTimeValue(keepAlive, "expirationTime").getMillis() + System.currentTimeMillis();
       }
       nextScrollId = new SearchAfterWrapper(sort, searchResponse.pointInTimeId(), expirationTimeMs).toScrollId();
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index e9113d339e81d..a06891699607b 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -285,6 +285,8 @@ bootstrap:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
   backfillBrowsePathsV2:
     enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
+  reprocessDefaultBrowsePathsV2:
+    enabled:  ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]}
   policies:
     file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
     # eg for local file
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
index 64f59780b887f..cbfeeaef860d3 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
@@ -193,7 +193,7 @@ BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nullable
    */
   @Nonnull
   ScrollResult fullTextScroll(@Nonnull List<String> entities, @Nonnull String input, @Nullable Filter postFilters,
-      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size, @Nullable SearchFlags searchFlags);
+      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size, @Nullable SearchFlags searchFlags);
 
   /**
    * Gets a list of documents that match given search request. The results are aggregated and filters are applied to the
@@ -210,7 +210,7 @@ ScrollResult fullTextScroll(@Nonnull List<String> entities, @Nonnull String inpu
    */
   @Nonnull
   ScrollResult structuredScroll(@Nonnull List<String> entities, @Nonnull String input, @Nullable Filter postFilters,
-      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size, @Nullable SearchFlags searchFlags);
+      @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size, @Nullable SearchFlags searchFlags);
 
   /**
    * Max result size returned by the underlying search backend

From e73e92699947084b5ecb1f5d3e0c5762dc446bbf Mon Sep 17 00:00:00 2001
From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com>
Date: Wed, 8 Nov 2023 12:32:41 +0530
Subject: [PATCH 175/259] feat(integration/fivetran): Fivetran connector
 integration (#9018)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../app/ingest/source/builder/constants.ts    |   4 +
 .../app/ingest/source/builder/sources.json    |   7 +
 datahub-web-react/src/images/fivetranlogo.png | Bin 0 -> 10230 bytes
 .../docs/sources/fivetran/fivetran_pre.md     |  86 +++
 .../docs/sources/fivetran/fivetran_recipe.yml |  43 ++
 metadata-ingestion/setup.py                   |   3 +
 .../datahub/api/entities/datajob/datajob.py   |  25 +-
 .../dataprocess/dataprocess_instance.py       |  27 +-
 metadata-ingestion/src/datahub/emitter/mcp.py |   4 +-
 .../datahub/ingestion/api/source_helpers.py   |  13 +-
 .../ingestion/source/fivetran/__init__.py     |   0
 .../ingestion/source/fivetran/config.py       | 145 ++++
 .../ingestion/source/fivetran/data_classes.py |  36 +
 .../ingestion/source/fivetran/fivetran.py     | 289 ++++++++
 .../source/fivetran/fivetran_log_api.py       | 147 ++++
 .../source/fivetran/fivetran_query.py         |  76 ++
 .../ingestion/source_config/sql/snowflake.py  |  82 ++-
 .../integration/fivetran/fivetran_golden.json | 658 ++++++++++++++++++
 .../integration/fivetran/test_fivetran.py     | 192 +++++
 .../main/resources/boot/data_platforms.json   |  10 +
 20 files changed, 1777 insertions(+), 70 deletions(-)
 create mode 100644 datahub-web-react/src/images/fivetranlogo.png
 create mode 100644 metadata-ingestion/docs/sources/fivetran/fivetran_pre.md
 create mode 100644 metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/fivetran/__init__.py
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py
 create mode 100644 metadata-ingestion/tests/integration/fivetran/fivetran_golden.json
 create mode 100644 metadata-ingestion/tests/integration/fivetran/test_fivetran.py

diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts
index dba8e8bb1dce6..fdb094d721304 100644
--- a/datahub-web-react/src/app/ingest/source/builder/constants.ts
+++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts
@@ -29,6 +29,7 @@ import databricksLogo from '../../../../images/databrickslogo.png';
 import verticaLogo from '../../../../images/verticalogo.png';
 import mlflowLogo from '../../../../images/mlflowlogo.png';
 import dynamodbLogo from '../../../../images/dynamodblogo.png';
+import fivetranLogo from '../../../../images/fivetranlogo.png';
 
 export const ATHENA = 'athena';
 export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
@@ -105,6 +106,8 @@ export const DBT_CLOUD = 'dbt-cloud';
 export const DBT_CLOUD_URN = `urn:li:dataPlatform:dbt`;
 export const VERTICA = 'vertica';
 export const VERTICA_URN = `urn:li:dataPlatform:${VERTICA}`;
+export const FIVETRAN = 'fivetran';
+export const FIVETRAN_URN = `urn:li:dataPlatform:${FIVETRAN}`;
 
 export const PLATFORM_URN_TO_LOGO = {
     [ATHENA_URN]: athenaLogo,
@@ -138,6 +141,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [SUPERSET_URN]: supersetLogo,
     [UNITY_CATALOG_URN]: databricksLogo,
     [VERTICA_URN]: verticaLogo,
+    [FIVETRAN_URN]: fivetranLogo,
 };
 
 export const SOURCE_TO_PLATFORM_URN = {
diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
index b18384909c33f..9619abebbd54e 100644
--- a/datahub-web-react/src/app/ingest/source/builder/sources.json
+++ b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -216,6 +216,13 @@
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/vertica/",
         "recipe": "source:\n    type: vertica\n    config:\n        # Coordinates\n        host_port: localhost:5433\n        # The name of the vertica database\n        database: Database_Name\n        # Credentials\n        username: Vertica_User\n        password: Vertica_Password\n\n        include_tables: true\n        include_views: true\n        include_projections: true\n        include_models: true\n        include_view_lineage: true\n        include_projection_lineage: true\n        profiling:\n            enabled: false\n        stateful_ingestion:\n            enabled: true  "
     },
+    {
+        "urn": "urn:li:dataPlatform:fivetran",
+        "name": "fivetran",
+        "displayName": "Fivetran",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/fivetran/",
+        "recipe": "source:\n    type: fivetran\n    config:\n        # Fivetran log connector destination server configurations\n        fivetran_log_config:\n            destination_platform: snowflake\n            destination_config:\n                # Coordinates\n                account_id: snowflake_account_id\n                warehouse: warehouse_name\n                database: snowflake_db\n                log_schema: fivetran_log_schema\n\n                # Credentials\n                username: ${SNOWFLAKE_USER}\n                password: ${SNOWFLAKE_PASS}\n                role: snowflake_role\n\n        # Optional - filter for certain connector names instead of ingesting everything.\n        # connector_patterns:\n        #     allow:\n        #         - connector_name\n\n        # Optional -- This mapping is optional and only required to configure platform-instance for source\n        # A mapping of Fivetran connector id to data platform instance\n        # sources_to_platform_instance:\n        #     calendar_elected:\n        #         platform_instance: cloud_postgres_instance\n        #         env: DEV\n\n        # Optional -- This mapping is optional and only required to configure platform-instance for destination.\n        # A mapping of Fivetran destination id to data platform instance\n        # destination_to_platform_instance:\n        #     calendar_elected:\n        #         platform_instance: cloud_postgres_instance\n        #         env: DEV"
+    },
     {
         "urn": "urn:li:dataPlatform:custom",
         "name": "custom",
diff --git a/datahub-web-react/src/images/fivetranlogo.png b/datahub-web-react/src/images/fivetranlogo.png
new file mode 100644
index 0000000000000000000000000000000000000000..d5c999ad2d86e99273971dd0d31a18fd5e94733b
GIT binary patch
literal 10230
zcmZX42{@G9`@c1NmIh-dEmUN(?;)BZ%h*YT?6QWj#Sp!d7?E8eOZG^zXW!RES(+?a
zLUvgye&?BaU)T44{jY1T>ABCj&;7a2=U&cvO#kR=UuU4@q$MFCVYq?3h9V&$1CL}R
z)a2lwmfSO)BqZb{x>^S6KqT=#BJukFJ`avaJdQ{_|7ZO8`S{^@mCw<?MvkYzgX<AV
z$p32r=)3=CkO+FL??iwj8YE7Ewa4~>8c~zvKOiu5ycXyamw<0D2ylt||NBkECHe!l
z29^Ugq7fhxM|}T7Bw9O0NB^h!zlgx#v4=!`LxLKSieqSC1kiYF3TP6sjv<a8h-zS#
zIC4CGEZPz55Y@mS&?KS|6^?xaSYQN*$GAWw>VPRQPIQ2%|G!g2CyAQJ)&M8RYl+K=
z;_)DO08{{dzyCDB2od_&2+=a}`<OH^3%EL-0!zT>f6X30106TQ6wxdgKb`_cz=tUQ
z?;!E{cosYmfsP-5DFV9H!xSk{0!Q8`ZG959;I7r*v$+~H&UK52B{zg(K5g%P#&WvU
z^IV^QbMi#&pLJcFu;t$#*iG9>dZ~lXgaz7VpZ3e16Qq}(Wy!gXG(O!-YNp!JzZ21F
z$W%r)edR5yoeEsumGRdL_p$E$tyBFEwaiOJ`6;SUPrj2!bKg@#c*i3%hF(7q+tsx2
zx2gHT)g<VfM~4^BOkSc4J3VvjY>;t5$GM7qCs})z;^FX+sBdwKk|`T#y+?j6wbC8|
z>Mwa!Qyb<U-@RaxU3s6wky*f?=;Dg@c;c6zANi=)+N#foew|W2b2Z%oV>Vdm#r{r0
z>HRg+$h6x!3ZwrF^cg+3%6Vn|AyB5gSA^lKKD9pyiD2%HYbpkQ<3H2)KP1{5cEos|
zk-NYks9EbYh++21dZs1B>0EhcOf}^a^N_IfYdcj`RQye3Wvqz#m{X_i%C$#DEnlqR
zyPtc*))=?^CKq;;UVdJ#`L<QnzjHudV*mN~gNLiDUKVRPMZK2GpYm((8+ko1HFo=(
zVzuP<=581^#pVL1%cnOd;x(R1-$ymu1($W@e%Suiq?!7h<;}}a-d^H%Pee33Tt<4S
z9r0QBx^PCt_37(+sUL&(^%yFptj8;D#C|@!hmrV(3$2)JpP*xO)!cu1Z{mU2T{)i_
zvGi1Pp$RV>kI%NxD?iEKlKUjK^2st5CR$wNw)RURTd#aJDqs1%yZ5v!Sl_YHI|q?Y
ze?+rl)_G^<<NXS8Q)%*C`Is?!eScVaVpWCwTJNtDS~|S??E1el1nTSl3f&Ia>@WIl
z9_83;Q?7ygm}*y1u;z*!auRU2yBsxU9x_ISI#|aw;3HtgL7KAUe=h%wMEqUaO!~H?
z7XKIBNeU)P?gS3X@jOOQZ6vK&AsuzZyBRm6vK@BVWxRyixqW0}WXVt`=LD|%bD^qE
ztgFa(hj4V-L`OtMhXt#sMQ{9i<<2ml-c@7WTSnIFf%8;glx>>8Wj(N_F|Ifg`4f6a
z%9e`PizPd)S4|t(b*G6Yi+F)WwD@V&eiKA`9XJU(i?xCUl4-u@<QZ(V*{%9R;>FRO
z)NX26Bk?Ax1Je`zV+wXQ)IU1ame9ZljZGD2>rPsk$PU{D)$Vf=HlGOF+oI@Ci$Q$2
zq-ef;G;#{1v4y(;EF{d32StT?vBYrDeHav-;(JS1&mere#m|16qg*X!7QvxC@#-Vl
z(JSneItF1gEaUbQL~ig!UN4$#YEfRQ002k9x2Q;;B<M$fs9L-2z<H%)^j?Ip(%|Km
zq%g{)3&`rU$n9M~7Z_r2tbIP~*Sm`I<YcUFp1?Jq-Mh-COlos(GYPBAkNlwg5jUR5
zk-K;GWjMl^R`f*)CO*ioa|DDyq>bvpmVAMeH_U4{Zyo0h`G{fM&JPo_<BaJl34<w-
zmi;2YmUbj5ZwSjr$b~cbA&W1v=m%!(BHZs!A*xQQtS8+9p16PQl#jdBlr&1#+l4D<
zW*i#><mLd-P{gUd`1BP&B(s8@IE0YCpYhdJMm4?7#y#`R6rwOF`7Y*io{Lm_FS8YK
z<iSr>7~i5am&`DlUKGZrU?)iM>~dapgy1K0zo<;zq=&sumUdr?$?WN`zs3ObQpn^E
zwFp351%>QRuAqU^z4KhObdM-w#(sj0imGVW&rv^j)3g<az0eVdmhzC<ueQQfx}!67
z_X2sXWkw%s1!~vlR~Zc^(X!K8c1zQjb33=@)cV<9YJ2BPCsw7q$4gMtd}zlda`^fd
zLIYQBBLr(mjJ&iCZNG!+XRuo4I=R9Lc3Qes>4njn(K5!(ep_>^UNtgk8<TFL+aO{O
zm;y7DU8dIi(&!&GV}AEt2N`giajTy@7~ZbXZ36}a$*g-ipu8}rPD3-<oLp%IOs&r<
zP#j%9m3&YPfRxDXm3hPH9<guirtT@He5nv#(`vJhLSs7MqAM@I3xKiC?fAyzTCv>y
zVaOrn7PSZ{iKC~(DqG#dYvMbm=za?@ICPq*HJRV8iSs>IOKwv`BW8#@z4$1&=)Hit
zbGmlmV84X>VKW4NPr)|hy}+YoGbM)sP7UOzX+)K7)ru_V6{lFByX=HdUNMR?Jy+$|
z{HR%eg9$R~e^?qgD0Ri1Vi*genz7a9>83{exvx%KpFtd700nW6l6O-Y6w9bfs5qCE
zmUwV$T&Awc;Qov{RD*Mxpg(ZQ?D#?4c{liR<nnjqvZL+kbGYs?P{MBuU0Mv>z(VMT
zDw*p~;a(RII*0bts?(qSI+vmTG8Z_w9`(MwJed%YLp3PT+}8WyH>D{~<%m3LcH6|U
z0=;5uySnGk>xtQ@ERlxcPQ}lsHvyD;7tvAE&Tr}Z8ihN{Zn($Z7y2q>ZrD#qTjdHb
zC_<Pg-opW2k}*vnlt+X9eo*1E|47)KSLL<Y8wb7T7gVJ6g6{3>c!S#t)1`ek6j}#q
z6|d09?VDV2Vqb^<c_*CokFDb)-b4x!qe~p`)@a4{oZliC5f!R2npkb6DsMH<jqndr
zl;W}YUm(x=7Q!MlrO&YIM3mkNKKCq#)&DjN{kQUMGsygbNRw8!G@+&Z&&u=K7Eq31
zrp*)XJo&r@Z}~hQppXp(&iQF$vMWNv0|NNAAaDwlnHj)L5U#^`$pOz~JPH75d<4>9
z3*Hf>eT?YG0xbfft|IQaC)<TYU#x%5ooelRHqDesksj#&L&MaRi#H5-dtN(%KC&u%
z@%u1?4FzjjgzDX<e}=C^bgSs^XNX9;@6l?5YOy|Sb&LXur8(HuaplpjHN@JAn%cPQ
zDmYD0-kWGL8`#?(kIZ!yh40o-U(Jde2tLsdhcNW*OlVlVH-#-9A^0}nz}QvZ&1kSG
z&7rOZy{NI+wV!T6Fr}Etl_gi?G*EPOYozx7J*#?G#qP^+AGB4BwvwrF6!Z2kX&CM9
zcL6@AeI(FqN-8xbb*Dmkr<52uIR#eN;26hsTd4n;ASC0b(`NqPT60{{_T`drarF==
z_-3t+U~?ks=U?_Pq}32I?n;Xs)W}7r(;cm=G$x`t;Lj>4-aUr1Bv&%ayp?VjvP*=V
z5jh=k?Oc=Pz*r#pf*(}Wl*xA!bjGXB>@AOZKFv+Ch>okIxFxo>0}_em(7%8FZj`l;
ztoHapcx2if@NH(fp!@@u7-nov#<9>-p3{0i`>Ce=dcR4@5Rp21Ue5+1EqRYZ1%W-f
zM>^jJDn(vac-+u6zE`brs#g>cu-2vKWMTJ(7L+nalwPlB2iVoypXP)tkDkaqK?e}3
zbWPkV6&t}Bvm>__mD7X)o`C91a~*(=ZW_7!=A4%Z+nSD0uuTA<g`V7e5<QDNO)H*I
z!J^!fv22OwHv3I9)`~7SPJ!(w`nMQVRC(v{i}#B8Uq{q(gYMZbXr7zZmgBsaQSTxI
zJB;sukADo*P=ngG1aQwH3XKfO$H1EWY}10-l41Ltbo6Nt*P)gGj|q&2`qT+LN6%uq
zAR6j2BriN`AXg;$-ns}7S)XoD`0fLB9txF)b|>=d>$9z3x4m!hP~R}E6Q7H;>RnK1
z*A)t+>`Pkyd3}0G$lZ=TN-ql62-@?A7o(dpHj&?QR-8QI$b*yi(SE@~JKhIYI#ee<
zMgt$Vgi^v~#7nb9+d9rVyjA!_KM53B>GN;N8rFSU)NY~eIpG>e`gufz>hy&r9~-S=
zXV`{YP?vn%(?+<;c<x-bkNOYG)Hy}xuhjWRNQCOlMe<0U2?-c&;`5&Y_QPdoZ3U#z
z_vc}5=j*m5*LtTT@AkU$>i>4~ql-OlX(Q4uaViS;`hjHKBV=sp#?w&oXB|_j)7dQB
z#DjuaV!JSCn1>eV&YDN*Rh)TEkoYZa{t*y&fyd<kD+>NT-!5dv0xeC^j4iz@tIk?+
z22gP0ED~3GH@|?X;*1!K_Wa0gY#>9R@p;D^A#{)8oJ7?dPBaw8{f}nVf0ATkMqk2Q
ziT2Pl5_V`EM#?<5R!&8e?6y#v0fxf0@{;cZYyw4<u1x+s>bo<1bqOXy<DTh`qNz#u
zZU5-zdpZU|0K||+B?Q-&(&8ly@nu*P5XTfxXtP5s$K4#fis#V$@U;U9KQJVh;M;!X
zpP<-`v!tRh?Sg8MFqh#nI-4M204V2Hzsc&75+M}XDb5c=#pUnzhHpS6eExaHf}g!V
zT&#8z^cQzj-@8+5T9wMsMTNeL>W1X#Wy(j*31q*2D6~Hd#2MJsH_D5T>RQRhbeKG2
zeCAXX<7Z!PgU}dymNELy`_i?<)h2oI!N%kPn@fb%Akh+PYwFQj7hor|hAH~F!x3Qb
z`MNUU2Y+M1Ze6xKdE;mwg*ouyld(_4>=<HhJZe^vP@n3hkJ77iL}j`&@wCZ%#Z8y2
zk`ojcbwPEUiW)Nu0g3_=4j0yXSx;yO5RF{KCmZb`G}dz=BNgmxTBGbXFt-!~A#T;a
zm-FfS{f*2-o2~=$`VF-I&3&dx`w@-6O5Xxbp7vi-O$Y*Q6fXh1d|Wb<-gWrs-?_|t
zXC)>M-4v~+{+;z%NU#2;D|E?kI9qj^zX0g2w!_;}bGzk@^a?m+(F@<k`~HI*nN4@x
zByTU8LqK*p!L13LN*j=;0^Umrb>du{GL{=jz}0KwX!~*3XoR32fwU4b`}<8v!D>Vb
z-}~90fQ#j=6BI9k?ko%6iYEaKsv(EI1R0Yrh!a_2#wX`7#QFJ76VUJr>0L_gNe?K9
zTws}5bU0-Si53~r_PHmUlm3;ADf$)uqaw3A(a*jHE+f4~Kyw2wKWs@FBZJ%$P`BBY
zMra_4&au)`#b%WK?ukfzYh^-UP{n3<g*D2Q$O;om!slqP@kfgd6iXV;+JUdkHYE!L
zw_s1de(Qj@y&(h$+rAnDmx&`dbsCsRhyjNH!61b79R3P!1SJY`@Jmn)sUT5UWWy)B
z*hlHn7eX{<JFF-TH!>3hl?s71E%KHgQ!#>vdgl1#BG4EkMhws$7j6o3)_xXur)Y74
zeEQigZ-bVlix`1oSd;48zUhhABP@%=WtG*=%>_~rl5(45EeZ%?L;=^x(b7}x$xW0L
z2~&au;S;FhP0t1tjw1ntWwB8klt7}FJuxowplSD@iK+bhGN?56_CGgpVJVR2{sXjq
z<rR0S83H-}8~xsLOg8t4`Eo#)CJVDkIZEk(yyR?@_lhSKe?k;`<>MZ-B}EhBl6i_U
zQsOsUCY6{W;1!l$G71OpBe2f^(m=Zfu71)9L@?71vswR^J}p-P3gHkEpL{MNaVwE1
zp+S@1hrjAnxdrhm^o`4AD)3($v*~3~IFpwIXgQ0i-nM_$RJ^6uv>3@X2%8C8<OXGW
zB}(smMy~2K6Doe~9m1VL_BE9H+B4}K;MJ{$m{lDc{kCyp8nIz!b3BDYh96khdTq(C
z6OABs8>F3>+o?Tu^+BA3yLU@ElIC>?>r}Cu^-3qA`Kcrc`4@OO6@$QXe}XC<%y69O
zw<J{0X_bBat%yV~9s>KoUSeN+(!G4$y@v)sEtMgS4&56JnyWtGqIw6Bi|sIg+a;bK
zF#)rMAiY&j^H7H4vI5u4$QM|6hnwY%JeIWZ$(w3oc3iw(Sl`z(pLhUO>2SeLm7l9T
z1<12P-{jjve%#Qn{1t7GePFasD?@7IW5>}`PtIDH@NSkb<Q4<>=3v>dJU$d_^lhE7
z@h><Y<zI{EB}81v&i+`D@Q|cjgg%Y=npxwZBwi0|!jl4&TG$4&&^?-tI!0wJNgXeD
zPd+QPvn;<Oe57X_b!3+#U{knq@sM=!Z<ue?=~3LBR*P{<W2;8$L()#(?Pye80XI@s
zUyqXJZ^R~JV-Wr>)_CQ4P~11QR^dP7x@BMFwaW^~gUrL+hF;9@g{+9KU2Q9<cXUpe
zN-G}Nqq6$PH^ch5Rkyh`^_coP?cyR&KJqLwcZpJ9IeshN6d#K<z(hwEyZ4Vez1VcQ
zIl+*{Rwrp{i8K|;sQ7i#Fm2peY;u|ei9Re<m^-nit6p%?&zWA<oX)TNlf<|4kCmp2
zyQ0J6eJSG|@tIit=Mx*F02j2LZb={;h*KzQr@h_D3i;S6Z{V?XTIDW}c7Cwo)#OFM
z_PezAxuo~s31!r!&Httt;;NGzd#ox{ijKUXEy5DNJ{ks7-^VXQW^^`3;btNYefsmO
zV4=(o!tH71$c#7{R)AC_e?Em@Xu6&W6(KPtyapDdXm0vl6a|;Bdi`3uIyKfj*x9-;
zfjJ)1vwgcXf+{p?GHD!A>3P}vN_V{zr#i;?Q1diGjtx3Y!Z-a&Ltt%b17KrOJ`2SN
z+lameVK4)yIVj?&L2t+|0)bz0xz9YT7A-s<m<vptS$u2PNne{rum^;MpJc+$6!EV0
z5b)dJJT<I!|9K^E?F;HH6w3YS0(0cN&*p!J$6AlBX(7kVuIA4ICU&^po(hN;DeBzV
z3Tk#Z-xPD|DE9Dm2&N}C?j{asB9R;t&+{(roHWc%88Elm6AV&*VJ!oUs$j#$r*WI%
zA)DbKo8xCPe)%BvdiS+}8RVd7JC36=J}<s>6U4D5^?ke8LRo6a=4+s7@L>mp(TkVT
zOKBniP!@Dy&tpb51|$2@)Oby|EkBZC^b&yzMnGVG^mH*I-i^j=m;rPnWI^@n_u?)#
z)$U5Nn<?u|*RaviF;}stk5)XP^tkTlY^HQV81v`tIUb07{6cY;kt&#WaRY{h)vsT_
z-H_9%oVo^WyJo86pPu)70JXLcBJ9p71(Fv2x!a`oUO-OH$>3AUPVvvXyx8=GY;#d6
z_iypf(UYEK2w=#%NI6O$9&d;tk7R>#<N?}QV;cQzZ6iJ}6>Le{wcR|<f>HP)MZ7&B
z3DC7c{5ioFUu;(TzCJx-gyCEG#;@)^CH4RZ^?^GWsws7(9=A;0G)S<V&WX0C^{iAU
zdw^H@{xrd9rB}PJky0M%#2H`?=!*84P$e+ii0LE8XE%pP@SJLPUnCXg=Ah#ri%_j9
z+m#@lvw(O!Md`K>$3LfoQDK##lFf(hOcy)u1|X|L^Rk&)?s_Y_-SRn^UBRa7e$o>~
zgQDMSO1_I#VJ(DEDaeD%{|ek=V?=o8X@NkKD!cOMyb8h8>Ysa_K{JbXZlCQHYs$b5
zKoO#FTX12?j$+eYpPb2N$@Y`~z#t6n34ZrIP{J&A=?uF&x6s&3`Fj8)in)1r$C6eE
znxi>*YZax-j@0u$_q$u(Aa;ohq~5pB5Bt<kD))kdbt;}BO`;ZFrW0}|<g+WcyIoS2
z_?R?A_zuYD&-`=xyeQCv3_J=lr$}YUP_Xo_29T#|s0t4mPG8S`Djm~jS6VO*NmCtY
z?rd)1j6cIb`6fbRkrHW|cS>i!8Q|U&)94)Rnj?9!x#C6uvI!TBh##PgkNa2hzlsI0
z8xX7wfbHR*v*Z2v!x+f=3v|38Y`81tSH~caZ+e(U%2ylQ@6ccJhg!jWy*-~AHVal9
z3)P1;{&yU%+(Z}allre{@F(<~kh-SendU`(e2NQaw$RO?Rf7MwM;}DM)P=_tAy7<@
zTnp8>E$=f8WO(dM`Kkso`Ao(XIKf%6tBTASgEJT$8};2avDv5!Gw`a$=iuu1IP<JJ
zipEUHkE9<H-$MbLe0DmYLRfT}p3ea@Ne)S{AfY{v7mUK}bV7Mh#@3&Hb7Ei*^__R>
zt~z*s85;SF<2cu;C`Nzr<^`2gXlHuy23R$fpNSrbqW2CDi7>dX>uW&Ue8BccX5nhL
zse&!|lP}<ei<&pY6z7=|a%B?YWN3svZ}p-l<F9SdKtKXGF$n)&$AQ1>LOseRTnZB}
zOxA4$m1i4V0OYcs!oAzjjXwtz$qAh2>yVG%pa7>fa;DXkCYg@)5JM)kOe>6u=`-4a
zf^-6eA&6gnqipOmF}~CQQc3E=7J#g=Sf9O3E<CY?+cbpSqoUyr=@FQdBh9IW!jR?j
zs*90ni~YrBJt)oR)=|3V$m*L{_Y(;*iqh3bnx1n&9ftg?`}L)Oo$m|JhmeOBXy@`F
z9O`Vw<WEqxlGm;JNc@anIiMOqeCtMDZ?xaK`Ph7aki4O&By*r0bAIaNUJSd0u=JmI
z%x_cP^P^TeED}GgJ2a7pq}E8mbk{Apm<#tK3|9OXlu6mC8wNe>zT}&4#xvC4T89Os
z$nig<p9rdxjL0n!M?cOydDiTw){)lxE{oqPP2_J_Csm9|=h)o1{<I#I`P^6?$6nsn
zrfL?cYtwh<xmmHKLRFfAaIxGNwT)~RDT3*O!6FuziEC=@9(AbX&uKQJ=Ued>^+E1&
zQzGe>7MXjh(zX$b&1&wEGbU#pYD_mGCr1Kkc}W6|w;Dn)^04a#e<_<%V`~qpF(Vrm
zPqQvEtm<+Xj&C*93Lmu|x-^{%S#zSWaHM~`EciUDW+rZOWPrKoP;xRX>tf$5Uc=83
zml3}^KRLo$9O&Otn*z>k+7}jlh0zOP($XJlW+;PqRy3<3r72+wwMwY;OVs$0>cWR-
zO#UiWMH*AyqjvM>n$DU3Fdz0_&Uc50^8Bp`k8c)46aC-Aj1R#pE5=9(493t2mHGkQ
zXv|b`uZ6`>H&x8<J7@Od{|-)2OS?8xx%scoU<sN!o49Iu$m|f=lm0%7zobn|6D^8?
zmon<sT*DJZnf{dF#eE;N!80G+wfh2IfajRq1SeZ_C=aiXY=1S3V)iFnRnd(Y+(aP%
zF$kc$)V)?)2p6p213RX_dp7&pg|^;rq1A;K+o?GU_VTvkf8aH$ZisEYPid(AyjjRq
zu;<3C>+?&hSre-~%Gho3*QsggC?r5Pu)BYoIQaW}NeS|cn85~}(mgKv(0Jt8#oE+c
zT^2#1Ry#V=yyLV&4E0afpJlDBKB+GJaM6Uv9{eD3ZyC)som&&Qey3lt%=nCpt$Cy3
zO1qlV8Z}KDF4EV__@$7^--}g|wUqG}+yv{g;hVUJ>A7%0gIkmzbx?ubtE;ZzsZujx
z#(fXQCI2jqnN?ANn~kLU$=Jeojah$|D;z=iWAr`Nyq~{kzLUb=Ngr?-kaQ##!MCuh
zFem-aC%pBNlHZ7yE&F#~uHdw<l<~{_XHmi>!<*;JFM7so#B4QcD^Q+)*?OVrf!X0T
zA9efPBCg36Uj+&<R)8jZ0`%zQYPsfXw^?<6Z9-HmSF{5j<!Wd5MF&cI^xe&~Oe9}2
z-jM{qlk~4f?2Fce?kA#`jncAx8DHzq)3ij@sJP9p3-pjVNu5fvzB~6Yvs0UDQ|BnQ
zx}$-~Tf6H`g}Fn`FTe(4-5IgydsSe4jWJW5w5KtjsL5Y=6;)f^6@@pVD8uO4u*7c;
zDhLfBdD8o;>sd|Y%hn@Lfty?+T5LtZw3A^o>Ru^;3)JL<POnAKGtjmw$F$b7WP3ou
z(X9U5;t~;Rr@kjR#jCN21Gf|?WJAr))rIewilT?522Sbr??SuOpu7IP<S-gqE^ABm
z;z~QToKu85{pb*OTgGTO^}wECR^Xj)c(^rw)&mf-v@04OAY*a0`pX3MWnraHcRDRB
zJf=-Nm?vAk6mlZpGv8+kd~F!VF&b~F$VI>VCN9J?8TeVF#Z;H>X?%owb0Cw%k?Wv4
z0U=QaJ@jGvv}Opw&~Sc=en}!6dtv3bJ~%mPtEkk&#&6(Q@?mQOY=n%S?iB+=UDPba
z6l(*2m=*p;(7p&MoaUuSW_ZO05-?|$f^9BlikbePC|7!C0A(w68>*?L_cLLM%9!hj
z#Z&eXhWJ+f1Ab(M9EK79$#Slw1QZY5WQ!|VHw+Y1DGf$i#CJ!WqEII?KzF+?nN&FT
z=Fm`%*XmLX&0)8g#P+G}y6XH<Hrk6C>k^=R&NMA4PV-8g(p~>YADlW<)d-N5mK-*B
z!4*P~RLKKth+$S(p0gXhpzT=hI>k%DLjyF6{MD?6>Ir*#8XlnQeP(YGA``W@CGO0c
z{*Q-Uk~Jr1iF%YF26p(T*Q^VkIqlA9xz3~}Vx5i3USF^u4dyrb>r)l^kXERQ4S^EA
z;=Hg>((ta4vqd)Dij#8{#aGC=)l_)yt<hkApS$*MF}PBFx_&|3?q}WPNG+XF$AWjy
z_y%k;N6Cn0)#b!@(GrJ+guaM?y@<#u(Vem-?of3vPh;h)Zw|<+BHxj9A3&|tP?oRG
z6tqXtHz#8JT}M;%0)K+4#m|QAy4)X72Tne6XT-oh;gz9AxJ$|QuHWLXUDJ}^r^3^e
zJDO10@5SDauRU-Hl_ZS4%LmDFfMdK9Y?ovkHOS%42!8#<o`SlI_2_3iK+iL6sfYl-
zn^!piP$EqEV+~=f+-v%r(yRbE{~4otp*KPw7P73P2F0o(EhyrnAwN9^UpxUfRGYh$
z3vwzk;?~A|N`Cu_?5sBe#>(6o3(=i#p?0A{_GI*Xi74D4ks&<51iT1F#3OS$=_f$=
z%_&ExXtmos+8IFebmZr&J6C-@ADgYHMww^jjh1|LKUQ<dtIQSj{_J`eSx|~HHk_dI
zs@s_y>J^-JEKR|GQoiAl=J6ElZ>bBMh_4JlKWCrjg>`T|1(o=*8Z$M=mviT=iQ)$+
zPt#<zui+ea*!b$sR(Aye*T*ipGxngVgL|z<ZicG+f{qLq+?E3^e(<Lx$Wm<Yo6DZY
z##+0L;3mzsxQBRRs1k!>9ab!7b!T+jV)~QRcA-sgsjIq>5mpq;0@hK=1^1qQJ|iUg
zkaMk!%(>vlt-o<Fk-=|Upetdv`!)^HPc+-9)5&);972ObLAf4Wv(h`1z)N990vZ*C
z>_(<2T9Jm;JbZP(KnQAv?SHr2q9%-*NUB-s1;-X!M=`DbRMxrLMDsZICgSOCC^&OR
zr|^eZQ^_hxiz1e^&vWc!7m5(uzg`ZgxjWaGyekHc{qzI_W#I>g-8b(;Fy_PR1#i?V
zT`5Pkc*1|V=UW-|odx{=U;)unucUP@m!9GkT@Zk*ISIX4Y?>V5V=KZ_uHJnbf{9Gf
zDA0fZJ#6>Z>XJEiQIoJdc0SzC!Lf*?7oBgC_P)wCib;EnS>k-rUKrFgVPnCmwKW+L
z6S(qM4%#LiP}9S@Y>BtPVvK#nCe;FXH&e~zRcEwA)j!Xb941)pfJVy8s#^nfnFn$k
zP39@JHK#Y$7f(7chUG>~s;#g{{4(EtQ`dIZCY(1s;LGLjyw=8+q3ZjIK9~gWv9+nA
zfdI;{YTr?|5pzcz^{Q64DrLvjw(`_g1$-=;sMZ=Lp_MhwikBFwBfpG11w~Ov()MPd
z#bd_ji(;#2!?Uedf7%%5i_4+{7;7TG7)Cz*x@WilWcO9xjmp&7`p7TuWK&t+RMw27
zDqb(S?=;8i!&p{`?O~u0e!$MHR;-k}f7Hlp{h4+~r%93bdg15u^EWExhsV1_R={Y3
zuN04YH}?yB`G?MqZ$Ei7J<~f|q~z3vQqZb<tq99GB7gn9{8I0-tSR&#_-?3cUn^0y
G!u%iBBTK^o

literal 0
HcmV?d00001

diff --git a/metadata-ingestion/docs/sources/fivetran/fivetran_pre.md b/metadata-ingestion/docs/sources/fivetran/fivetran_pre.md
new file mode 100644
index 0000000000000..949e408215e6f
--- /dev/null
+++ b/metadata-ingestion/docs/sources/fivetran/fivetran_pre.md
@@ -0,0 +1,86 @@
+## Integration Details
+
+This source extracts the following:
+
+- Connectors in fivetran as Data Pipelines and Data Jobs to represent data lineage information between source and destination.
+- Connector sources - DataJob input Datasets.
+- Connector destination - DataJob output Datasets.
+- Connector runs - DataProcessInstances as DataJob runs.
+
+## Configuration Notes
+
+1. Fivetran supports the fivetran platform connector to dump the log events and connectors, destinations, users and roles metadata in your destination.
+2. You need to setup and start the initial sync of the fivetran platform connector before using this source. Refer [link](https://fivetran.com/docs/logs/fivetran-platform/setup-guide).
+3. Once initial sync up of your fivetran platform connector is done, you need to provide the fivetran platform connector's destination platform and its configuration in the recipe.
+
+## Concept mapping 
+
+| Fivetran		   | Datahub												    |
+|--------------------------|--------------------------------------------------------------------------------------------------------|
+| `Connector`              | [DataJob](https://datahubproject.io/docs/generated/metamodel/entities/datajob/)       	            |
+| `Source`                 | [Dataset](https://datahubproject.io/docs/generated/metamodel/entities/dataset/)                        |
+| `Destination`            | [Dataset](https://datahubproject.io/docs/generated/metamodel/entities/dataset/)                        |
+| `Connector Run`          | [DataProcessInstance](https://datahubproject.io/docs/generated/metamodel/entities/dataprocessinstance) |
+
+Source and destination are mapped to Dataset as an Input and Output of Connector.
+
+## Current limitations
+
+Works only for Snowflake destination for now.
+
+## Snowflake destination Configuration Guide
+1. If your fivetran platform connector destination is snowflake, you need to provide user details and its role with correct privileges in order to fetch metadata.
+2. Snowflake system admin can follow this guide to create a fivetran_datahub role, assign it the required privileges, and assign it to a user by executing the following Snowflake commands from a user with the ACCOUNTADMIN role or MANAGE GRANTS privilege.
+
+```sql
+create or replace role fivetran_datahub;
+
+// Grant access to a warehouse to run queries to view metadata
+grant operate, usage on warehouse "<your-warehouse>" to role fivetran_datahub;
+
+// Grant access to view database and schema in which your log and metadata tables exist
+grant usage on DATABASE "<fivetran-log-database>" to role fivetran_datahub;
+grant usage on SCHEMA "<fivetran-log-database>"."<fivetran-log-schema>" to role fivetran_datahub;
+
+// Grant access to execute select query on schema in which your log and metadata tables exist
+grant select on all tables in SCHEMA "<fivetran-log-database>"."<fivetran-log-schema>" to role fivetran_datahub;
+
+// Grant the fivetran_datahub to the snowflake user.
+grant role fivetran_datahub to user snowflake_user;
+```
+
+## Advanced Configurations
+
+### Working with Platform Instances
+If you've multiple instances of source/destination systems that are referred in your `fivetran` setup, you'd need to configure platform instance for these systems in `fivetran` recipe to generate correct lineage edges. Refer the document [Working with Platform Instances](https://datahubproject.io/docs/platform-instances) to understand more about this.
+
+While configuration of platform instance for source system you need to provide connector id as key and for destination system provide destination id as key.
+
+#### Example - Multiple Postgres Source Connectors each reading from different postgres instance
+```yml
+    # Map of connector source to platform instance
+    sources_to_platform_instance:
+      postgres_connector_id1: 
+        platform_instance: cloud_postgres_instance
+        env: PROD
+
+      postgres_connector_id2:
+        platform_instance: local_postgres_instance
+        env: DEV
+```
+
+#### Example - Multiple Snowflake Destinations each writing to different snowflake instance
+```yml
+    # Map of destination to platform instance
+    destination_to_platform_instance:
+      snowflake_destination_id1: 
+        platform_instance: prod_snowflake_instance
+        env: PROD
+
+      snowflake_destination_id2:
+        platform_instance: dev_snowflake_instance
+        env: PROD
+```
+
+
+
diff --git a/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml b/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml
new file mode 100644
index 0000000000000..7c654df59723c
--- /dev/null
+++ b/metadata-ingestion/docs/sources/fivetran/fivetran_recipe.yml
@@ -0,0 +1,43 @@
+source:
+  type: fivetran
+  config:
+    # Fivetran log connector destination server configurations
+    fivetran_log_config:
+      destination_platform: snowflake
+      destination_config:
+        # Coordinates
+        account_id: "abc48144"
+        warehouse: "COMPUTE_WH"
+        database: "MY_SNOWFLAKE_DB"
+        log_schema: "FIVETRAN_LOG"
+
+        # Credentials
+        username: "${SNOWFLAKE_USER}"
+        password: "${SNOWFLAKE_PASS}"
+        role: "snowflake_role"
+
+    # Optional - filter for certain connector names instead of ingesting everything.
+    # connector_patterns:
+    #   allow:
+    #     - connector_name
+
+    # Optional -- A mapping of the connector's all sources to its database.
+    # sources_to_database:
+    #   connector_id: source_db
+    
+    # Optional -- This mapping is optional and only required to configure platform-instance for source
+    # A mapping of Fivetran connector id to data platform instance
+    # sources_to_platform_instance:
+    #   connector_id:
+    #     platform_instance: cloud_instance
+    #     env: DEV
+
+    # Optional -- This mapping is optional and only required to configure platform-instance for destination.
+    # A mapping of Fivetran destination id to data platform instance
+    # destination_to_platform_instance:
+    #   destination_id:
+    #     platform_instance: cloud_instance
+    #     env: DEV
+
+sink:
+  # sink configs
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index afce8dcee840b..2392fce058061 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -395,6 +395,7 @@
     "powerbi-report-server": powerbi_report_server,
     "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.1"},
     "unity-catalog": databricks | sqllineage_lib,
+    "fivetran": snowflake_common,
 }
 
 # This is mainly used to exclude plugins from the Docker image.
@@ -525,6 +526,7 @@
             "nifi",
             "vertica",
             "mode",
+            "fivetran",
             "kafka-connect",
         ]
         if plugin
@@ -629,6 +631,7 @@
         "unity-catalog = datahub.ingestion.source.unity.source:UnityCatalogSource",
         "gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource",
         "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource",
+        "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource",
     ],
     "datahub.ingestion.transformer.plugins": [
         "simple_remove_dataset_ownership = datahub.ingestion.transformer.remove_dataset_ownership:SimpleRemoveDatasetOwnership",
diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py
index 0face6415bacc..6c42e830e223b 100644
--- a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py
+++ b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py
@@ -100,7 +100,9 @@ def generate_tags_aspect(self) -> Iterable[GlobalTagsClass]:
         )
         return [tags]
 
-    def generate_mcp(self) -> Iterable[MetadataChangeProposalWrapper]:
+    def generate_mcp(
+        self, materialize_iolets: bool = True
+    ) -> Iterable[MetadataChangeProposalWrapper]:
         mcp = MetadataChangeProposalWrapper(
             entityUrn=str(self.urn),
             aspect=DataJobInfoClass(
@@ -113,7 +115,9 @@ def generate_mcp(self) -> Iterable[MetadataChangeProposalWrapper]:
         )
         yield mcp
 
-        yield from self.generate_data_input_output_mcp()
+        yield from self.generate_data_input_output_mcp(
+            materialize_iolets=materialize_iolets
+        )
 
         for owner in self.generate_ownership_aspect():
             mcp = MetadataChangeProposalWrapper(
@@ -144,7 +148,9 @@ def emit(
         for mcp in self.generate_mcp():
             emitter.emit(mcp, callback)
 
-    def generate_data_input_output_mcp(self) -> Iterable[MetadataChangeProposalWrapper]:
+    def generate_data_input_output_mcp(
+        self, materialize_iolets: bool
+    ) -> Iterable[MetadataChangeProposalWrapper]:
         mcp = MetadataChangeProposalWrapper(
             entityUrn=str(self.urn),
             aspect=DataJobInputOutputClass(
@@ -157,10 +163,9 @@ def generate_data_input_output_mcp(self) -> Iterable[MetadataChangeProposalWrapp
         yield mcp
 
         # Force entity materialization
-        for iolet in self.inlets + self.outlets:
-            mcp = MetadataChangeProposalWrapper(
-                entityUrn=str(iolet),
-                aspect=StatusClass(removed=False),
-            )
-
-            yield mcp
+        if materialize_iolets:
+            for iolet in self.inlets + self.outlets:
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=str(iolet),
+                    aspect=StatusClass(removed=False),
+                )
diff --git a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py
index cf6080c7072e6..2f07e4a112f93 100644
--- a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py
+++ b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py
@@ -220,12 +220,10 @@ def emit_process_end(
             self._emit_mcp(mcp, emitter, callback)
 
     def generate_mcp(
-        self, created_ts_millis: Optional[int] = None
+        self, created_ts_millis: Optional[int] = None, materialize_iolets: bool = True
     ) -> Iterable[MetadataChangeProposalWrapper]:
-        """
-        Generates mcps from the object
-        :rtype: Iterable[MetadataChangeProposalWrapper]
-        """
+        """Generates mcps from the object"""
+
         mcp = MetadataChangeProposalWrapper(
             entityUrn=str(self.urn),
             aspect=DataProcessInstanceProperties(
@@ -253,7 +251,7 @@ def generate_mcp(
         )
         yield mcp
 
-        yield from self.generate_inlet_outlet_mcp()
+        yield from self.generate_inlet_outlet_mcp(materialize_iolets=materialize_iolets)
 
     @staticmethod
     def _emit_mcp(
@@ -329,7 +327,9 @@ def from_dataflow(dataflow: DataFlow, id: str) -> "DataProcessInstance":
         dpi._template_object = dataflow
         return dpi
 
-    def generate_inlet_outlet_mcp(self) -> Iterable[MetadataChangeProposalWrapper]:
+    def generate_inlet_outlet_mcp(
+        self, materialize_iolets: bool
+    ) -> Iterable[MetadataChangeProposalWrapper]:
         if self.inlets:
             mcp = MetadataChangeProposalWrapper(
                 entityUrn=str(self.urn),
@@ -349,10 +349,9 @@ def generate_inlet_outlet_mcp(self) -> Iterable[MetadataChangeProposalWrapper]:
             yield mcp
 
         # Force entity materialization
-        for iolet in self.inlets + self.outlets:
-            mcp = MetadataChangeProposalWrapper(
-                entityUrn=str(iolet),
-                aspect=StatusClass(removed=False),
-            )
-
-            yield mcp
+        if materialize_iolets:
+            for iolet in self.inlets + self.outlets:
+                yield MetadataChangeProposalWrapper(
+                    entityUrn=str(iolet),
+                    aspect=StatusClass(removed=False),
+                )
diff --git a/metadata-ingestion/src/datahub/emitter/mcp.py b/metadata-ingestion/src/datahub/emitter/mcp.py
index 9085ac152ea0b..d6aa695665e4e 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp.py
@@ -240,7 +240,7 @@ def from_obj_require_wrapper(
         return mcp
 
     def as_workunit(
-        self, *, treat_errors_as_warnings: bool = False
+        self, *, treat_errors_as_warnings: bool = False, is_primary_source: bool = True
     ) -> "MetadataWorkUnit":
         from datahub.ingestion.api.workunit import MetadataWorkUnit
 
@@ -254,10 +254,12 @@ def as_workunit(
                 id=f"{self.entityUrn}-{self.aspectName}-{ts}",
                 mcp=self,
                 treat_errors_as_warnings=treat_errors_as_warnings,
+                is_primary_source=is_primary_source,
             )
 
         return MetadataWorkUnit(
             id=f"{self.entityUrn}-{self.aspectName}",
             mcp=self,
             treat_errors_as_warnings=treat_errors_as_warnings,
+            is_primary_source=is_primary_source,
         )
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
index 2ce9e07bc57bc..fae260226195c 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
@@ -17,6 +17,7 @@
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.emitter.mce_builder import make_dataplatform_instance_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.emitter.mcp_builder import entity_supports_aspect
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.schema_classes import (
     BrowsePathEntryClass,
@@ -64,9 +65,9 @@ def auto_status_aspect(
     """
     For all entities that don't have a status aspect, add one with removed set to false.
     """
-
     all_urns: Set[str] = set()
     status_urns: Set[str] = set()
+    skip_urns: Set[str] = set()
     for wu in stream:
         urn = wu.get_urn()
         all_urns.add(urn)
@@ -89,9 +90,17 @@ def auto_status_aspect(
         else:
             raise ValueError(f"Unexpected type {type(wu.metadata)}")
 
+        if not isinstance(
+            wu.metadata, MetadataChangeEventClass
+        ) and not entity_supports_aspect(wu.metadata.entityType, StatusClass):
+            # If any entity does not support aspect 'status' then skip that entity from adding status aspect.
+            # Example like dataProcessInstance doesn't suppport status aspect.
+            # If not skipped gives error: java.lang.RuntimeException: Unknown aspect status for entity dataProcessInstance
+            skip_urns.add(urn)
+
         yield wu
 
-    for urn in sorted(all_urns - status_urns):
+    for urn in sorted(all_urns - status_urns - skip_urns):
         yield MetadataChangeProposalWrapper(
             entityUrn=urn,
             aspect=StatusClass(removed=False),
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
new file mode 100644
index 0000000000000..b0843182c5cac
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py
@@ -0,0 +1,145 @@
+import logging
+from dataclasses import dataclass, field as dataclass_field
+from typing import Dict, List, Optional
+
+import pydantic
+from pydantic import Field, root_validator
+
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalSourceReport,
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+)
+from datahub.ingestion.source_config.sql.snowflake import BaseSnowflakeConfig
+
+logger = logging.getLogger(__name__)
+
+
+class Constant:
+    """
+    keys used in fivetran plugin
+    """
+
+    ORCHESTRATOR = "fivetran"
+    # table column name
+    SOURCE_SCHEMA_NAME = "source_schema_name"
+    SOURCE_TABLE_NAME = "source_table_name"
+    SOURCE_TABLE_ID = "source_table_id"
+    SOURCE_COLUMN_NAME = "source_column_name"
+    DESTINATION_SCHEMA_NAME = "destination_schema_name"
+    DESTINATION_TABLE_NAME = "destination_table_name"
+    DESTINATION_TABLE_ID = "destination_table_id"
+    DESTINATION_COLUMN_NAME = "destination_column_name"
+    SYNC_ID = "sync_id"
+    MESSAGE_DATA = "message_data"
+    TIME_STAMP = "time_stamp"
+    STATUS = "status"
+    USER_ID = "user_id"
+    GIVEN_NAME = "given_name"
+    FAMILY_NAME = "family_name"
+    CONNECTOR_ID = "connector_id"
+    CONNECTOR_NAME = "connector_name"
+    CONNECTOR_TYPE_ID = "connector_type_id"
+    PAUSED = "paused"
+    SYNC_FREQUENCY = "sync_frequency"
+    DESTINATION_ID = "destination_id"
+    CONNECTING_USER_ID = "connecting_user_id"
+    # Job status constants
+    SUCCESSFUL = "SUCCESSFUL"
+    FAILURE_WITH_TASK = "FAILURE_WITH_TASK"
+    CANCELED = "CANCELED"
+
+
+KNOWN_DATA_PLATFORM_MAPPING = {
+    "postgres": "postgres",
+    "snowflake": "snowflake",
+}
+
+
+class DestinationConfig(BaseSnowflakeConfig):
+    database: str = Field(description="The fivetran connector log database.")
+    log_schema: str = Field(description="The fivetran connector log schema.")
+
+
+class FivetranLogConfig(ConfigModel):
+    destination_platform: str = pydantic.Field(
+        default="snowflake",
+        description="The destination platform where fivetran connector log tables are dumped.",
+    )
+    destination_config: Optional[DestinationConfig] = pydantic.Field(
+        default=None,
+        description="If destination platform is 'snowflake', provide snowflake configuration.",
+    )
+
+    @root_validator(pre=True)
+    def validate_destination_platfrom_and_config(cls, values: Dict) -> Dict:
+        destination_platform = values["destination_platform"]
+        if destination_platform == "snowflake":
+            if "destination_config" not in values:
+                raise ValueError(
+                    "If destination platform is 'snowflake', user must provide snowflake destination configuration in the recipe."
+                )
+        else:
+            raise ValueError(
+                f"Destination platform '{destination_platform}' is not yet supported."
+            )
+        return values
+
+
+@dataclass
+class FivetranSourceReport(StaleEntityRemovalSourceReport):
+    connectors_scanned: int = 0
+    filtered_connectors: List[str] = dataclass_field(default_factory=list)
+
+    def report_connectors_scanned(self, count: int = 1) -> None:
+        self.connectors_scanned += count
+
+    def report_connectors_dropped(self, model: str) -> None:
+        self.filtered_connectors.append(model)
+
+
+class PlatformDetail(ConfigModel):
+    platform_instance: Optional[str] = pydantic.Field(
+        default=None,
+        description="The instance of the platform that all assets produced by this recipe belong to",
+    )
+    env: str = pydantic.Field(
+        default=DEFAULT_ENV,
+        description="The environment that all assets produced by DataHub platform ingestion source belong to",
+    )
+
+
+class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
+    fivetran_log_config: FivetranLogConfig = pydantic.Field(
+        description="Fivetran log connector destination server configurations.",
+    )
+    connector_patterns: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for connectors to filter in ingestion.",
+    )
+    include_column_lineage: bool = Field(
+        default=True,
+        description="Populates table->table column lineage.",
+    )
+    sources_to_database: Dict[str, str] = pydantic.Field(
+        default={},
+        description="A mapping of the connector's all sources to its database. Use connector id as key.",
+    )
+    # Configuration for stateful ingestion
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field(
+        default=None, description="Airbyte Stateful Ingestion Config."
+    )
+    # Fivetran connector all sources to platform instance mapping
+    sources_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field(
+        default={},
+        description="A mapping of the connector's all sources dataset to platform instance. Use connector id as key.",
+    )
+    # Fivetran destination to platform instance mapping
+    destination_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field(
+        default={},
+        description="A mapping of destination dataset to platform instance. Use destination id as key.",
+    )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py
new file mode 100644
index 0000000000000..82bb5f3467c2a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py
@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class ColumnLineage:
+    source_column: str
+    destination_column: str
+
+
+@dataclass
+class TableLineage:
+    source_table: str
+    destination_table: str
+    column_lineage: List[ColumnLineage]
+
+
+@dataclass
+class Connector:
+    connector_id: str
+    connector_name: str
+    connector_type: str
+    paused: bool
+    sync_frequency: int
+    destination_id: str
+    user_name: str
+    table_lineage: List[TableLineage]
+    jobs: List["Job"]
+
+
+@dataclass
+class Job:
+    job_id: str
+    start_time: int
+    end_time: int
+    status: str
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
new file mode 100644
index 0000000000000..c0395b4e4e796
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
@@ -0,0 +1,289 @@
+import logging
+from typing import Dict, Iterable, List, Optional
+
+import datahub.emitter.mce_builder as builder
+from datahub.api.entities.datajob import DataFlow, DataJob
+from datahub.api.entities.dataprocess.dataprocess_instance import (
+    DataProcessInstance,
+    InstanceRunResult,
+)
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SourceCapability,
+    SupportStatus,
+    capability,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.fivetran.config import (
+    KNOWN_DATA_PLATFORM_MAPPING,
+    Constant,
+    FivetranSourceConfig,
+    FivetranSourceReport,
+    PlatformDetail,
+)
+from datahub.ingestion.source.fivetran.data_classes import Connector, Job
+from datahub.ingestion.source.fivetran.fivetran_log_api import FivetranLogAPI
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionSourceBase,
+)
+from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
+    FineGrainedLineage,
+    FineGrainedLineageDownstreamType,
+    FineGrainedLineageUpstreamType,
+)
+from datahub.metadata.schema_classes import StatusClass
+from datahub.utilities.urns.data_flow_urn import DataFlowUrn
+from datahub.utilities.urns.dataset_urn import DatasetUrn
+
+# Logger instance
+logger = logging.getLogger(__name__)
+
+
+@platform_name("Fivetran")
+@config_class(FivetranSourceConfig)
+@support_status(SupportStatus.INCUBATING)
+@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default, can be disabled via configuration `include_column_lineage`",
+)
+class FivetranSource(StatefulIngestionSourceBase):
+    """
+    This plugin extracts fivetran users, connectors, destinations and sync history.
+    This plugin is in beta and has only been tested on Snowflake connector.
+    """
+
+    config: FivetranSourceConfig
+    report: FivetranSourceReport
+    platform: str = "fivetran"
+
+    def __init__(self, config: FivetranSourceConfig, ctx: PipelineContext):
+        super(FivetranSource, self).__init__(config, ctx)
+        self.config = config
+        self.report = FivetranSourceReport()
+
+        self.audit_log = FivetranLogAPI(self.config.fivetran_log_config)
+
+        # Create and register the stateful ingestion use-case handler.
+        self.stale_entity_removal_handler = StaleEntityRemovalHandler.create(
+            self, self.config, self.ctx
+        )
+
+    def _extend_lineage(self, connector: Connector, datajob: DataJob) -> None:
+        input_dataset_urn_list: List[DatasetUrn] = []
+        output_dataset_urn_list: List[DatasetUrn] = []
+        fine_grained_lineage: List[FineGrainedLineage] = []
+
+        source_platform_detail: PlatformDetail = PlatformDetail()
+        destination_platform_detail: PlatformDetail = PlatformDetail()
+        # Get platform details for connector source
+        source_platform_detail = self.config.sources_to_platform_instance.get(
+            connector.connector_id, PlatformDetail()
+        )
+
+        # Get platform details for destination
+        destination_platform_detail = self.config.destination_to_platform_instance.get(
+            connector.destination_id, PlatformDetail()
+        )
+
+        # Get database for connector source
+        # TODO: Once Fivetran exposes this, we shouldn't ask for it via config.
+        source_database: Optional[str] = self.config.sources_to_database.get(
+            connector.connector_id
+        )
+
+        if connector.connector_type in KNOWN_DATA_PLATFORM_MAPPING:
+            source_platform = KNOWN_DATA_PLATFORM_MAPPING[connector.connector_type]
+        else:
+            source_platform = connector.connector_type
+            logger.info(
+                f"Fivetran connector source type: {connector.connector_type} is not supported to mapped with Datahub dataset entity."
+            )
+
+        for table_lineage in connector.table_lineage:
+            input_dataset_urn = DatasetUrn.create_from_ids(
+                platform_id=source_platform,
+                table_name=f"{source_database.lower()}.{table_lineage.source_table}"
+                if source_database
+                else table_lineage.source_table,
+                env=source_platform_detail.env,
+                platform_instance=source_platform_detail.platform_instance,
+            )
+            input_dataset_urn_list.append(input_dataset_urn)
+
+            output_dataset_urn: Optional[DatasetUrn] = None
+            if self.audit_log.fivetran_log_database:
+                output_dataset_urn = DatasetUrn.create_from_ids(
+                    platform_id=self.config.fivetran_log_config.destination_platform,
+                    table_name=f"{self.audit_log.fivetran_log_database.lower()}.{table_lineage.destination_table}",
+                    env=destination_platform_detail.env,
+                    platform_instance=destination_platform_detail.platform_instance,
+                )
+                output_dataset_urn_list.append(output_dataset_urn)
+
+            if self.config.include_column_lineage:
+                for column_lineage in table_lineage.column_lineage:
+                    fine_grained_lineage.append(
+                        FineGrainedLineage(
+                            upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
+                            upstreams=[
+                                builder.make_schema_field_urn(
+                                    str(input_dataset_urn),
+                                    column_lineage.source_column,
+                                )
+                            ]
+                            if input_dataset_urn
+                            else [],
+                            downstreamType=FineGrainedLineageDownstreamType.FIELD,
+                            downstreams=[
+                                builder.make_schema_field_urn(
+                                    str(output_dataset_urn),
+                                    column_lineage.destination_column,
+                                )
+                            ]
+                            if output_dataset_urn
+                            else [],
+                        )
+                    )
+
+        datajob.inlets.extend(input_dataset_urn_list)
+        datajob.outlets.extend(output_dataset_urn_list)
+        datajob.fine_grained_lineages.extend(fine_grained_lineage)
+        return None
+
+    def _generate_dataflow_from_connector(self, connector: Connector) -> DataFlow:
+        return DataFlow(
+            orchestrator=Constant.ORCHESTRATOR,
+            id=connector.connector_id,
+            env=self.config.env,
+            name=connector.connector_name,
+            platform_instance=self.config.platform_instance,
+        )
+
+    def _generate_datajob_from_connector(self, connector: Connector) -> DataJob:
+        dataflow_urn = DataFlowUrn.create_from_ids(
+            orchestrator=Constant.ORCHESTRATOR,
+            flow_id=connector.connector_id,
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+        )
+        datajob = DataJob(
+            id=connector.connector_id,
+            flow_urn=dataflow_urn,
+            name=connector.connector_name,
+            owners={connector.user_name},
+        )
+
+        job_property_bag: Dict[str, str] = {}
+        allowed_connection_keys = [
+            Constant.PAUSED,
+            Constant.SYNC_FREQUENCY,
+            Constant.DESTINATION_ID,
+        ]
+        for key in allowed_connection_keys:
+            if hasattr(connector, key) and getattr(connector, key) is not None:
+                job_property_bag[key] = repr(getattr(connector, key))
+        datajob.properties = job_property_bag
+
+        # Map connector source and destination table with dataset entity
+        # Also extend the fine grained lineage of column if include_column_lineage is True
+        self._extend_lineage(connector=connector, datajob=datajob)
+
+        # TODO: Add fine grained lineages of dataset after FineGrainedLineageDownstreamType.DATASET enabled
+
+        return datajob
+
+    def _generate_dpi_from_job(self, job: Job, datajob: DataJob) -> DataProcessInstance:
+        return DataProcessInstance.from_datajob(
+            datajob=datajob,
+            id=job.job_id,
+            clone_inlets=True,
+            clone_outlets=True,
+        )
+
+    def _get_dpi_workunits(
+        self, job: Job, dpi: DataProcessInstance
+    ) -> Iterable[MetadataWorkUnit]:
+        status_result_map: Dict[str, InstanceRunResult] = {
+            Constant.SUCCESSFUL: InstanceRunResult.SUCCESS,
+            Constant.FAILURE_WITH_TASK: InstanceRunResult.FAILURE,
+            Constant.CANCELED: InstanceRunResult.SKIPPED,
+        }
+        if job.status not in status_result_map:
+            logger.debug(
+                f"Status should be either SUCCESSFUL, FAILURE_WITH_TASK or CANCELED and it was "
+                f"{job.status}"
+            )
+            return []
+        result = status_result_map[job.status]
+        start_timestamp_millis = job.start_time * 1000
+        for mcp in dpi.generate_mcp(
+            created_ts_millis=start_timestamp_millis, materialize_iolets=False
+        ):
+            yield mcp.as_workunit()
+        for mcp in dpi.start_event_mcp(start_timestamp_millis):
+            yield mcp.as_workunit()
+        for mcp in dpi.end_event_mcp(
+            end_timestamp_millis=job.end_time * 1000,
+            result=result,
+            result_type=Constant.ORCHESTRATOR,
+        ):
+            yield mcp.as_workunit()
+
+    def _get_connector_workunits(
+        self, connector: Connector
+    ) -> Iterable[MetadataWorkUnit]:
+        self.report.report_connectors_scanned()
+        # Create dataflow entity with same name as connector name
+        dataflow = self._generate_dataflow_from_connector(connector)
+        for mcp in dataflow.generate_mcp():
+            yield mcp.as_workunit()
+
+        # Map Fivetran's connector entity with Datahub's datajob entity
+        datajob = self._generate_datajob_from_connector(connector)
+        for mcp in datajob.generate_mcp(materialize_iolets=True):
+            if mcp.entityType == "dataset" and isinstance(mcp.aspect, StatusClass):
+                # While we "materialize" the referenced datasets, we don't want them
+                # to be tracked by stateful ingestion.
+                yield mcp.as_workunit(is_primary_source=False)
+            else:
+                yield mcp.as_workunit()
+
+        # Map Fivetran's job/sync history entity with Datahub's data process entity
+        for job in connector.jobs:
+            dpi = self._generate_dpi_from_job(job, datajob)
+            yield from self._get_dpi_workunits(job, dpi)
+
+    @classmethod
+    def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
+        config = FivetranSourceConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            self.stale_entity_removal_handler.workunit_processor,
+        ]
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        """
+        Datahub Ingestion framework invoke this method
+        """
+        logger.info("Fivetran plugin execution is started")
+        connectors = self.audit_log.get_connectors_list()
+        for connector in connectors:
+            if not self.config.connector_patterns.allowed(connector.connector_name):
+                self.report.report_connectors_dropped(connector.connector_name)
+                continue
+            logger.info(f"Processing connector id: {connector.connector_id}")
+            yield from self._get_connector_workunits(connector)
+
+    def get_report(self) -> SourceReport:
+        return self.report
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
new file mode 100644
index 0000000000000..d5d146559d918
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py
@@ -0,0 +1,147 @@
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from sqlalchemy import create_engine
+
+from datahub.ingestion.source.fivetran.config import Constant, FivetranLogConfig
+from datahub.ingestion.source.fivetran.data_classes import (
+    ColumnLineage,
+    Connector,
+    Job,
+    TableLineage,
+)
+from datahub.ingestion.source.fivetran.fivetran_query import FivetranLogQuery
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class FivetranLogAPI:
+    def __init__(self, fivetran_log_config: FivetranLogConfig) -> None:
+        self.fivetran_log_database: Optional[str] = None
+        self.fivetran_log_config = fivetran_log_config
+        self.engine = self._get_log_destination_engine()
+
+    def _get_log_destination_engine(self) -> Any:
+        destination_platform = self.fivetran_log_config.destination_platform
+        engine = None
+        # For every destination, create sqlalchemy engine,
+        # select the database and schema and set fivetran_log_database class variable
+        if destination_platform == "snowflake":
+            snowflake_destination_config = self.fivetran_log_config.destination_config
+            if snowflake_destination_config is not None:
+                engine = create_engine(
+                    snowflake_destination_config.get_sql_alchemy_url(),
+                    **snowflake_destination_config.get_options(),
+                )
+                engine.execute(
+                    FivetranLogQuery.use_schema(
+                        snowflake_destination_config.database,
+                        snowflake_destination_config.log_schema,
+                    )
+                )
+                self.fivetran_log_database = snowflake_destination_config.database
+        return engine
+
+    def _query(self, query: str) -> List[Dict]:
+        logger.debug("Query : {}".format(query))
+        resp = self.engine.execute(query)
+        return [row for row in resp]
+
+    def _get_table_lineage(self, connector_id: str) -> List[TableLineage]:
+        table_lineage_result = self._query(
+            FivetranLogQuery.get_table_lineage_query(connector_id=connector_id)
+        )
+        table_lineage_list: List[TableLineage] = []
+        for table_lineage in table_lineage_result:
+            column_lineage_result = self._query(
+                FivetranLogQuery.get_column_lineage_query(
+                    source_table_id=table_lineage[Constant.SOURCE_TABLE_ID],
+                    destination_table_id=table_lineage[Constant.DESTINATION_TABLE_ID],
+                )
+            )
+            column_lineage_list: List[ColumnLineage] = [
+                ColumnLineage(
+                    source_column=column_lineage[Constant.SOURCE_COLUMN_NAME],
+                    destination_column=column_lineage[Constant.DESTINATION_COLUMN_NAME],
+                )
+                for column_lineage in column_lineage_result
+            ]
+            table_lineage_list.append(
+                TableLineage(
+                    source_table=f"{table_lineage[Constant.SOURCE_SCHEMA_NAME]}.{table_lineage[Constant.SOURCE_TABLE_NAME]}",
+                    destination_table=f"{table_lineage[Constant.DESTINATION_SCHEMA_NAME]}.{table_lineage[Constant.DESTINATION_TABLE_NAME]}",
+                    column_lineage=column_lineage_list,
+                )
+            )
+
+        return table_lineage_list
+
+    def _get_jobs_list(self, connector_id: str) -> List[Job]:
+        jobs: List[Job] = []
+        sync_start_logs = {
+            row[Constant.SYNC_ID]: row
+            for row in self._query(
+                FivetranLogQuery.get_sync_start_logs_query(connector_id=connector_id)
+            )
+        }
+        sync_end_logs = {
+            row[Constant.SYNC_ID]: row
+            for row in self._query(
+                FivetranLogQuery.get_sync_end_logs_query(connector_id=connector_id)
+            )
+        }
+        for sync_id in sync_start_logs.keys():
+            if sync_end_logs.get(sync_id) is None:
+                # If no sync-end event log for this sync id that means sync is still in progress
+                continue
+
+            message_data = json.loads(sync_end_logs[sync_id][Constant.MESSAGE_DATA])
+            if isinstance(message_data, str):
+                # Sometimes message_data contains json string inside string
+                # Ex: '"{\"status\":\"SUCCESSFUL\"}"'
+                # Hence, need to do json loads twice.
+                message_data = json.loads(message_data)
+
+            jobs.append(
+                Job(
+                    job_id=sync_id,
+                    start_time=round(
+                        sync_start_logs[sync_id][Constant.TIME_STAMP].timestamp()
+                    ),
+                    end_time=round(
+                        sync_end_logs[sync_id][Constant.TIME_STAMP].timestamp()
+                    ),
+                    status=message_data[Constant.STATUS],
+                )
+            )
+        return jobs
+
+    def _get_user_name(self, user_id: str) -> str:
+        user_details = self._query(FivetranLogQuery.get_user_query(user_id=user_id))[0]
+        return (
+            f"{user_details[Constant.GIVEN_NAME]} {user_details[Constant.FAMILY_NAME]}"
+        )
+
+    def get_connectors_list(self) -> List[Connector]:
+        connectors: List[Connector] = []
+        connector_list = self._query(FivetranLogQuery.get_connectors_query())
+        for connector in connector_list:
+            connectors.append(
+                Connector(
+                    connector_id=connector[Constant.CONNECTOR_ID],
+                    connector_name=connector[Constant.CONNECTOR_NAME],
+                    connector_type=connector[Constant.CONNECTOR_TYPE_ID],
+                    paused=connector[Constant.PAUSED],
+                    sync_frequency=connector[Constant.SYNC_FREQUENCY],
+                    destination_id=connector[Constant.DESTINATION_ID],
+                    user_name=self._get_user_name(
+                        connector[Constant.CONNECTING_USER_ID]
+                    ),
+                    table_lineage=self._get_table_lineage(
+                        connector[Constant.CONNECTOR_ID]
+                    ),
+                    jobs=self._get_jobs_list(connector[Constant.CONNECTOR_ID]),
+                )
+            )
+        return connectors
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py
new file mode 100644
index 0000000000000..4f52fcd5d884f
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py
@@ -0,0 +1,76 @@
+class FivetranLogQuery:
+    @staticmethod
+    def use_schema(db_name: str, schema_name: str) -> str:
+        return f'use schema "{db_name}"."{schema_name}"'
+
+    @staticmethod
+    def get_connectors_query() -> str:
+        return """
+        SELECT connector_id as "CONNECTOR_ID",
+        connecting_user_id as "CONNECTING_USER_ID",
+        connector_type_id as "CONNECTOR_TYPE_ID",
+        connector_name as "CONNECTOR_NAME",
+        paused as "PAUSED",
+        sync_frequency as "SYNC_FREQUENCY",
+        destination_id as "DESTINATION_ID"
+        FROM CONNECTOR
+        WHERE _fivetran_deleted = FALSE"""
+
+    @staticmethod
+    def get_user_query(user_id: str) -> str:
+        return f"""
+        SELECT id as "USER_ID",
+        given_name as "GIVEN_NAME",
+        family_name as "FAMILY_NAME"
+        FROM USER
+        WHERE id = '{user_id}'"""
+
+    @staticmethod
+    def get_sync_start_logs_query(
+        connector_id: str,
+    ) -> str:
+        return f"""
+        SELECT time_stamp as "TIME_STAMP",
+        sync_id as "SYNC_ID"
+        FROM LOG
+        WHERE message_event = 'sync_start'
+        and connector_id = '{connector_id}' order by time_stamp"""
+
+    @staticmethod
+    def get_sync_end_logs_query(connector_id: str) -> str:
+        return f"""
+        SELECT time_stamp as "TIME_STAMP",
+        sync_id as "SYNC_ID",
+        message_data as "MESSAGE_DATA"
+        FROM LOG
+        WHERE message_event = 'sync_end'
+        and connector_id = '{connector_id}' order by time_stamp"""
+
+    @staticmethod
+    def get_table_lineage_query(connector_id: str) -> str:
+        return f"""
+        SELECT stm.id as "SOURCE_TABLE_ID",
+        stm.name as "SOURCE_TABLE_NAME",
+        ssm.name as "SOURCE_SCHEMA_NAME",
+        dtm.id as "DESTINATION_TABLE_ID",
+        dtm.name as "DESTINATION_TABLE_NAME",
+        dsm.name as "DESTINATION_SCHEMA_NAME"
+        FROM table_lineage as tl
+        JOIN source_table_metadata as stm on tl.source_table_id = stm.id
+        JOIN destination_table_metadata as dtm on tl.destination_table_id = dtm.id
+        JOIN source_schema_metadata as ssm on stm.schema_id = ssm.id
+        JOIN destination_schema_metadata as dsm on dtm.schema_id = dsm.id
+        WHERE stm.connector_id = '{connector_id}'"""
+
+    @staticmethod
+    def get_column_lineage_query(
+        source_table_id: str, destination_table_id: str
+    ) -> str:
+        return f"""
+        SELECT scm.name as "SOURCE_COLUMN_NAME",
+        dcm.name as "DESTINATION_COLUMN_NAME"
+        FROM column_lineage as cl
+        JOIN source_column_metadata as scm on
+        (cl.source_column_id = scm.id and scm.table_id = {source_table_id})
+        JOIN destination_column_metadata as dcm on
+        (cl.destination_column_id = dcm.id and dcm.table_id = {destination_table_id})"""
diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
index c3e8c175f1de5..9fc697018ecd6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
@@ -12,7 +12,7 @@
     OAUTH_AUTHENTICATOR,
 )
 
-from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
 from datahub.configuration.oauth import OAuthConfiguration, OAuthIdentityProvider
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
@@ -42,9 +42,14 @@
 SNOWFLAKE_HOST_SUFFIX = ".snowflakecomputing.com"
 
 
-class BaseSnowflakeConfig(BaseTimeWindowConfig):
+class BaseSnowflakeConfig(ConfigModel):
     # Note: this config model is also used by the snowflake-usage source.
 
+    options: dict = pydantic.Field(
+        default_factory=dict,
+        description="Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.",
+    )
+
     scheme: str = "snowflake"
     username: Optional[str] = pydantic.Field(
         default=None, description="Snowflake username."
@@ -82,14 +87,6 @@ class BaseSnowflakeConfig(BaseTimeWindowConfig):
         default=None, description="Snowflake warehouse."
     )
     role: Optional[str] = pydantic.Field(default=None, description="Snowflake role.")
-    include_table_lineage: bool = pydantic.Field(
-        default=True,
-        description="If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role and Snowflake Enterprise Edition or above.",
-    )
-    include_view_lineage: bool = pydantic.Field(
-        default=True,
-        description="If enabled, populates the snowflake view->table and table->view lineages. Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.",
-    )
     connect_args: Optional[Dict[str, Any]] = pydantic.Field(
         default=None,
         description="Connect args to pass to Snowflake SqlAlchemy driver",
@@ -166,18 +163,6 @@ def _check_oauth_config(oauth_config: Optional[OAuthConfiguration]) -> None:
                 "but should be set when using use_certificate false for oauth_config"
             )
 
-    @pydantic.root_validator()
-    def validate_include_view_lineage(cls, values):
-        if (
-            "include_table_lineage" in values
-            and not values.get("include_table_lineage")
-            and values.get("include_view_lineage")
-        ):
-            raise ValueError(
-                "include_table_lineage must be True for include_view_lineage to be set."
-            )
-        return values
-
     def get_sql_alchemy_url(
         self,
         database: Optional[str] = None,
@@ -261,28 +246,8 @@ def get_connect_args(self) -> dict:
         self._computed_connect_args = connect_args
         return connect_args
 
-
-class SnowflakeConfig(BaseSnowflakeConfig, SQLCommonConfig):
-    database_pattern: AllowDenyPattern = AllowDenyPattern(
-        deny=[r"^UTIL_DB$", r"^SNOWFLAKE$", r"^SNOWFLAKE_SAMPLE_DATA$"]
-    )
-
-    ignore_start_time_lineage: bool = False
-    upstream_lineage_in_report: bool = False
-
-    def get_sql_alchemy_url(
-        self,
-        database: Optional[str] = None,
-        username: Optional[str] = None,
-        password: Optional[pydantic.SecretStr] = None,
-        role: Optional[str] = None,
-    ) -> str:
-        return super().get_sql_alchemy_url(
-            database=database, username=username, password=password, role=role
-        )
-
     def get_options(self) -> dict:
-        options_connect_args: Dict = super().get_connect_args()
+        options_connect_args: Dict = self.get_connect_args()
         options_connect_args.update(self.options.get("connect_args", {}))
         self.options["connect_args"] = options_connect_args
         return self.options
@@ -372,3 +337,34 @@ def get_connection(self) -> snowflake.connector.SnowflakeConnection:
         else:
             # not expected to be here
             raise Exception("Not expected to be here.")
+
+
+class SnowflakeConfig(BaseSnowflakeConfig, BaseTimeWindowConfig, SQLCommonConfig):
+
+    include_table_lineage: bool = pydantic.Field(
+        default=True,
+        description="If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role and Snowflake Enterprise Edition or above.",
+    )
+    include_view_lineage: bool = pydantic.Field(
+        default=True,
+        description="If enabled, populates the snowflake view->table and table->view lineages. Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.",
+    )
+
+    database_pattern: AllowDenyPattern = AllowDenyPattern(
+        deny=[r"^UTIL_DB$", r"^SNOWFLAKE$", r"^SNOWFLAKE_SAMPLE_DATA$"]
+    )
+
+    ignore_start_time_lineage: bool = False
+    upstream_lineage_in_report: bool = False
+
+    @pydantic.root_validator()
+    def validate_include_view_lineage(cls, values):
+        if (
+            "include_table_lineage" in values
+            and not values.get("include_table_lineage")
+            and values.get("include_view_lineage")
+        ):
+            raise ValueError(
+                "include_table_lineage must be True for include_view_lineage to be set."
+            )
+        return values
diff --git a/metadata-ingestion/tests/integration/fivetran/fivetran_golden.json b/metadata-ingestion/tests/integration/fivetran/fivetran_golden.json
new file mode 100644
index 0000000000000..a72c960a72296
--- /dev/null
+++ b/metadata-ingestion/tests/integration/fivetran/fivetran_golden.json
@@ -0,0 +1,658 @@
+[
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "dataFlowInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "postgres"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:fivetran"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "paused": "False",
+                "sync_frequency": "1440",
+                "destination_id": "'interval_unconstitutional'"
+            },
+            "name": "postgres",
+            "type": {
+                "string": "COMMAND"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "dataJobInputOutput",
+    "aspect": {
+        "json": {
+            "inputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)"
+            ],
+            "outputDatasets": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)"
+            ],
+            "inputDatajobs": [],
+            "fineGrainedLineages": [
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD),name)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV),id)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD),id)"
+                    ],
+                    "confidenceScore": 1.0
+                },
+                {
+                    "upstreamType": "FIELD_SET",
+                    "upstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV),name)"
+                    ],
+                    "downstreamType": "FIELD",
+                    "downstreams": [
+                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD),name)"
+                    ],
+                    "confidenceScore": 1.0
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:Shubham Jagtap",
+                    "type": "DEVELOPER",
+                    "source": {
+                        "type": "SERVICE"
+                    }
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:fivetran"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "globalTags",
+    "aspect": {
+        "json": {
+            "tags": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "4c9a03d6-eded-4422-a46a-163266e58243",
+            "type": "BATCH_SCHEDULED",
+            "created": {
+                "time": 1695191853000,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+            "upstreamInstances": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceInput",
+    "aspect": {
+        "json": {
+            "inputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceOutput",
+    "aspect": {
+        "json": {
+            "outputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1695191853000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "status": "STARTED"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1695191885000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SUCCESS",
+                "nativeResultType": "fivetran"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "f773d1e9-c791-48f4-894f-8cf9b3dfc834",
+            "type": "BATCH_SCHEDULED",
+            "created": {
+                "time": 1696343730000,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+            "upstreamInstances": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceInput",
+    "aspect": {
+        "json": {
+            "inputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceOutput",
+    "aspect": {
+        "json": {
+            "outputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1696343730000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "status": "STARTED"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1696343732000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "SKIPPED",
+                "nativeResultType": "fivetran"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {},
+            "name": "63c2fc85-600b-455f-9ba0-f576522465be",
+            "type": "BATCH_SCHEDULED",
+            "created": {
+                "time": 1696343755000,
+                "actor": "urn:li:corpuser:datahub"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRelationships",
+    "aspect": {
+        "json": {
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+            "upstreamInstances": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceInput",
+    "aspect": {
+        "json": {
+            "inputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)",
+                "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceOutput",
+    "aspect": {
+        "json": {
+            "outputs": [
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.employee,PROD)",
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_database.postgres_public.company,PROD)"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1696343755000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "status": "STARTED"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataProcessInstance",
+    "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d",
+    "changeType": "UPSERT",
+    "aspectName": "dataProcessInstanceRunEvent",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1696343790000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "status": "COMPLETE",
+            "result": {
+                "type": "FAILURE",
+                "nativeResultType": "fivetran"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataFlow",
+    "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1654621200000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py
new file mode 100644
index 0000000000000..62b3df12e1b9d
--- /dev/null
+++ b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py
@@ -0,0 +1,192 @@
+import datetime
+from unittest import mock
+from unittest.mock import MagicMock
+
+import pytest
+from freezegun import freeze_time
+
+from datahub.ingestion.run.pipeline import Pipeline
+from datahub.ingestion.source.fivetran.config import DestinationConfig
+from datahub.ingestion.source.fivetran.fivetran_query import FivetranLogQuery
+from tests.test_helpers import mce_helpers
+
+FROZEN_TIME = "2022-06-07 17:00:00"
+
+
+def default_query_results(query):
+    if query == FivetranLogQuery.use_schema("TEST_DATABASE", "TEST_SCHEMA"):
+        return []
+    elif query == FivetranLogQuery.get_connectors_query():
+        return [
+            {
+                "connector_id": "calendar_elected",
+                "connecting_user_id": "reapply_phone",
+                "connector_type_id": "postgres",
+                "connector_name": "postgres",
+                "paused": False,
+                "sync_frequency": 1440,
+                "destination_id": "interval_unconstitutional",
+            },
+        ]
+    elif query == FivetranLogQuery.get_table_lineage_query("calendar_elected"):
+        return [
+            {
+                "source_table_id": "10040",
+                "source_table_name": "employee",
+                "source_schema_name": "public",
+                "destination_table_id": "7779",
+                "destination_table_name": "employee",
+                "destination_schema_name": "postgres_public",
+            },
+            {
+                "source_table_id": "10041",
+                "source_table_name": "company",
+                "source_schema_name": "public",
+                "destination_table_id": "7780",
+                "destination_table_name": "company",
+                "destination_schema_name": "postgres_public",
+            },
+        ]
+    elif query == FivetranLogQuery.get_column_lineage_query(
+        "10040", "7779"
+    ) or query == FivetranLogQuery.get_column_lineage_query("10041", "7780"):
+        return [
+            {
+                "source_column_name": "id",
+                "destination_column_name": "id",
+            },
+            {
+                "source_column_name": "name",
+                "destination_column_name": "name",
+            },
+        ]
+    elif query == FivetranLogQuery.get_user_query("reapply_phone"):
+        return [
+            {
+                "user_id": "reapply_phone",
+                "given_name": "Shubham",
+                "family_name": "Jagtap",
+            }
+        ]
+    elif query == FivetranLogQuery.get_sync_start_logs_query("calendar_elected"):
+        return [
+            {
+                "time_stamp": datetime.datetime(2023, 9, 20, 6, 37, 32, 606000),
+                "sync_id": "4c9a03d6-eded-4422-a46a-163266e58243",
+            },
+            {
+                "time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 30, 345000),
+                "sync_id": "f773d1e9-c791-48f4-894f-8cf9b3dfc834",
+            },
+            {
+                "time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 55, 401000),
+                "sync_id": "63c2fc85-600b-455f-9ba0-f576522465be",
+            },
+        ]
+    elif query == FivetranLogQuery.get_sync_end_logs_query("calendar_elected"):
+        return [
+            {
+                "time_stamp": datetime.datetime(2023, 9, 20, 6, 38, 5, 56000),
+                "sync_id": "4c9a03d6-eded-4422-a46a-163266e58243",
+                "message_data": '"{\\"status\\":\\"SUCCESSFUL\\"}"',
+            },
+            {
+                "time_stamp": datetime.datetime(2023, 10, 3, 14, 35, 31, 512000),
+                "sync_id": "f773d1e9-c791-48f4-894f-8cf9b3dfc834",
+                "message_data": '"{\\"reason\\":\\"Sync has been cancelled because of a user action in the dashboard.Standard Config updated.\\",\\"status\\":\\"CANCELED\\"}"',
+            },
+            {
+                "time_stamp": datetime.datetime(2023, 10, 3, 14, 36, 29, 678000),
+                "sync_id": "63c2fc85-600b-455f-9ba0-f576522465be",
+                "message_data": '"{\\"reason\\":\\"java.lang.RuntimeException: FATAL: too many connections for role \\\\\\"hxwraqld\\\\\\"\\",\\"taskType\\":\\"reconnect\\",\\"status\\":\\"FAILURE_WITH_TASK\\"}"',
+            },
+        ]
+    # Unreachable code
+    raise Exception(f"Unknown query {query}")
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_fivetran_basic(pytestconfig, tmp_path):
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/fivetran"
+
+    # Run the metadata ingestion pipeline.
+    output_file = tmp_path / "fivetran_test_events.json"
+    golden_file = test_resources_dir / "fivetran_golden.json"
+
+    with mock.patch(
+        "datahub.ingestion.source.fivetran.fivetran_log_api.create_engine"
+    ) as mock_create_engine:
+        connection_magic_mock = MagicMock()
+        connection_magic_mock.execute.side_effect = default_query_results
+
+        mock_create_engine.return_value = connection_magic_mock
+
+        pipeline = Pipeline.create(
+            {
+                "run_id": "powerbi-test",
+                "source": {
+                    "type": "fivetran",
+                    "config": {
+                        "fivetran_log_config": {
+                            "destination_platform": "snowflake",
+                            "destination_config": {
+                                "account_id": "TESTID",
+                                "warehouse": "TEST_WH",
+                                "username": "test",
+                                "password": "test@123",
+                                "database": "TEST_DATABASE",
+                                "role": "TESTROLE",
+                                "log_schema": "TEST_SCHEMA",
+                            },
+                        },
+                        "connector_patterns": {
+                            "allow": [
+                                "postgres",
+                            ]
+                        },
+                        "sources_to_database": {
+                            "calendar_elected": "postgres_db",
+                        },
+                        "sources_to_platform_instance": {
+                            "calendar_elected": {
+                                "env": "DEV",
+                            }
+                        },
+                    },
+                },
+                "sink": {
+                    "type": "file",
+                    "config": {
+                        "filename": f"{output_file}",
+                    },
+                },
+            }
+        )
+
+    pipeline.run()
+    pipeline.raise_from_status()
+    golden_file = "fivetran_golden.json"
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=f"{output_file}",
+        golden_path=f"{test_resources_dir}/{golden_file}",
+    )
+
+
+@freeze_time(FROZEN_TIME)
+def test_fivetran_snowflake_destination_config(pytestconfig, tmp_path):
+    snowflake_dest = DestinationConfig(
+        account_id="TESTID",
+        warehouse="TEST_WH",
+        username="test",
+        password="test@123",
+        database="TEST_DATABASE",
+        role="TESTROLE",
+        log_schema="TEST_SCHEMA",
+    )
+    assert (
+        snowflake_dest.get_sql_alchemy_url()
+        == "snowflake://test:test%40123@TESTID?application=acryl_datahub&authenticator=SNOWFLAKE&role=TESTROLE&warehouse=TEST_WH"
+    )
diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json
index 3d956c5774ded..3c70eda8561b8 100644
--- a/metadata-service/war/src/main/resources/boot/data_platforms.json
+++ b/metadata-service/war/src/main/resources/boot/data_platforms.json
@@ -564,5 +564,15 @@
       "type": "KEY_VALUE_STORE",
       "logoUrl": "/assets/platforms/dynamodblogo.png"
     }
+  },
+  {
+    "urn": "urn:li:dataPlatform:fivetran",
+    "aspect": {
+      "datasetNameDelimiter": ".",
+      "name": "fivetran",
+      "displayName": "Fivetran",
+      "type": "OTHERS",
+      "logoUrl": "/assets/platforms/fivetranlogo.png"
+    }
   }
 ]

From 399e032dfa2b4bf87b7b406e7b009e34e99a1003 Mon Sep 17 00:00:00 2001
From: deepgarg-visa <149145061+deepgarg-visa@users.noreply.github.com>
Date: Wed, 8 Nov 2023 22:32:13 +0530
Subject: [PATCH 176/259] feat(neo4j): Allow datahub to connect to specific
 neo4j database (#9179)

Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
---
 docker/docker-compose.override.yml            |  4 ++
 .../docker-compose-m1.quickstart.yml          | 54 +++++++++----------
 .../quickstart/docker-compose.quickstart.yml  | 54 +++++++++----------
 .../src/main/resources/application.yml        |  1 +
 .../common/Neo4jGraphServiceFactory.java      |  7 ++-
 5 files changed, 65 insertions(+), 55 deletions(-)

diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml
index 225aa01fa4e4f..0907f47d70c3c 100644
--- a/docker/docker-compose.override.yml
+++ b/docker/docker-compose.override.yml
@@ -7,8 +7,12 @@ services:
     environment:
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
     - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
+    - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
     volumes:
     - ${HOME}/.datahub/plugins:/etc/datahub/plugins
+  datahub-upgrade:
+    environment:
+      - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
   mysql-setup:
     container_name: mysql-setup
     hostname: mysql-setup
diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml
index c96baf37551b2..613718306abef 100644
--- a/docker/quickstart/docker-compose-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-m1.quickstart.yml
@@ -81,32 +81,32 @@ services:
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
     - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
     - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
-    - EBEAN_DATASOURCE_USERNAME=datahub
-    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
     - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
-    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
-    - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - EBEAN_DATASOURCE_USERNAME=datahub
     - ELASTICSEARCH_HOST=elasticsearch
-    - ELASTICSEARCH_PORT=9200
-    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
-    - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-    - NEO4J_HOST=http://neo4j:7474
-    - NEO4J_URI=bolt://neo4j
-    - NEO4J_USERNAME=neo4j
-    - NEO4J_PASSWORD=datahub
-    - JAVA_OPTS=-Xms1g -Xmx1g
-    - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
-    - GRAPH_SERVICE_IMPL=neo4j
+    - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
+    - ELASTICSEARCH_PORT=9200
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - ENTITY_SERVICE_ENABLE_RETENTION=true
+    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
+    - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
+    - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
+    - JAVA_OPTS=-Xms1g -Xmx1g
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - MAE_CONSUMER_ENABLED=true
     - MCE_CONSUMER_ENABLED=true
+    - METADATA_SERVICE_AUTH_ENABLED=false
+    - NEO4J_HOST=http://neo4j:7474
+    - NEO4J_PASSWORD=datahub
+    - NEO4J_URI=bolt://neo4j
+    - NEO4J_USERNAME=neo4j
     - PE_CONSUMER_ENABLED=true
     - UI_INGESTION_ENABLED=true
-    - METADATA_SERVICE_AUTH_ENABLED=false
     healthcheck:
       interval: 1s
       retries: 3
@@ -134,23 +134,23 @@ services:
       neo4j:
         condition: service_healthy
     environment:
-    - EBEAN_DATASOURCE_USERNAME=datahub
-    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - BACKFILL_BROWSE_PATHS_V2=true
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
     - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
-    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
-    - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - EBEAN_DATASOURCE_USERNAME=datahub
+    - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
     - ELASTICSEARCH_HOST=elasticsearch
-    - ELASTICSEARCH_PORT=9200
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
-    - GRAPH_SERVICE_IMPL=elasticsearch
-    - DATAHUB_GMS_HOST=datahub-gms
-    - DATAHUB_GMS_PORT=8080
+    - ELASTICSEARCH_PORT=9200
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
-    - BACKFILL_BROWSE_PATHS_V2=true
+    - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
     hostname: datahub-upgrade
     image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
index 8a66521cbb522..30ccbae59be74 100644
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@@ -81,32 +81,32 @@ services:
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
     - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
     - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
-    - EBEAN_DATASOURCE_USERNAME=datahub
-    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
     - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
-    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
-    - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - EBEAN_DATASOURCE_USERNAME=datahub
     - ELASTICSEARCH_HOST=elasticsearch
-    - ELASTICSEARCH_PORT=9200
-    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
-    - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-    - NEO4J_HOST=http://neo4j:7474
-    - NEO4J_URI=bolt://neo4j
-    - NEO4J_USERNAME=neo4j
-    - NEO4J_PASSWORD=datahub
-    - JAVA_OPTS=-Xms1g -Xmx1g
-    - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
-    - GRAPH_SERVICE_IMPL=neo4j
+    - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
+    - ELASTICSEARCH_PORT=9200
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - ENTITY_SERVICE_ENABLE_RETENTION=true
+    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
+    - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
+    - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
+    - JAVA_OPTS=-Xms1g -Xmx1g
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - MAE_CONSUMER_ENABLED=true
     - MCE_CONSUMER_ENABLED=true
+    - METADATA_SERVICE_AUTH_ENABLED=false
+    - NEO4J_HOST=http://neo4j:7474
+    - NEO4J_PASSWORD=datahub
+    - NEO4J_URI=bolt://neo4j
+    - NEO4J_USERNAME=neo4j
     - PE_CONSUMER_ENABLED=true
     - UI_INGESTION_ENABLED=true
-    - METADATA_SERVICE_AUTH_ENABLED=false
     healthcheck:
       interval: 1s
       retries: 3
@@ -134,23 +134,23 @@ services:
       neo4j:
         condition: service_healthy
     environment:
-    - EBEAN_DATASOURCE_USERNAME=datahub
-    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - BACKFILL_BROWSE_PATHS_V2=true
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
     - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
-    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
-    - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - EBEAN_DATASOURCE_USERNAME=datahub
+    - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
     - ELASTICSEARCH_HOST=elasticsearch
-    - ELASTICSEARCH_PORT=9200
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
-    - GRAPH_SERVICE_IMPL=elasticsearch
-    - DATAHUB_GMS_HOST=datahub-gms
-    - DATAHUB_GMS_PORT=8080
+    - ELASTICSEARCH_PORT=9200
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
-    - BACKFILL_BROWSE_PATHS_V2=true
+    - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
     hostname: datahub-upgrade
     image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index a06891699607b..46aa02d98572e 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -251,6 +251,7 @@ neo4j:
   username: ${NEO4J_USERNAME:neo4j}
   password: ${NEO4J_PASSWORD:datahub}
   uri: ${NEO4J_URI:bolt://localhost}
+  database: ${NEO4J_DATABASE:graph.db}
   maxConnectionPoolSize: ${NEO4J_MAX_CONNECTION_POOL_SIZE:100}
   maxConnectionAcquisitionTimeout: ${NEO4J_MAX_CONNECTION_ACQUISITION_TIMEOUT_IN_SECONDS:60}
   maxConnectionLifetimeInSeconds: ${NEO4j_MAX_CONNECTION_LIFETIME_IN_SECONDS:3600}
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jGraphServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jGraphServiceFactory.java
index e62dfd50f897d..87670ce10f481 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jGraphServiceFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jGraphServiceFactory.java
@@ -6,8 +6,10 @@
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import javax.annotation.Nonnull;
 import org.neo4j.driver.Driver;
+import org.neo4j.driver.SessionConfig;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
@@ -24,10 +26,13 @@ public class Neo4jGraphServiceFactory {
   @Qualifier("entityRegistry")
   private EntityRegistry entityRegistry;
 
+  @Value("${neo4j.database}")
+  private String neo4jDatabase;
+
   @Bean(name = "neo4jGraphService")
   @Nonnull
   protected Neo4jGraphService getInstance() {
     LineageRegistry lineageRegistry = new LineageRegistry(entityRegistry);
-    return new Neo4jGraphService(lineageRegistry, neo4jDriver);
+    return new Neo4jGraphService(lineageRegistry, neo4jDriver, SessionConfig.forDatabase(neo4jDatabase));
   }
 }

From 332d4afaab39e4b9e9ff73a48e3bfec9b21fe0b5 Mon Sep 17 00:00:00 2001
From: Gabe Lyons <itsgabelyons@gmail.com>
Date: Wed, 8 Nov 2023 10:22:09 -0800
Subject: [PATCH 177/259] feat(subtypes): support subtypes for charts in the UI
 (#9186)

---
 .../java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java | 4 ++++
 datahub-graphql-core/src/main/resources/entity.graphql      | 5 +++++
 datahub-web-react/src/app/entity/chart/ChartEntity.tsx      | 4 ++++
 .../src/app/entity/chart/preview/ChartPreview.tsx           | 5 ++++-
 datahub-web-react/src/graphql/chart.graphql                 | 3 +++
 datahub-web-react/src/graphql/lineage.graphql               | 3 +++
 datahub-web-react/src/graphql/search.graphql                | 6 ++++++
 metadata-models/src/main/resources/entity-registry.yml      | 1 +
 8 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index b99f712034fe0..b0b26f073876c 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -1433,6 +1433,10 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("statsSummary", new ChartStatsSummaryResolver(this.timeseriesAspectService))
             .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
             .dataFetcher("exists", new EntityExistsResolver(entityService))
+            .dataFetcher("subTypes", new SubTypesResolver(
+                this.entityClient,
+                "chart",
+                "subTypes"))
         );
         builder.type("ChartInfo", typeWiring -> typeWiring
             .dataFetcher("inputs", new LoadableTypeBatchResolver<>(datasetType,
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index b37a8f34fa056..035f756a10d55 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -5249,6 +5249,11 @@ type Chart implements EntityWithRelationships & Entity & BrowsableEntity {
     Whether or not this entity exists on DataHub
     """
     exists: Boolean
+
+    """
+    Sub Types that this entity implements
+    """
+    subTypes: SubTypes
 }
 
 """
diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx
index 0f1b6dbf3d660..fc898dec9d93a 100644
--- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx
+++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx
@@ -154,10 +154,12 @@ export class ChartEntity implements Entity<Chart> {
     getOverridePropertiesFromEntity = (chart?: Chart | null): GenericEntityProperties => {
         // TODO: Get rid of this once we have correctly formed platform coming back.
         const name = chart?.properties?.name;
+        const subTypes = chart?.subTypes;
         const externalUrl = chart?.properties?.externalUrl;
         return {
             name,
             externalUrl,
+            entityTypeOverride: subTypes ? capitalizeFirstLetterOnly(subTypes.typeNames?.[0]) : '',
         };
     };
 
@@ -187,6 +189,7 @@ export class ChartEntity implements Entity<Chart> {
         return (
             <ChartPreview
                 urn={data.urn}
+                subType={data.subTypes?.typeNames?.[0]}
                 platform={data?.platform?.properties?.displayName || capitalizeFirstLetterOnly(data?.platform?.name)}
                 platformInstanceId={data.dataPlatformInstance?.instanceId}
                 name={data.properties?.name}
@@ -222,6 +225,7 @@ export class ChartEntity implements Entity<Chart> {
             type: EntityType.Chart,
             icon: entity?.platform?.properties?.logoUrl || undefined,
             platform: entity?.platform,
+            subtype: entity?.subTypes?.typeNames?.[0] || undefined,
         };
     };
 
diff --git a/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx b/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx
index 7d0fc143043e2..b7fbd63ee231e 100644
--- a/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx
+++ b/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx
@@ -15,6 +15,7 @@ import {
     EntityPath,
 } from '../../../../types.generated';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
+import { capitalizeFirstLetterOnly } from '../../../shared/textUtil';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import { IconStyleType } from '../../Entity';
 import { ChartStatsSummary as ChartStatsSummaryView } from '../shared/ChartStatsSummary';
@@ -43,6 +44,7 @@ export const ChartPreview = ({
     snippet,
     degree,
     paths,
+    subType,
 }: {
     urn: string;
     platform?: string;
@@ -67,6 +69,7 @@ export const ChartPreview = ({
     snippet?: React.ReactNode | null;
     degree?: number;
     paths?: EntityPath[];
+    subType?: string | null;
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
 
@@ -76,7 +79,7 @@ export const ChartPreview = ({
             name={name || ''}
             urn={urn}
             description={description || ''}
-            type="Chart"
+            type={capitalizeFirstLetterOnly(subType) || 'Chart'}
             typeIcon={entityRegistry.getIcon(EntityType.Chart, 14, IconStyleType.ACCENT)}
             logoUrl={logoUrl || ''}
             platform={platform}
diff --git a/datahub-web-react/src/graphql/chart.graphql b/datahub-web-react/src/graphql/chart.graphql
index d4d3c3c918408..a4b430720fa3d 100644
--- a/datahub-web-react/src/graphql/chart.graphql
+++ b/datahub-web-react/src/graphql/chart.graphql
@@ -100,6 +100,9 @@ query getChart($urn: String!) {
             canEditLineage
             canEditEmbed
         }
+        subTypes {
+            typeNames
+        }
     }
 }
 
diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql
index 52385dee8631a..8fdfb696e0894 100644
--- a/datahub-web-react/src/graphql/lineage.graphql
+++ b/datahub-web-react/src/graphql/lineage.graphql
@@ -165,6 +165,9 @@ fragment lineageNodeProperties on EntityWithRelationships {
         status {
             removed
         }
+        subTypes {
+            typeNames
+        }
     }
     ... on Dataset {
         name
diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql
index 2297c2d0c1d07..876be12fd335b 100644
--- a/datahub-web-react/src/graphql/search.graphql
+++ b/datahub-web-react/src/graphql/search.graphql
@@ -105,6 +105,9 @@ fragment autoCompleteFields on Entity {
         parentContainers {
             ...parentContainersFields
         }
+        subTypes {
+            typeNames
+        }
     }
     ... on DataFlow {
         orchestrator
@@ -550,6 +553,9 @@ fragment searchResultFields on Entity {
                 }
             }
         }
+        subTypes {
+            typeNames
+        }
     }
     ... on DataFlow {
         flowId
diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml
index 11d0f74305d7b..a5296d074093b 100644
--- a/metadata-models/src/main/resources/entity-registry.yml
+++ b/metadata-models/src/main/resources/entity-registry.yml
@@ -120,6 +120,7 @@ entities:
       - globalTags
       - glossaryTerms
       - browsePathsV2
+      - subTypes
   - name: dashboard
     keyAspect: dashboardKey
     aspects:

From 72135914109a241aa11ceaeb68b9ac56134e7e64 Mon Sep 17 00:00:00 2001
From: Chris Collins <chriscollins3456@gmail.com>
Date: Wed, 8 Nov 2023 14:36:33 -0500
Subject: [PATCH 178/259] feat(ui) Debounce auto-complete in search bar (#9205)

---
 datahub-web-react/src/app/home/HomePageHeader.tsx   | 6 ++++--
 datahub-web-react/src/app/search/SearchablePage.tsx | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
index 0052d54f562eb..c881109f6e419 100644
--- a/datahub-web-react/src/app/home/HomePageHeader.tsx
+++ b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -1,6 +1,7 @@
 import React, { useEffect, useMemo, useState } from 'react';
 import { useHistory } from 'react-router';
 import { Typography, Image, Row, Button, Tag } from 'antd';
+import { debounce } from 'lodash';
 import styled, { useTheme } from 'styled-components/macro';
 import { RightOutlined } from '@ant-design/icons';
 import { ManageAccount } from '../shared/ManageAccount';
@@ -24,6 +25,7 @@ import { getAutoCompleteInputFromQuickFilter } from '../search/utils/filterUtils
 import { useUserContext } from '../context/useUserContext';
 import AcrylDemoBanner from './AcrylDemoBanner';
 import DemoButton from '../entity/shared/components/styled/DemoButton';
+import { HALF_SECOND_IN_MS } from '../entity/shared/tabs/Dataset/Queries/utils/constants';
 
 const Background = styled.div`
     width: 100%;
@@ -176,7 +178,7 @@ export const HomePageHeader = () => {
         });
     };
 
-    const onAutoComplete = (query: string) => {
+    const onAutoComplete = debounce((query: string) => {
         if (query && query.trim() !== '') {
             getAutoCompleteResultsForMultiple({
                 variables: {
@@ -189,7 +191,7 @@ export const HomePageHeader = () => {
                 },
             });
         }
-    };
+    }, HALF_SECOND_IN_MS);
 
     const onClickExploreAll = () => {
         analytics.event({
diff --git a/datahub-web-react/src/app/search/SearchablePage.tsx b/datahub-web-react/src/app/search/SearchablePage.tsx
index 489687050c749..9d02d85d3634c 100644
--- a/datahub-web-react/src/app/search/SearchablePage.tsx
+++ b/datahub-web-react/src/app/search/SearchablePage.tsx
@@ -1,5 +1,6 @@
 import React, { useEffect, useState } from 'react';
 import { useHistory, useLocation } from 'react-router';
+import { debounce } from 'lodash';
 import * as QueryString from 'query-string';
 import { useTheme } from 'styled-components';
 import { SearchHeader } from './SearchHeader';
@@ -17,6 +18,7 @@ import { getAutoCompleteInputFromQuickFilter } from './utils/filterUtils';
 import { useQuickFiltersContext } from '../../providers/QuickFiltersContext';
 import { useUserContext } from '../context/useUserContext';
 import { useSelectedSortOption } from './context/SearchContext';
+import { HALF_SECOND_IN_MS } from '../entity/shared/tabs/Dataset/Queries/utils/constants';
 
 const styles = {
     children: {
@@ -93,7 +95,7 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) =>
         });
     };
 
-    const autoComplete = (query: string) => {
+    const autoComplete = debounce((query: string) => {
         if (query && query.trim() !== '') {
             getAutoCompleteResults({
                 variables: {
@@ -105,7 +107,7 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) =>
                 },
             });
         }
-    };
+    }, HALF_SECOND_IN_MS);
 
     // Load correct autocomplete results on initial page load.
     useEffect(() => {

From 70692b44e995eab252a2344496141acdf6181908 Mon Sep 17 00:00:00 2001
From: Gabe Lyons <itsgabelyons@gmail.com>
Date: Wed, 8 Nov 2023 12:49:23 -0800
Subject: [PATCH 179/259] fix(lineage): magical lineage layout fix (#9187)

---
 .../src/app/lineage/utils/layoutTree.ts       | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/datahub-web-react/src/app/lineage/utils/layoutTree.ts b/datahub-web-react/src/app/lineage/utils/layoutTree.ts
index cc704007049c2..a972a62308f07 100644
--- a/datahub-web-react/src/app/lineage/utils/layoutTree.ts
+++ b/datahub-web-react/src/app/lineage/utils/layoutTree.ts
@@ -32,6 +32,21 @@ function getParentRelationship(direction: Direction, parent: VizNode | null, nod
     return directionRelationships?.find((r) => r?.entity?.urn === node?.urn);
 }
 
+// this utility function is to help make sure layouts that contain many references to the same URN don't struggle laying out that URN.
+function firstAppearanceIndices(arr) {
+    const seen = new Set(); // To track which strings have been seen
+    const result = [] as number[];
+
+    for (let i = 0; i < arr.length; i++) {
+        if (!seen.has(arr[i])) {
+            seen.add(arr[i]); // Add the string to the set
+            result.push(i); // Save the index
+        }
+    }
+
+    return result;
+}
+
 function layoutNodesForOneDirection(
     data: NodeData,
     direction: Direction,
@@ -54,12 +69,10 @@ function layoutNodesForOneDirection(
     while (nodesInCurrentLayer.length > 0) {
         // if we've already added a node to the viz higher up dont add it again
         const urnsToAddInCurrentLayer = Array.from(new Set(nodesInCurrentLayer.map(({ node }) => node.urn || '')));
-        const nodesToAddInCurrentLayer = urnsToAddInCurrentLayer
-            .filter((urn, pos) => urnsToAddInCurrentLayer.indexOf(urn) === pos)
-            .filter((urn) => !nodesByUrn[urn || '']);
+        const positionsToAddInCurrentLayer = firstAppearanceIndices(urnsToAddInCurrentLayer);
 
         const filteredNodesInCurrentLayer = nodesInCurrentLayer
-            .filter(({ node }) => nodesToAddInCurrentLayer.indexOf(node.urn || '') > -1)
+            .filter((_, idx) => positionsToAddInCurrentLayer.indexOf(idx) > -1)
             .filter(({ node }) => node.status?.removed !== true);
 
         const layerSize = filteredNodesInCurrentLayer.length;

From f87983d69dc62db5c58dc114f8796dcb9eb1cc95 Mon Sep 17 00:00:00 2001
From: John Joyce <john@acryl.io>
Date: Wed, 8 Nov 2023 13:29:37 -0800
Subject: [PATCH 180/259] refactor(pdl): Refactoring Assertion model enums out 
 (#9191)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../linkedin/assertion/AssertionResult.pdl    | 19 +--------------
 .../assertion/AssertionResultType.pdl         | 23 +++++++++++++++++++
 .../linkedin/assertion/AssertionRunEvent.pdl  |  7 +-----
 .../linkedin/assertion/AssertionRunStatus.pdl | 12 ++++++++++
 4 files changed, 37 insertions(+), 24 deletions(-)
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultType.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunStatus.pdl

diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl
index ded84e1969153..935f3e5976dfa 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResult.pdl
@@ -9,24 +9,7 @@ record AssertionResult {
   */
   @TimeseriesField = {}
   @Searchable = {}
-  type: enum AssertionResultType {
-    /**
-    *  The Assertion has not yet been fully evaluated
-    */
-    INIT
-    /**
-    *  The Assertion Succeeded
-    */
-    SUCCESS
-    /**
-    *  The Assertion Failed
-    */
-    FAILURE
-    /**
-    *  The Assertion encountered an Error
-    */
-    ERROR
-  }
+  type: AssertionResultType
 
   /**
    * Number of rows for evaluated batch
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultType.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultType.pdl
new file mode 100644
index 0000000000000..8954d94cced7b
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultType.pdl
@@ -0,0 +1,23 @@
+namespace com.linkedin.assertion
+
+/**
+*  The final result of evaluating an assertion, e.g. SUCCESS, FAILURE, or ERROR.
+*/
+enum AssertionResultType {
+  /**
+  *  The Assertion has not yet been fully evaluated
+  */
+  INIT
+  /**
+  *  The Assertion Succeeded
+  */
+  SUCCESS
+  /**
+  *  The Assertion Failed
+  */
+  FAILURE
+  /**
+  *  The Assertion encountered an Error
+  */
+  ERROR
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl
index 14f1204232740..55bcae77273db 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunEvent.pdl
@@ -39,12 +39,7 @@ record AssertionRunEvent {
   * The status of the assertion run as per this timeseries event.
   */
   @TimeseriesField = {}
-  status: enum AssertionRunStatus {
-    /**
-     * The Assertion Run has completed
-     */
-    COMPLETE
-  }
+  status: AssertionRunStatus
 
   /**
   * Results of assertion, present if the status is COMPLETE
diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunStatus.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunStatus.pdl
new file mode 100644
index 0000000000000..e4e17925ede82
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionRunStatus.pdl
@@ -0,0 +1,12 @@
+namespace com.linkedin.assertion
+
+
+/**
+* The lifecycle status of an assertion run.
+*/
+enum AssertionRunStatus {
+  /**
+   * The Assertion Run has completed
+   */
+  COMPLETE
+}
\ No newline at end of file

From f38c8087bb508a779d94d04967a9c449f6d93126 Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Wed, 8 Nov 2023 22:38:15 +0000
Subject: [PATCH 181/259] feat(auth): Add roles to policy engine validation
 logic (#9178)

---
 .../authorization/AuthorizedActors.java       |  1 +
 .../authorization/AuthorizerChain.java        |  5 +
 .../authorization/DataHubAuthorizer.java      |  8 +-
 .../datahub/authorization/PolicyEngine.java   | 43 +++-----
 .../authorization/DataHubAuthorizerTest.java  | 97 ++++++++++++++++---
 .../authorization/PolicyEngineTest.java       | 54 ++++++++++-
 .../datahub/plugins/test/TestAuthorizer.java  |  2 +-
 7 files changed, 162 insertions(+), 48 deletions(-)

diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java
index aec99e1b1e57a..5a9990552bb34 100644
--- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java
+++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java
@@ -15,6 +15,7 @@ public class AuthorizedActors {
   String privilege;
   List<Urn> users;
   List<Urn> groups;
+  List<Urn> roles;
   boolean allUsers;
   boolean allGroups;
 }
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java
index f8eca541e1efb..7e7a1de176f06 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java
@@ -126,11 +126,16 @@ private AuthorizedActors mergeAuthorizedActors(@Nullable AuthorizedActors origin
       mergedGroups = new ArrayList<>(groups);
     }
 
+    Set<Urn> roles = new HashSet<>(original.getRoles());
+    roles.addAll(other.getRoles());
+    List<Urn> mergedRoles = new ArrayList<>(roles);
+
     return AuthorizedActors.builder()
         .allUsers(original.isAllUsers() || other.isAllUsers())
         .allGroups(original.isAllGroups() || other.isAllGroups())
         .users(mergedUsers)
         .groups(mergedGroups)
+        .roles(mergedRoles)
         .build();
   }
 
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
index f8f99475de23e..956d635c7901a 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java
@@ -133,6 +133,7 @@ public AuthorizedActors authorizedActors(
 
     final List<Urn> authorizedUsers = new ArrayList<>();
     final List<Urn> authorizedGroups = new ArrayList<>();
+    final List<Urn> authorizedRoles = new ArrayList<>();
     boolean allUsers = false;
     boolean allGroups = false;
 
@@ -153,16 +154,17 @@ public AuthorizedActors authorizedActors(
       // Step 3: For each matching policy, add actors that are authorized.
       authorizedUsers.addAll(matchingActors.getUsers());
       authorizedGroups.addAll(matchingActors.getGroups());
-      if (matchingActors.allUsers()) {
+      authorizedRoles.addAll(matchingActors.getRoles());
+      if (matchingActors.getAllUsers()) {
         allUsers = true;
       }
-      if (matchingActors.allGroups()) {
+      if (matchingActors.getAllGroups()) {
         allGroups = true;
       }
     }
 
     // Step 4: Return all authorized users and groups.
-    return new AuthorizedActors(privilege, authorizedUsers, authorizedGroups, allUsers, allGroups);
+    return new AuthorizedActors(privilege, authorizedUsers, authorizedGroups, authorizedRoles, allUsers, allGroups);
   }
 
   /**
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java
index f8c017ea74e1f..da0ae26f2b1da 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java
@@ -32,7 +32,10 @@
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
 import lombok.RequiredArgsConstructor;
+import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
 
 import static com.linkedin.metadata.Constants.*;
@@ -75,6 +78,7 @@ public PolicyActors getMatchingActors(
       final Optional<ResolvedEntitySpec> resource) {
     final List<Urn> users = new ArrayList<>();
     final List<Urn> groups = new ArrayList<>();
+    final List<Urn> roles = new ArrayList<>();
     boolean allUsers = false;
     boolean allGroups = false;
     if (policyMatchesResource(policy, resource)) {
@@ -96,6 +100,9 @@ public PolicyActors getMatchingActors(
       if (actorFilter.getGroups() != null) {
         groups.addAll(actorFilter.getGroups());
       }
+      if (actorFilter.getRoles() != null) {
+        roles.addAll(actorFilter.getRoles());
+      }
 
       // 2. Fetch Actors based on resource ownership.
       if (actorFilter.isResourceOwners() && resource.isPresent()) {
@@ -104,7 +111,7 @@ public PolicyActors getMatchingActors(
         groups.addAll(groupOwners(owners));
       }
     }
-    return new PolicyActors(users, groups, allUsers, allGroups);
+    return new PolicyActors(users, groups, roles, allUsers, allGroups);
   }
 
   private boolean isPolicyApplicable(
@@ -438,34 +445,14 @@ public boolean isGranted() {
   /**
    * Class used to represent all valid users of a policy.
    */
+  @Value
+  @AllArgsConstructor(access = AccessLevel.PUBLIC)
   public static class PolicyActors {
-    final List<Urn> _users;
-    final List<Urn> _groups;
-    final Boolean _allUsers;
-    final Boolean _allGroups;
-
-    public PolicyActors(final List<Urn> users, final List<Urn> groups, final Boolean allUsers, final Boolean allGroups) {
-      _users = users;
-      _groups = groups;
-      _allUsers = allUsers;
-      _allGroups = allGroups;
-    }
-
-    public List<Urn> getUsers() {
-      return _users;
-    }
-
-    public List<Urn> getGroups() {
-      return _groups;
-    }
-
-    public Boolean allUsers() {
-      return _allUsers;
-    }
-
-    public Boolean allGroups() {
-      return _allGroups;
-    }
+    List<Urn> users;
+    List<Urn> groups;
+    List<Urn> roles;
+    Boolean allUsers;
+    Boolean allGroups;
   }
 
   private List<Urn> userOwners(final Set<String> owners) {
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
index babb1c5d00ee8..b0b206001209c 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/DataHubAuthorizerTest.java
@@ -21,6 +21,7 @@
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.identity.RoleMembership;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntity;
@@ -55,6 +56,7 @@
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.assertFalse;
 
 
 public class DataHubAuthorizerTest {
@@ -63,6 +65,7 @@ public class DataHubAuthorizerTest {
 
   private static final Urn PARENT_DOMAIN_URN = UrnUtils.getUrn("urn:li:domain:parent");
   private static final Urn CHILD_DOMAIN_URN = UrnUtils.getUrn("urn:li:domain:child");
+  private static final Urn USER_WITH_ADMIN_ROLE = UrnUtils.getUrn("urn:li:corpuser:user-with-admin");
 
   private EntityClient _entityClient;
   private DataHubAuthorizer _dataHubAuthorizer;
@@ -92,40 +95,56 @@ public void setupTest() throws Exception {
     final EnvelopedAspectMap childDomainPolicyAspectMap = new EnvelopedAspectMap();
     childDomainPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(childDomainPolicy.data())));
 
+    final Urn adminPolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:4");
+    final DataHubActorFilter actorFilter = new DataHubActorFilter();
+    actorFilter.setRoles(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataHubRole:Admin"))));
+    final DataHubPolicyInfo adminPolicy = createDataHubPolicyInfoFor(true, ImmutableList.of("EDIT_USER_PROFILE"), null, actorFilter);
+    final EnvelopedAspectMap adminPolicyAspectMap = new EnvelopedAspectMap();
+    adminPolicyAspectMap.put(DATAHUB_POLICY_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(adminPolicy.data())));
+
     final ScrollResult policySearchResult1 = new ScrollResult()
             .setScrollId("1")
-            .setNumEntities(4)
+            .setNumEntities(5)
             .setEntities(
                     new SearchEntityArray(
                             ImmutableList.of(new SearchEntity().setEntity(activePolicyUrn))));
 
     final ScrollResult policySearchResult2 = new ScrollResult()
             .setScrollId("2")
-            .setNumEntities(4)
+            .setNumEntities(5)
             .setEntities(
                     new SearchEntityArray(
                             ImmutableList.of(new SearchEntity().setEntity(inactivePolicyUrn))));
 
     final ScrollResult policySearchResult3 = new ScrollResult()
             .setScrollId("3")
-            .setNumEntities(4)
+            .setNumEntities(5)
             .setEntities(
                     new SearchEntityArray(
                             ImmutableList.of(new SearchEntity().setEntity(parentDomainPolicyUrn))));
 
     final ScrollResult policySearchResult4 = new ScrollResult()
-            .setNumEntities(4)
+        .setScrollId("4")
+        .setNumEntities(5)
             .setEntities(
                     new SearchEntityArray(
                             ImmutableList.of(
                                     new SearchEntity().setEntity(childDomainPolicyUrn))));
 
+    final ScrollResult policySearchResult5 = new ScrollResult()
+        .setNumEntities(5)
+        .setEntities(
+            new SearchEntityArray(
+                ImmutableList.of(
+                    new SearchEntity().setEntity(adminPolicyUrn))));
+
     when(_entityClient.scrollAcrossEntities(eq(List.of("dataHubPolicy")), eq(""), isNull(), any(), isNull(),
             anyInt(), eq(new SearchFlags().setFulltext(true).setSkipAggregates(true).setSkipHighlighting(true).setSkipCache(true)), any()))
             .thenReturn(policySearchResult1)
             .thenReturn(policySearchResult2)
             .thenReturn(policySearchResult3)
-            .thenReturn(policySearchResult4);
+            .thenReturn(policySearchResult4)
+            .thenReturn(policySearchResult5);
 
     when(_entityClient.batchGetV2(eq(POLICY_ENTITY_NAME), any(), eq(null), any())).thenAnswer(args -> {
       Set<Urn> inputUrns = args.getArgument(1);
@@ -140,6 +159,8 @@ public void setupTest() throws Exception {
           return Map.of(parentDomainPolicyUrn, new EntityResponse().setUrn(parentDomainPolicyUrn).setAspects(parentDomainPolicyAspectMap));
         case "urn:li:dataHubPolicy:3":
           return Map.of(childDomainPolicyUrn, new EntityResponse().setUrn(childDomainPolicyUrn).setAspects(childDomainPolicyAspectMap));
+        case "urn:li:dataHubPolicy:4":
+          return Map.of(adminPolicyUrn, new EntityResponse().setUrn(adminPolicyUrn).setAspects(adminPolicyAspectMap));
         default:
           throw new IllegalStateException();
       }
@@ -167,6 +188,10 @@ public void setupTest() throws Exception {
     when(_entityClient.batchGetV2(any(), eq(Collections.singleton(PARENT_DOMAIN_URN)), eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), any()))
         .thenReturn(createDomainPropertiesBatchResponse(null));
 
+    // Mocks to reach role membership for a user urn
+    when(_entityClient.batchGetV2(any(), eq(Collections.singleton(USER_WITH_ADMIN_ROLE)), eq(Collections.singleton(ROLE_MEMBERSHIP_ASPECT_NAME)), any())
+    ).thenReturn(createUserRoleMembershipBatchResponse(USER_WITH_ADMIN_ROLE, UrnUtils.getUrn("urn:li:dataHubRole:Admin")));
+
     final Authentication systemAuthentication = new Authentication(
         new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID),
         ""
@@ -302,6 +327,32 @@ public void testAuthorizedActorsActivePolicy() throws Exception {
     ));
   }
 
+  @Test
+  public void testAuthorizedRoleActivePolicy() throws Exception {
+    final AuthorizedActors actors =
+        _dataHubAuthorizer.authorizedActors("EDIT_USER_PROFILE", // Should be inside the active policy.
+            Optional.of(new EntitySpec("dataset", "urn:li:dataset:1")));
+
+    assertFalse(actors.isAllUsers());
+    assertFalse(actors.isAllGroups());
+    assertEquals(new HashSet<>(actors.getUsers()), ImmutableSet.of());
+    assertEquals(new HashSet<>(actors.getGroups()), ImmutableSet.of());
+    assertEquals(new HashSet<>(actors.getRoles()), ImmutableSet.of(UrnUtils.getUrn("urn:li:dataHubRole:Admin")));
+  }
+
+  @Test
+  public void testAuthorizationBasedOnRoleIsAllowed() {
+    EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
+
+    AuthorizationRequest request = new AuthorizationRequest(
+        USER_WITH_ADMIN_ROLE.toString(),
+        "EDIT_USER_PROFILE",
+        Optional.of(resourceSpec)
+    );
+
+    assertEquals(_dataHubAuthorizer.authorize(request).getType(), AuthorizationResult.Type.ALLOW);
+  }
+
   @Test
   public void testAuthorizationOnDomainWithPrivilegeIsAllowed() {
     EntitySpec resourceSpec = new EntitySpec("dataset", "urn:li:dataset:test");
@@ -342,13 +393,6 @@ public void testAuthorizationOnDomainWithoutPrivilegeIsDenied() {
   }
 
   private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List<String> privileges, @Nullable final Urn domain) throws Exception {
-    final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo();
-    dataHubPolicyInfo.setType(METADATA_POLICY_TYPE);
-    dataHubPolicyInfo.setState(active ? ACTIVE_POLICY_STATE : INACTIVE_POLICY_STATE);
-    dataHubPolicyInfo.setPrivileges(new StringArray(privileges));
-    dataHubPolicyInfo.setDisplayName("My Test Display");
-    dataHubPolicyInfo.setDescription("My test display!");
-    dataHubPolicyInfo.setEditable(true);
 
     List<Urn> users = ImmutableList.of(Urn.createFromString("urn:li:corpuser:user1"), Urn.createFromString("urn:li:corpuser:user2"));
     List<Urn> groups = ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group1"), Urn.createFromString("urn:li:corpGroup:group2"));
@@ -359,6 +403,20 @@ private DataHubPolicyInfo createDataHubPolicyInfo(boolean active, List<String> p
     actorFilter.setAllGroups(true);
     actorFilter.setUsers(new UrnArray(users));
     actorFilter.setGroups(new UrnArray(groups));
+
+    return createDataHubPolicyInfoFor(active, privileges, domain, actorFilter);
+  }
+
+  private DataHubPolicyInfo createDataHubPolicyInfoFor(boolean active, List<String> privileges,
+      @Nullable final Urn domain, DataHubActorFilter actorFilter) throws Exception {
+    final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo();
+    dataHubPolicyInfo.setType(METADATA_POLICY_TYPE);
+    dataHubPolicyInfo.setState(active ? ACTIVE_POLICY_STATE : INACTIVE_POLICY_STATE);
+    dataHubPolicyInfo.setPrivileges(new StringArray(privileges));
+    dataHubPolicyInfo.setDisplayName("My Test Display");
+    dataHubPolicyInfo.setDescription("My test display!");
+    dataHubPolicyInfo.setEditable(true);
+
     dataHubPolicyInfo.setActors(actorFilter);
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
@@ -429,6 +487,21 @@ private Map<Urn, EntityResponse> createDomainPropertiesBatchResponse(@Nullable f
     return batchResponse;
   }
 
+  private Map<Urn, EntityResponse> createUserRoleMembershipBatchResponse(final Urn userUrn, @Nullable final Urn roleUrn) {
+    final Map<Urn, EntityResponse> batchResponse = new HashMap<>();
+    final EntityResponse response = new EntityResponse();
+    EnvelopedAspectMap aspectMap = new EnvelopedAspectMap();
+    final RoleMembership membership = new RoleMembership();
+    if (roleUrn != null) {
+      membership.setRoles(new UrnArray(roleUrn));
+    }
+    aspectMap.put(ROLE_MEMBERSHIP_ASPECT_NAME, new EnvelopedAspect()
+        .setValue(new com.linkedin.entity.Aspect(membership.data())));
+    response.setAspects(aspectMap);
+    batchResponse.put(userUrn, response);
+    return batchResponse;
+  }
+
   private AuthorizerContext createAuthorizerContext(final Authentication systemAuthentication, final EntityClient entityClient) {
     return new AuthorizerContext(Collections.emptyMap(), new DefaultEntitySpecResolver(systemAuthentication, entityClient));
   }
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java
index be8c948f8ef89..2790c16ba75e6 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java
@@ -1041,6 +1041,7 @@ public void testGetMatchingActorsResourceMatch() throws Exception {
         Urn.createFromString("urn:li:corpuser:user2"))));
     actorFilter.setGroups(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group1"),
         Urn.createFromString("urn:li:corpGroup:group2"))));
+    actorFilter.setRoles(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:role:Admin"))));
     dataHubPolicyInfo.setActors(actorFilter);
 
     final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
@@ -1056,8 +1057,8 @@ public void testGetMatchingActorsResourceMatch() throws Exception {
             Collections.emptySet(), Collections.emptySet());
     PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec));
 
-    assertTrue(actors.allUsers());
-    assertTrue(actors.allGroups());
+    assertTrue(actors.getAllUsers());
+    assertTrue(actors.getAllGroups());
 
     assertEquals(actors.getUsers(),
         ImmutableList.of(Urn.createFromString("urn:li:corpuser:user1"), Urn.createFromString("urn:li:corpuser:user2"),
@@ -1068,6 +1069,8 @@ public void testGetMatchingActorsResourceMatch() throws Exception {
         Urn.createFromString("urn:li:corpGroup:group2"), Urn.createFromString(AUTHORIZED_GROUP) // Resource Owner
     ));
 
+    assertEquals(actors.getRoles(), ImmutableList.of(Urn.createFromString("urn:li:role:Admin")));
+
     // Verify aspect client called, entity client not called.
     verify(_entityClient, times(0)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
         eq(null), any());
@@ -1106,15 +1109,58 @@ public void testGetMatchingActorsNoResourceMatch() throws Exception {
         buildEntityResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy.
     PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec));
 
-    assertFalse(actors.allUsers());
-    assertFalse(actors.allGroups());
+    assertFalse(actors.getAllUsers());
+    assertFalse(actors.getAllGroups());
     assertEquals(actors.getUsers(), Collections.emptyList());
     assertEquals(actors.getGroups(), Collections.emptyList());
+    //assertEquals(actors.getRoles(), Collections.emptyList());
 
     // Verify no network calls
     verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any());
   }
 
+  @Test
+  public void testGetMatchingActorsByRoleResourceMatch() throws Exception {
+    final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo();
+    dataHubPolicyInfo.setType(METADATA_POLICY_TYPE);
+    dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE);
+    dataHubPolicyInfo.setPrivileges(new StringArray("EDIT_ENTITY_TAGS"));
+    dataHubPolicyInfo.setDisplayName("My Test Display");
+    dataHubPolicyInfo.setDescription("My test display!");
+    dataHubPolicyInfo.setEditable(true);
+
+    final DataHubActorFilter actorFilter = new DataHubActorFilter();
+    actorFilter.setResourceOwners(true);
+    actorFilter.setAllUsers(false);
+    actorFilter.setAllGroups(false);
+    actorFilter.setRoles(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataHubRole:Editor"))));
+    dataHubPolicyInfo.setActors(actorFilter);
+
+    final DataHubResourceFilter resourceFilter = new DataHubResourceFilter();
+    resourceFilter.setAllResources(false);
+    resourceFilter.setType("dataset");
+    StringArray resourceUrns = new StringArray();
+    resourceUrns.add(RESOURCE_URN);
+    resourceFilter.setResources(resourceUrns);
+    dataHubPolicyInfo.setResources(resourceFilter);
+
+    ResolvedEntitySpec resourceSpec = buildEntityResolvers("dataset", RESOURCE_URN, ImmutableSet.of(),
+        Collections.emptySet(), Collections.emptySet());
+
+    PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec));
+
+    assertFalse(actors.getAllUsers());
+    assertFalse(actors.getAllGroups());
+
+    assertEquals(actors.getUsers(), ImmutableList.of());
+    assertEquals(actors.getGroups(), ImmutableList.of());
+    assertEquals(actors.getRoles(), ImmutableList.of(Urn.createFromString("urn:li:dataHubRole:Editor")));
+
+    // Verify aspect client called, entity client not called.
+    verify(_entityClient, times(0)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)),
+        eq(null), any());
+  }
+
   private Ownership createOwnershipAspect(final Boolean addUserOwner, final Boolean addGroupOwner) throws Exception {
     final Ownership ownershipAspect = new Ownership();
     final OwnerArray owners = new OwnerArray();
diff --git a/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java b/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java
index 442ac1b0d287b..e5f3e223ff505 100644
--- a/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java
+++ b/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthorizer.java
@@ -75,7 +75,7 @@ public AuthorizationResult authorize(@Nonnull AuthorizationRequest request) {
 
   @Override
   public AuthorizedActors authorizedActors(String privilege, Optional<EntitySpec> resourceSpec) {
-    return new AuthorizedActors("ALL", null, null, true, true);
+    return new AuthorizedActors("ALL", null, null, null, true, true);
   }
 }
 

From f73ecfdcbbc35437fcb80c9e27e78908dae23ea7 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Wed, 8 Nov 2023 18:17:49 -0500
Subject: [PATCH 182/259] style(ingest/tableau): Rename tableau_constant to c
 (#9207)

---
 .../src/datahub/ingestion/source/tableau.py   | 597 ++++++++----------
 .../ingestion/source/tableau_common.py        |  14 +-
 2 files changed, 272 insertions(+), 339 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
index 4bc40b0aac964..08df7599510f4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
@@ -59,7 +59,7 @@
 )
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source import tableau_constant
+from datahub.ingestion.source import tableau_constant as c
 from datahub.ingestion.source.common.subtypes import (
     BIContainerSubTypes,
     DatasetSubTypes,
@@ -720,16 +720,12 @@ def get_connection_object_page(
                 query, connection_type, query_filter, count, offset, False
             )
 
-        if tableau_constant.ERRORS in query_data:
-            errors = query_data[tableau_constant.ERRORS]
+        if c.ERRORS in query_data:
+            errors = query_data[c.ERRORS]
             if all(
                 # The format of the error messages is highly unpredictable, so we have to
                 # be extra defensive with our parsing.
-                error
-                and (error.get(tableau_constant.EXTENSIONS) or {}).get(
-                    tableau_constant.SEVERITY
-                )
-                == tableau_constant.WARNING
+                error and (error.get(c.EXTENSIONS) or {}).get(c.SEVERITY) == c.WARNING
                 for error in errors
             ):
                 self.report.report_warning(key=connection_type, reason=f"{errors}")
@@ -737,14 +733,14 @@ def get_connection_object_page(
                 raise RuntimeError(f"Query {connection_type} error: {errors}")
 
         connection_object = (
-            query_data.get(tableau_constant.DATA).get(connection_type, {})
-            if query_data.get(tableau_constant.DATA)
+            query_data.get(c.DATA).get(connection_type, {})
+            if query_data.get(c.DATA)
             else {}
         )
 
-        total_count = connection_object.get(tableau_constant.TOTAL_COUNT, 0)
-        has_next_page = connection_object.get(tableau_constant.PAGE_INFO, {}).get(
-            tableau_constant.HAS_NEXT_PAGE, False
+        total_count = connection_object.get(c.TOTAL_COUNT, 0)
+        has_next_page = connection_object.get(c.PAGE_INFO, {}).get(
+            c.HAS_NEXT_PAGE, False
         )
         return connection_object, total_count, has_next_page
 
@@ -781,7 +777,7 @@ def get_connection_objects(
 
             offset += count
 
-            for obj in connection_objects.get(tableau_constant.NODES) or []:
+            for obj in connection_objects.get(c.NODES) or []:
                 yield obj
 
     def emit_workbooks(self) -> Iterable[MetadataWorkUnit]:
@@ -790,11 +786,11 @@ def emit_workbooks(self) -> Iterable[MetadataWorkUnit]:
                 project.name for project in self.tableau_project_registry.values()
             ]
             project_names_str: str = json.dumps(project_names)
-            projects = f"{tableau_constant.PROJECT_NAME_WITH_IN}: {project_names_str}"
+            projects = f"{c.PROJECT_NAME_WITH_IN}: {project_names_str}"
 
             for workbook in self.get_connection_objects(
                 workbook_graphql_query,
-                tableau_constant.WORKBOOKS_CONNECTION,
+                c.WORKBOOKS_CONNECTION,
                 projects,
                 page_size_override=self.config.workbook_page_size,
             ):
@@ -804,11 +800,9 @@ def emit_workbooks(self) -> Iterable[MetadataWorkUnit]:
                 # however Tableau supports projectLuidWithin in Tableau Cloud June 2022 / Server 2022.3 and later.
                 project_luid: Optional[str] = self._get_workbook_project_luid(workbook)
                 if project_luid not in self.tableau_project_registry.keys():
-                    wrk_name: Optional[str] = workbook.get(tableau_constant.NAME)
-                    wrk_id: Optional[str] = workbook.get(tableau_constant.ID)
-                    prj_name: Optional[str] = workbook.get(
-                        tableau_constant.PROJECT_NAME
-                    )
+                    wrk_name: Optional[str] = workbook.get(c.NAME)
+                    wrk_id: Optional[str] = workbook.get(c.ID)
+                    prj_name: Optional[str] = workbook.get(c.PROJECT_NAME)
 
                     logger.debug(
                         f"Skipping workbook {wrk_name}({wrk_id}) as it is project {prj_name}({project_luid}) not "
@@ -818,25 +812,22 @@ def emit_workbooks(self) -> Iterable[MetadataWorkUnit]:
 
                 yield from self.emit_workbook_as_container(workbook)
 
-                for sheet in workbook.get(tableau_constant.SHEETS, []):
-                    self.sheet_ids.append(sheet[tableau_constant.ID])
+                for sheet in workbook.get(c.SHEETS, []):
+                    self.sheet_ids.append(sheet[c.ID])
 
-                for dashboard in workbook.get(tableau_constant.DASHBOARDS, []):
-                    self.dashboard_ids.append(dashboard[tableau_constant.ID])
+                for dashboard in workbook.get(c.DASHBOARDS, []):
+                    self.dashboard_ids.append(dashboard[c.ID])
 
-                for ds in workbook.get(tableau_constant.EMBEDDED_DATA_SOURCES, []):
-                    self.embedded_datasource_ids_being_used.append(
-                        ds[tableau_constant.ID]
-                    )
+                for ds in workbook.get(c.EMBEDDED_DATA_SOURCES, []):
+                    self.embedded_datasource_ids_being_used.append(ds[c.ID])
 
     def _track_custom_sql_ids(self, field: dict) -> None:
         # Tableau shows custom sql datasource as a table in ColumnField's upstreamColumns.
-        for column in field.get(tableau_constant.UPSTREAM_COLUMNS, []):
+        for column in field.get(c.UPSTREAM_COLUMNS, []):
             table_id = (
-                column.get(tableau_constant.TABLE, {}).get(tableau_constant.ID)
-                if column.get(tableau_constant.TABLE)
-                and column[tableau_constant.TABLE][tableau_constant.TYPE_NAME]
-                == tableau_constant.CUSTOM_SQL_TABLE
+                column.get(c.TABLE, {}).get(c.ID)
+                if column.get(c.TABLE)
+                and column[c.TABLE][c.TYPE_NAME] == c.CUSTOM_SQL_TABLE
                 else None
             )
 
@@ -861,15 +852,15 @@ def _create_upstream_table_lineage(
         # and published datasource have same upstreamTables in this case.
         if upstream_tables and is_embedded_ds:
             logger.debug(
-                f"Embedded datasource {datasource.get(tableau_constant.ID)} has upstreamDatasources.\
+                f"Embedded datasource {datasource.get(c.ID)} has upstreamDatasources.\
                 Setting only upstreamDatasources lineage. The upstreamTables lineage \
                     will be set via upstream published datasource."
             )
         else:
             # This adds an edge to upstream DatabaseTables using `upstreamTables`
             upstreams, id_to_urn = self.get_upstream_tables(
-                datasource.get(tableau_constant.UPSTREAM_TABLES, []),
-                datasource.get(tableau_constant.NAME),
+                datasource.get(c.UPSTREAM_TABLES, []),
+                datasource.get(c.NAME),
                 browse_path,
                 is_custom_sql=False,
             )
@@ -878,23 +869,23 @@ def _create_upstream_table_lineage(
 
             # This adds an edge to upstream CustomSQLTables using `fields`.`upstreamColumns`.`table`
             csql_upstreams, csql_id_to_urn = self.get_upstream_csql_tables(
-                datasource.get(tableau_constant.FIELDS) or [],
+                datasource.get(c.FIELDS) or [],
             )
             upstream_tables.extend(csql_upstreams)
             table_id_to_urn.update(csql_id_to_urn)
 
         logger.debug(
-            f"A total of {len(upstream_tables)} upstream table edges found for datasource {datasource[tableau_constant.ID]}"
+            f"A total of {len(upstream_tables)} upstream table edges found for datasource {datasource[c.ID]}"
         )
 
         datasource_urn = builder.make_dataset_urn_with_platform_instance(
             platform=self.platform,
-            name=datasource[tableau_constant.ID],
+            name=datasource[c.ID],
             platform_instance=self.config.platform_instance,
             env=self.config.env,
         )
 
-        if datasource.get(tableau_constant.FIELDS):
+        if datasource.get(c.FIELDS):
             if self.config.extract_column_level_lineage:
                 # Find fine grained lineage for datasource column to datasource column edge,
                 # upstream columns may be from same datasource
@@ -912,20 +903,20 @@ def _create_upstream_table_lineage(
                 fine_grained_lineages.extend(upstream_columns)
 
                 logger.debug(
-                    f"A total of {len(fine_grained_lineages)} upstream column edges found for datasource {datasource[tableau_constant.ID]}"
+                    f"A total of {len(fine_grained_lineages)} upstream column edges found for datasource {datasource[c.ID]}"
                 )
 
         return upstream_tables, fine_grained_lineages
 
     def get_upstream_datasources(self, datasource: dict) -> List[Upstream]:
         upstream_tables = []
-        for ds in datasource.get(tableau_constant.UPSTREAM_DATA_SOURCES, []):
-            if ds[tableau_constant.ID] not in self.datasource_ids_being_used:
-                self.datasource_ids_being_used.append(ds[tableau_constant.ID])
+        for ds in datasource.get(c.UPSTREAM_DATA_SOURCES, []):
+            if ds[c.ID] not in self.datasource_ids_being_used:
+                self.datasource_ids_being_used.append(ds[c.ID])
 
             upstream_ds_urn = builder.make_dataset_urn_with_platform_instance(
                 platform=self.platform,
-                name=ds[tableau_constant.ID],
+                name=ds[c.ID],
                 platform_instance=self.config.platform_instance,
                 env=self.config.env,
             )
@@ -943,20 +934,15 @@ def get_upstream_csql_tables(
         csql_id_to_urn = {}
 
         for field in fields:
-            if not field.get(tableau_constant.UPSTREAM_COLUMNS):
+            if not field.get(c.UPSTREAM_COLUMNS):
                 continue
-            for upstream_col in field[tableau_constant.UPSTREAM_COLUMNS]:
+            for upstream_col in field[c.UPSTREAM_COLUMNS]:
                 if (
                     upstream_col
-                    and upstream_col.get(tableau_constant.TABLE)
-                    and upstream_col.get(tableau_constant.TABLE)[
-                        tableau_constant.TYPE_NAME
-                    ]
-                    == tableau_constant.CUSTOM_SQL_TABLE
+                    and upstream_col.get(c.TABLE)
+                    and upstream_col.get(c.TABLE)[c.TYPE_NAME] == c.CUSTOM_SQL_TABLE
                 ):
-                    upstream_table_id = upstream_col.get(tableau_constant.TABLE)[
-                        tableau_constant.ID
-                    ]
+                    upstream_table_id = upstream_col.get(c.TABLE)[c.ID]
 
                     csql_urn = builder.make_dataset_urn_with_platform_instance(
                         platform=self.platform,
@@ -986,18 +972,18 @@ def get_upstream_tables(
         for table in tables:
             # skip upstream tables when there is no column info when retrieving datasource
             # Lineage and Schema details for these will be taken care in self.emit_custom_sql_datasources()
-            num_tbl_cols: Optional[int] = table.get(
-                tableau_constant.COLUMNS_CONNECTION
-            ) and table[tableau_constant.COLUMNS_CONNECTION].get("totalCount")
+            num_tbl_cols: Optional[int] = table.get(c.COLUMNS_CONNECTION) and table[
+                c.COLUMNS_CONNECTION
+            ].get("totalCount")
             if not is_custom_sql and not num_tbl_cols:
                 logger.debug(
-                    f"Skipping upstream table with id {table[tableau_constant.ID]}, no columns: {table}"
+                    f"Skipping upstream table with id {table[c.ID]}, no columns: {table}"
                 )
                 continue
-            elif table[tableau_constant.NAME] is None:
+            elif table[c.NAME] is None:
                 self.report.num_upstream_table_skipped_no_name += 1
                 logger.warning(
-                    f"Skipping upstream table {table[tableau_constant.ID]} from lineage since its name is none: {table}"
+                    f"Skipping upstream table {table[c.ID]} from lineage since its name is none: {table}"
                 )
                 continue
 
@@ -1014,7 +1000,7 @@ def get_upstream_tables(
                 self.config.platform_instance_map,
                 self.config.lineage_overrides,
             )
-            table_id_to_urn[table[tableau_constant.ID]] = table_urn
+            table_id_to_urn[table[c.ID]] = table_urn
 
             upstream_table = Upstream(
                 dataset=table_urn,
@@ -1029,13 +1015,13 @@ def get_upstream_tables(
             if table_urn not in self.database_tables:
                 self.database_tables[table_urn] = DatabaseTable(
                     urn=table_urn,
-                    id=table[tableau_constant.ID],
+                    id=table[c.ID],
                     num_cols=num_tbl_cols,
                     paths={table_path} if table_path else set(),
                 )
             else:
                 self.database_tables[table_urn].update_table(
-                    table[tableau_constant.ID], num_tbl_cols, table_path
+                    table[c.ID], num_tbl_cols, table_path
                 )
 
         return upstream_tables, table_id_to_urn
@@ -1047,24 +1033,24 @@ def get_upstream_columns_of_fields_in_datasource(
         table_id_to_urn: Dict[str, str],
     ) -> List[FineGrainedLineage]:
         fine_grained_lineages = []
-        for field in datasource.get(tableau_constant.FIELDS) or []:
-            field_name = field.get(tableau_constant.NAME)
+        for field in datasource.get(c.FIELDS) or []:
+            field_name = field.get(c.NAME)
             # upstreamColumns lineage will be set via upstreamFields.
             # such as for CalculatedField
             if (
                 not field_name
-                or not field.get(tableau_constant.UPSTREAM_COLUMNS)
-                or field.get(tableau_constant.UPSTREAM_FIELDS)
+                or not field.get(c.UPSTREAM_COLUMNS)
+                or field.get(c.UPSTREAM_FIELDS)
             ):
                 continue
             input_columns = []
-            for upstream_col in field.get(tableau_constant.UPSTREAM_COLUMNS):
+            for upstream_col in field.get(c.UPSTREAM_COLUMNS):
                 if not upstream_col:
                     continue
-                name = upstream_col.get(tableau_constant.NAME)
+                name = upstream_col.get(c.NAME)
                 upstream_table_id = (
-                    upstream_col.get(tableau_constant.TABLE)[tableau_constant.ID]
-                    if upstream_col.get(tableau_constant.TABLE)
+                    upstream_col.get(c.TABLE)[c.ID]
+                    if upstream_col.get(c.TABLE)
                     else None
                 )
                 if (
@@ -1110,23 +1096,21 @@ def get_upstream_fields_of_field_in_datasource(
         self, datasource: dict, datasource_urn: str
     ) -> List[FineGrainedLineage]:
         fine_grained_lineages = []
-        for field in datasource.get(tableau_constant.FIELDS) or []:
-            field_name = field.get(tableau_constant.NAME)
+        for field in datasource.get(c.FIELDS) or []:
+            field_name = field.get(c.NAME)
             # It is observed that upstreamFields gives one-hop field
             # lineage, and not multi-hop field lineage
             # This behavior is as desired in our case.
-            if not field_name or not field.get(tableau_constant.UPSTREAM_FIELDS):
+            if not field_name or not field.get(c.UPSTREAM_FIELDS):
                 continue
             input_fields = []
-            for upstream_field in field.get(tableau_constant.UPSTREAM_FIELDS):
+            for upstream_field in field.get(c.UPSTREAM_FIELDS):
                 if not upstream_field:
                     continue
-                name = upstream_field.get(tableau_constant.NAME)
+                name = upstream_field.get(c.NAME)
                 upstream_ds_id = (
-                    upstream_field.get(tableau_constant.DATA_SOURCE)[
-                        tableau_constant.ID
-                    ]
-                    if upstream_field.get(tableau_constant.DATA_SOURCE)
+                    upstream_field.get(c.DATA_SOURCE)[c.ID]
+                    if upstream_field.get(c.DATA_SOURCE)
                     else None
                 )
                 if name and upstream_ds_id:
@@ -1212,35 +1196,37 @@ def get_upstream_fields_from_custom_sql(
         return fine_grained_lineages
 
     def get_transform_operation(self, field: dict) -> str:
-        field_type = field[tableau_constant.TYPE_NAME]
+        field_type = field[c.TYPE_NAME]
         if field_type in (
-            tableau_constant.DATA_SOURCE_FIELD,
-            tableau_constant.COLUMN_FIELD,
+            c.DATA_SOURCE_FIELD,
+            c.COLUMN_FIELD,
         ):
-            op = tableau_constant.IDENTITY  # How to specify exact same
-        elif field_type == tableau_constant.CALCULATED_FIELD:
+            op = c.IDENTITY  # How to specify exact same
+        elif field_type == c.CALCULATED_FIELD:
             op = field_type
-            if field.get(tableau_constant.FORMULA):
-                op += f"formula: {field.get(tableau_constant.FORMULA)}"
+            if field.get(c.FORMULA):
+                op += f"formula: {field.get(c.FORMULA)}"
         else:
             op = field_type  # BinField, CombinedField, etc
 
         return op
 
     def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]:
-        custom_sql_filter = f"{tableau_constant.ID_WITH_IN}: {json.dumps(self.custom_sql_ids_being_used)}"
+        custom_sql_filter = (
+            f"{c.ID_WITH_IN}: {json.dumps(self.custom_sql_ids_being_used)}"
+        )
 
         custom_sql_connection = list(
             self.get_connection_objects(
                 custom_sql_graphql_query,
-                tableau_constant.CUSTOM_SQL_TABLE_CONNECTION,
+                c.CUSTOM_SQL_TABLE_CONNECTION,
                 custom_sql_filter,
             )
         )
         unique_custom_sql = get_unique_custom_sql(custom_sql_connection)
 
         for csql in unique_custom_sql:
-            csql_id: str = csql[tableau_constant.ID]
+            csql_id: str = csql[c.ID]
             csql_urn = builder.make_dataset_urn_with_platform_instance(
                 platform=self.platform,
                 name=csql_id,
@@ -1256,40 +1242,33 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]:
 
             datasource_name = None
             project = None
-            if len(csql[tableau_constant.DATA_SOURCES]) > 0:
+            if len(csql[c.DATA_SOURCES]) > 0:
                 # CustomSQLTable id owned by exactly one tableau data source
                 logger.debug(
-                    f"Number of datasources referencing CustomSQLTable: {len(csql[tableau_constant.DATA_SOURCES])}"
+                    f"Number of datasources referencing CustomSQLTable: {len(csql[c.DATA_SOURCES])}"
                 )
 
-                datasource = csql[tableau_constant.DATA_SOURCES][0]
-                datasource_name = datasource.get(tableau_constant.NAME)
+                datasource = csql[c.DATA_SOURCES][0]
+                datasource_name = datasource.get(c.NAME)
                 if datasource.get(
-                    tableau_constant.TYPE_NAME
-                ) == tableau_constant.EMBEDDED_DATA_SOURCE and datasource.get(
-                    tableau_constant.WORKBOOK
-                ):
+                    c.TYPE_NAME
+                ) == c.EMBEDDED_DATA_SOURCE and datasource.get(c.WORKBOOK):
                     datasource_name = (
-                        f"{datasource.get(tableau_constant.WORKBOOK).get(tableau_constant.NAME)}/{datasource_name}"
-                        if datasource_name
-                        and datasource.get(tableau_constant.WORKBOOK).get(
-                            tableau_constant.NAME
-                        )
+                        f"{datasource.get(c.WORKBOOK).get(c.NAME)}/{datasource_name}"
+                        if datasource_name and datasource.get(c.WORKBOOK).get(c.NAME)
                         else None
                     )
                     logger.debug(
                         f"Adding datasource {datasource_name}({datasource.get('id')}) to container"
                     )
                     yield from add_entity_to_container(
-                        self.gen_workbook_key(
-                            datasource[tableau_constant.WORKBOOK][tableau_constant.ID]
-                        ),
-                        tableau_constant.DATASET,
+                        self.gen_workbook_key(datasource[c.WORKBOOK][c.ID]),
+                        c.DATASET,
                         dataset_snapshot.urn,
                     )
                 project = self._get_project_browse_path_name(datasource)
 
-                tables = csql.get(tableau_constant.TABLES, [])
+                tables = csql.get(c.TABLES, [])
 
                 if tables:
                     # lineage from custom sql -> datasets/tables #
@@ -1306,9 +1285,8 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]:
             #  Schema Metadata
             # if condition is needed as graphQL return "cloumns": None
             columns: List[Dict[Any, Any]] = (
-                cast(List[Dict[Any, Any]], csql.get(tableau_constant.COLUMNS))
-                if tableau_constant.COLUMNS in csql
-                and csql.get(tableau_constant.COLUMNS) is not None
+                cast(List[Dict[Any, Any]], csql.get(c.COLUMNS))
+                if c.COLUMNS in csql and csql.get(c.COLUMNS) is not None
                 else []
             )
             schema_metadata = self.get_schema_metadata_for_custom_sql(columns)
@@ -1320,7 +1298,7 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]:
             if project and datasource_name:
                 browse_paths = BrowsePathsClass(
                     paths=[
-                        f"/{self.config.env.lower()}/{self.platform}/{project}/{datasource[tableau_constant.NAME]}"
+                        f"/{self.config.env.lower()}/{self.platform}/{project}/{datasource[c.NAME]}"
                     ]
                 )
                 dataset_snapshot.aspects.append(browse_paths)
@@ -1328,27 +1306,25 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]:
                 logger.debug(f"Browse path not set for Custom SQL table {csql_id}")
 
             dataset_properties = DatasetPropertiesClass(
-                name=csql.get(tableau_constant.NAME),
-                description=csql.get(tableau_constant.DESCRIPTION),
+                name=csql.get(c.NAME),
+                description=csql.get(c.DESCRIPTION),
             )
 
             dataset_snapshot.aspects.append(dataset_properties)
 
-            if csql.get(tableau_constant.QUERY):
+            if csql.get(c.QUERY):
                 view_properties = ViewPropertiesClass(
                     materialized=False,
-                    viewLanguage=tableau_constant.SQL,
-                    viewLogic=clean_query(csql[tableau_constant.QUERY]),
+                    viewLanguage=c.SQL,
+                    viewLogic=clean_query(csql[c.QUERY]),
                 )
                 dataset_snapshot.aspects.append(view_properties)
 
             yield self.get_metadata_change_event(dataset_snapshot)
             yield self.get_metadata_change_proposal(
                 dataset_snapshot.urn,
-                aspect_name=tableau_constant.SUB_TYPES,
-                aspect=SubTypesClass(
-                    typeNames=[DatasetSubTypes.VIEW, tableau_constant.CUSTOM_SQL]
-                ),
+                aspect_name=c.SUB_TYPES,
+                aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW, c.CUSTOM_SQL]),
             )
 
     def get_schema_metadata_for_custom_sql(
@@ -1359,21 +1335,19 @@ def get_schema_metadata_for_custom_sql(
         for field in columns:
             # Datasource fields
 
-            if field.get(tableau_constant.NAME) is None:
+            if field.get(c.NAME) is None:
                 self.report.num_csql_field_skipped_no_name += 1
                 logger.warning(
-                    f"Skipping field {field[tableau_constant.ID]} from schema since its name is none"
+                    f"Skipping field {field[c.ID]} from schema since its name is none"
                 )
                 continue
-            nativeDataType = field.get(
-                tableau_constant.REMOTE_TYPE, tableau_constant.UNKNOWN
-            )
+            nativeDataType = field.get(c.REMOTE_TYPE, c.UNKNOWN)
             TypeClass = FIELD_TYPE_MAPPING.get(nativeDataType, NullTypeClass)
             schema_field = SchemaField(
-                fieldPath=field[tableau_constant.NAME],
+                fieldPath=field[c.NAME],
                 type=SchemaFieldDataType(type=TypeClass()),
                 nativeDataType=nativeDataType,
-                description=field.get(tableau_constant.DESCRIPTION),
+                description=field.get(c.DESCRIPTION),
             )
             fields.append(schema_field)
 
@@ -1391,28 +1365,25 @@ def _get_published_datasource_project_luid(self, ds: dict) -> Optional[str]:
         # This is fallback in case "get all datasources" query fails for some reason.
         # It is possible due to https://github.com/tableau/server-client-python/issues/1210
         if (
-            ds.get(tableau_constant.LUID)
-            and ds[tableau_constant.LUID] not in self.datasource_project_map.keys()
+            ds.get(c.LUID)
+            and ds[c.LUID] not in self.datasource_project_map.keys()
             and self.report.get_all_datasources_query_failed
         ):
             logger.debug(
-                f"published datasource {ds.get(tableau_constant.NAME)} project_luid not found."
-                f" Running get datasource query for {ds[tableau_constant.LUID]}"
+                f"published datasource {ds.get(c.NAME)} project_luid not found."
+                f" Running get datasource query for {ds[c.LUID]}"
             )
             # Query and update self.datasource_project_map with luid
-            self._query_published_datasource_for_project_luid(ds[tableau_constant.LUID])
+            self._query_published_datasource_for_project_luid(ds[c.LUID])
 
         if (
-            ds.get(tableau_constant.LUID)
-            and ds[tableau_constant.LUID] in self.datasource_project_map.keys()
-            and self.datasource_project_map[ds[tableau_constant.LUID]]
-            in self.tableau_project_registry
+            ds.get(c.LUID)
+            and ds[c.LUID] in self.datasource_project_map.keys()
+            and self.datasource_project_map[ds[c.LUID]] in self.tableau_project_registry
         ):
-            return self.datasource_project_map[ds[tableau_constant.LUID]]
+            return self.datasource_project_map[ds[c.LUID]]
 
-        logger.debug(
-            f"published datasource {ds.get(tableau_constant.NAME)} project_luid not found"
-        )
+        logger.debug(f"published datasource {ds.get(c.NAME)} project_luid not found")
 
         return None
 
@@ -1437,60 +1408,52 @@ def _query_published_datasource_for_project_luid(self, ds_luid: str) -> None:
             logger.debug("Error stack trace", exc_info=True)
 
     def _get_workbook_project_luid(self, wb: dict) -> Optional[str]:
-        if wb.get(tableau_constant.LUID) and self.workbook_project_map.get(
-            wb[tableau_constant.LUID]
-        ):
-            return self.workbook_project_map[wb[tableau_constant.LUID]]
+        if wb.get(c.LUID) and self.workbook_project_map.get(wb[c.LUID]):
+            return self.workbook_project_map[wb[c.LUID]]
 
-        logger.debug(f"workbook {wb.get(tableau_constant.NAME)} project_luid not found")
+        logger.debug(f"workbook {wb.get(c.NAME)} project_luid not found")
 
         return None
 
     def _get_embedded_datasource_project_luid(self, ds: dict) -> Optional[str]:
-        if ds.get(tableau_constant.WORKBOOK):
+        if ds.get(c.WORKBOOK):
             project_luid: Optional[str] = self._get_workbook_project_luid(
-                ds[tableau_constant.WORKBOOK]
+                ds[c.WORKBOOK]
             )
 
             if project_luid and project_luid in self.tableau_project_registry:
                 return project_luid
 
-        logger.debug(
-            f"embedded datasource {ds.get(tableau_constant.NAME)} project_luid not found"
-        )
+        logger.debug(f"embedded datasource {ds.get(c.NAME)} project_luid not found")
 
         return None
 
     def _get_datasource_project_luid(self, ds: dict) -> Optional[str]:
         # Only published and embedded data-sources are supported
-        ds_type: Optional[str] = ds.get(tableau_constant.TYPE_NAME)
+        ds_type: Optional[str] = ds.get(c.TYPE_NAME)
         if ds_type not in (
-            tableau_constant.PUBLISHED_DATA_SOURCE,
-            tableau_constant.EMBEDDED_DATA_SOURCE,
+            c.PUBLISHED_DATA_SOURCE,
+            c.EMBEDDED_DATA_SOURCE,
         ):
             logger.debug(
-                f"datasource {ds.get(tableau_constant.NAME)} type {ds.get(tableau_constant.TYPE_NAME)} is "
+                f"datasource {ds.get(c.NAME)} type {ds.get(c.TYPE_NAME)} is "
                 f"unsupported"
             )
             return None
 
         func_selector: Any = {
-            tableau_constant.PUBLISHED_DATA_SOURCE: self._get_published_datasource_project_luid,
-            tableau_constant.EMBEDDED_DATA_SOURCE: self._get_embedded_datasource_project_luid,
+            c.PUBLISHED_DATA_SOURCE: self._get_published_datasource_project_luid,
+            c.EMBEDDED_DATA_SOURCE: self._get_embedded_datasource_project_luid,
         }
 
         return func_selector[ds_type](ds)
 
     @staticmethod
     def _get_datasource_project_name(ds: dict) -> Optional[str]:
-        if ds.get(
-            tableau_constant.TYPE_NAME
-        ) == tableau_constant.EMBEDDED_DATA_SOURCE and ds.get(
-            tableau_constant.WORKBOOK
-        ):
-            return ds[tableau_constant.WORKBOOK].get(tableau_constant.PROJECT_NAME)
-        if ds.get(tableau_constant.TYPE_NAME) == tableau_constant.PUBLISHED_DATA_SOURCE:
-            return ds.get(tableau_constant.PROJECT_NAME)
+        if ds.get(c.TYPE_NAME) == c.EMBEDDED_DATA_SOURCE and ds.get(c.WORKBOOK):
+            return ds[c.WORKBOOK].get(c.PROJECT_NAME)
+        if ds.get(c.TYPE_NAME) == c.PUBLISHED_DATA_SOURCE:
+            return ds.get(c.PROJECT_NAME)
         return None
 
     def _get_project_browse_path_name(self, ds: dict) -> Optional[str]:
@@ -1502,7 +1465,7 @@ def _get_project_browse_path_name(self, ds: dict) -> Optional[str]:
         project_luid = self._get_datasource_project_luid(ds)
         if project_luid is None:
             logger.warning(
-                f"Could not load project hierarchy for datasource {ds.get(tableau_constant.NAME)}. Please check permissions."
+                f"Could not load project hierarchy for datasource {ds.get(c.NAME)}. Please check permissions."
             )
             logger.debug(f"datasource = {ds}")
             return None
@@ -1515,7 +1478,7 @@ def _create_lineage_to_upstream_tables(
         # This adds an edge to upstream DatabaseTables using `upstreamTables`
         upstream_tables, _ = self.get_upstream_tables(
             tables,
-            datasource.get(tableau_constant.NAME) or "",
+            datasource.get(c.NAME) or "",
             self._get_project_browse_path_name(datasource),
             is_custom_sql=True,
         )
@@ -1524,7 +1487,7 @@ def _create_lineage_to_upstream_tables(
             upstream_lineage = UpstreamLineage(upstreams=upstream_tables)
             yield self.get_metadata_change_proposal(
                 csql_urn,
-                aspect_name=tableau_constant.UPSTREAM_LINEAGE,
+                aspect_name=c.UPSTREAM_LINEAGE,
                 aspect=upstream_lineage,
             )
 
@@ -1547,22 +1510,19 @@ def parse_custom_sql(
             ]
         ],
     ) -> Optional["SqlParsingResult"]:
-        database_info = datasource.get(tableau_constant.DATABASE) or {}
+        database_info = datasource.get(c.DATABASE) or {}
 
-        if datasource.get(tableau_constant.IS_UNSUPPORTED_CUSTOM_SQL) in (None, False):
+        if datasource.get(c.IS_UNSUPPORTED_CUSTOM_SQL) in (None, False):
             logger.debug(f"datasource {datasource_urn} is not created from custom sql")
             return None
 
-        if (
-            tableau_constant.NAME not in database_info
-            or tableau_constant.CONNECTION_TYPE not in database_info
-        ):
+        if c.NAME not in database_info or c.CONNECTION_TYPE not in database_info:
             logger.debug(
                 f"database information is missing from datasource {datasource_urn}"
             )
             return None
 
-        query = datasource.get(tableau_constant.QUERY)
+        query = datasource.get(c.QUERY)
         if query is None:
             logger.debug(
                 f"raw sql query is not available for datasource {datasource_urn}"
@@ -1571,13 +1531,13 @@ def parse_custom_sql(
 
         logger.debug(f"Parsing sql={query}")
 
-        upstream_db = database_info.get(tableau_constant.NAME)
+        upstream_db = database_info.get(c.NAME)
 
         if func_overridden_info is not None:
             # Override the information as per configuration
             upstream_db, platform_instance, platform, _ = func_overridden_info(
-                database_info[tableau_constant.CONNECTION_TYPE],
-                database_info.get(tableau_constant.NAME),
+                database_info[c.CONNECTION_TYPE],
+                database_info.get(c.NAME),
                 self.config.platform_instance_map,
                 self.config.lineage_overrides,
             )
@@ -1631,7 +1591,7 @@ def _create_lineage_from_unsupported_csql(
 
         yield self.get_metadata_change_proposal(
             csql_urn,
-            aspect_name=tableau_constant.UPSTREAM_LINEAGE,
+            aspect_name=c.UPSTREAM_LINEAGE,
             aspect=upstream_lineage,
         )
 
@@ -1642,10 +1602,10 @@ def _get_schema_metadata_for_datasource(
         for field in datasource_fields:
             # check datasource - custom sql relations from a field being referenced
             self._track_custom_sql_ids(field)
-            if field.get(tableau_constant.NAME) is None:
+            if field.get(c.NAME) is None:
                 self.report.num_upstream_table_skipped_no_name += 1
                 logger.warning(
-                    f"Skipping field {field[tableau_constant.ID]} from schema since its name is none"
+                    f"Skipping field {field[c.ID]} from schema since its name is none"
                 )
                 continue
 
@@ -1678,7 +1638,7 @@ def get_metadata_change_proposal(
         aspect: Union["UpstreamLineage", "SubTypesClass"],
     ) -> MetadataWorkUnit:
         return MetadataChangeProposalWrapper(
-            entityType=tableau_constant.DATASET,
+            entityType=c.DATASET,
             changeType=ChangeTypeClass.UPSERT,
             entityUrn=urn,
             aspectName=aspect_name,
@@ -1696,10 +1656,8 @@ def emit_datasource(
             datasource_info = datasource
 
         browse_path = self._get_project_browse_path_name(datasource)
-        logger.debug(
-            f"datasource {datasource.get(tableau_constant.NAME)} browse-path {browse_path}"
-        )
-        datasource_id = datasource[tableau_constant.ID]
+        logger.debug(f"datasource {datasource.get(c.NAME)} browse-path {browse_path}")
+        datasource_id = datasource[c.ID]
         datasource_urn = builder.make_dataset_urn_with_platform_instance(
             self.platform, datasource_id, self.config.platform_instance, self.config.env
         )
@@ -1713,13 +1671,10 @@ def emit_datasource(
 
         # Browse path
 
-        if (
-            browse_path
-            and is_embedded_ds
-            and workbook
-            and workbook.get(tableau_constant.NAME)
-        ):
-            browse_path = f"{browse_path}/{workbook[tableau_constant.NAME].replace('/', REPLACE_SLASH_CHAR)}"
+        if browse_path and is_embedded_ds and workbook and workbook.get(c.NAME):
+            browse_path = (
+                f"{browse_path}/{workbook[c.NAME].replace('/', REPLACE_SLASH_CHAR)}"
+            )
 
         if browse_path:
             browse_paths = BrowsePathsClass(
@@ -1729,12 +1684,10 @@ def emit_datasource(
 
         # Ownership
         owner = (
-            self._get_ownership(
-                datasource_info[tableau_constant.OWNER][tableau_constant.USERNAME]
-            )
+            self._get_ownership(datasource_info[c.OWNER][c.USERNAME])
             if datasource_info
-            and datasource_info.get(tableau_constant.OWNER)
-            and datasource_info[tableau_constant.OWNER].get(tableau_constant.USERNAME)
+            and datasource_info.get(c.OWNER)
+            and datasource_info[c.OWNER].get(c.USERNAME)
             else None
         )
         if owner is not None:
@@ -1742,24 +1695,22 @@ def emit_datasource(
 
         # Dataset properties
         dataset_props = DatasetPropertiesClass(
-            name=datasource.get(tableau_constant.NAME),
-            description=datasource.get(tableau_constant.DESCRIPTION),
+            name=datasource.get(c.NAME),
+            description=datasource.get(c.DESCRIPTION),
             customProperties=self.get_custom_props_from_dict(
                 datasource,
                 [
-                    tableau_constant.HAS_EXTRACTS,
-                    tableau_constant.EXTRACT_LAST_REFRESH_TIME,
-                    tableau_constant.EXTRACT_LAST_INCREMENTAL_UPDATE_TIME,
-                    tableau_constant.EXTRACT_LAST_UPDATE_TIME,
+                    c.HAS_EXTRACTS,
+                    c.EXTRACT_LAST_REFRESH_TIME,
+                    c.EXTRACT_LAST_INCREMENTAL_UPDATE_TIME,
+                    c.EXTRACT_LAST_UPDATE_TIME,
                 ],
             ),
         )
         dataset_snapshot.aspects.append(dataset_props)
 
         # Upstream Tables
-        if datasource.get(tableau_constant.UPSTREAM_TABLES) or datasource.get(
-            tableau_constant.UPSTREAM_DATA_SOURCES
-        ):
+        if datasource.get(c.UPSTREAM_TABLES) or datasource.get(c.UPSTREAM_DATA_SOURCES):
             # datasource -> db table relations
             (
                 upstream_tables,
@@ -1779,13 +1730,13 @@ def emit_datasource(
                 )
                 yield self.get_metadata_change_proposal(
                     datasource_urn,
-                    aspect_name=tableau_constant.UPSTREAM_LINEAGE,
+                    aspect_name=c.UPSTREAM_LINEAGE,
                     aspect=upstream_lineage,
                 )
 
         # Datasource Fields
         schema_metadata = self._get_schema_metadata_for_datasource(
-            datasource.get(tableau_constant.FIELDS, [])
+            datasource.get(c.FIELDS, [])
         )
         if schema_metadata is not None:
             dataset_snapshot.aspects.append(schema_metadata)
@@ -1793,7 +1744,7 @@ def emit_datasource(
         yield self.get_metadata_change_event(dataset_snapshot)
         yield self.get_metadata_change_proposal(
             dataset_snapshot.urn,
-            aspect_name=tableau_constant.SUB_TYPES,
+            aspect_name=c.SUB_TYPES,
             aspect=SubTypesClass(
                 typeNames=(
                     ["Embedded Data Source"]
@@ -1809,7 +1760,7 @@ def emit_datasource(
         if container_key is not None:
             yield from add_entity_to_container(
                 container_key,
-                tableau_constant.DATASET,
+                c.DATASET,
                 dataset_snapshot.urn,
             )
 
@@ -1822,10 +1773,10 @@ def _get_datasource_container_key(
         container_key: Optional[ContainerKey] = None
         if is_embedded_ds:  # It is embedded then parent is container is workbook
             if workbook is not None:
-                container_key = self.gen_workbook_key(workbook[tableau_constant.ID])
+                container_key = self.gen_workbook_key(workbook[c.ID])
             else:
                 logger.warning(
-                    f"Parent container not set for embedded datasource {datasource[tableau_constant.ID]}"
+                    f"Parent container not set for embedded datasource {datasource[c.ID]}"
                 )
         else:
             parent_project_luid = self._get_published_datasource_project_luid(
@@ -1836,17 +1787,19 @@ def _get_datasource_container_key(
                 container_key = self.gen_project_key(parent_project_luid)
             else:
                 logger.warning(
-                    f"Parent container not set for published datasource {datasource[tableau_constant.ID]}"
+                    f"Parent container not set for published datasource {datasource[c.ID]}"
                 )
 
         return container_key
 
     def emit_published_datasources(self) -> Iterable[MetadataWorkUnit]:
-        datasource_filter = f"{tableau_constant.ID_WITH_IN}: {json.dumps(self.datasource_ids_being_used)}"
+        datasource_filter = (
+            f"{c.ID_WITH_IN}: {json.dumps(self.datasource_ids_being_used)}"
+        )
 
         for datasource in self.get_connection_objects(
             published_datasource_graphql_query,
-            tableau_constant.PUBLISHED_DATA_SOURCES_CONNECTION,
+            c.PUBLISHED_DATA_SOURCES_CONNECTION,
             datasource_filter,
         ):
             yield from self.emit_datasource(datasource)
@@ -1855,11 +1808,13 @@ def emit_upstream_tables(self) -> Iterable[MetadataWorkUnit]:
         database_table_id_to_urn_map: Dict[str, str] = dict()
         for urn, tbl in self.database_tables.items():
             database_table_id_to_urn_map[tbl.id] = urn
-        tables_filter = f"{tableau_constant.ID_WITH_IN}: {json.dumps(list(database_table_id_to_urn_map.keys()))}"
+        tables_filter = (
+            f"{c.ID_WITH_IN}: {json.dumps(list(database_table_id_to_urn_map.keys()))}"
+        )
 
         for table in self.get_connection_objects(
             database_tables_graphql_query,
-            tableau_constant.DATABASE_TABLES_CONNECTION,
+            c.DATABASE_TABLES_CONNECTION,
             tables_filter,
         ):
             yield from self.emit_table(table, database_table_id_to_urn_map)
@@ -1867,11 +1822,9 @@ def emit_upstream_tables(self) -> Iterable[MetadataWorkUnit]:
     def emit_table(
         self, table: dict, database_table_id_to_urn_map: Dict[str, str]
     ) -> Iterable[MetadataWorkUnit]:
-        database_table = self.database_tables[
-            database_table_id_to_urn_map[table[tableau_constant.ID]]
-        ]
-        columns = table.get(tableau_constant.COLUMNS, [])
-        is_embedded = table.get(tableau_constant.IS_EMBEDDED) or False
+        database_table = self.database_tables[database_table_id_to_urn_map[table[c.ID]]]
+        columns = table.get(c.COLUMNS, [])
+        is_embedded = table.get(c.IS_EMBEDDED) or False
         if not is_embedded and not self.config.ingest_tables_external:
             logger.debug(
                 f"Skipping external table {database_table.urn} as ingest_tables_external is set to False"
@@ -1907,21 +1860,19 @@ def get_schema_metadata_for_table(
         if columns:
             fields = []
             for field in columns:
-                if field.get(tableau_constant.NAME) is None:
+                if field.get(c.NAME) is None:
                     self.report.num_table_field_skipped_no_name += 1
                     logger.warning(
-                        f"Skipping field {field[tableau_constant.ID]} from schema since its name is none"
+                        f"Skipping field {field[c.ID]} from schema since its name is none"
                     )
                     continue
-                nativeDataType = field.get(
-                    tableau_constant.REMOTE_TYPE, tableau_constant.UNKNOWN
-                )
+                nativeDataType = field.get(c.REMOTE_TYPE, c.UNKNOWN)
                 TypeClass = FIELD_TYPE_MAPPING.get(nativeDataType, NullTypeClass)
 
                 schema_field = SchemaField(
-                    fieldPath=field[tableau_constant.NAME],
+                    fieldPath=field[c.NAME],
                     type=SchemaFieldDataType(type=TypeClass()),
-                    description=field.get(tableau_constant.DESCRIPTION),
+                    description=field.get(c.DESCRIPTION),
                     nativeDataType=nativeDataType,
                 )
 
@@ -1941,11 +1892,9 @@ def get_schema_metadata_for_table(
     def get_sheetwise_upstream_datasources(self, sheet: dict) -> set:
         sheet_upstream_datasources = set()
 
-        for field in sheet.get(tableau_constant.DATA_SOURCE_FIELDS) or []:
-            if field and field.get(tableau_constant.DATA_SOURCE):
-                sheet_upstream_datasources.add(
-                    field[tableau_constant.DATA_SOURCE][tableau_constant.ID]
-                )
+        for field in sheet.get(c.DATA_SOURCE_FIELDS) or []:
+            if field and field.get(c.DATA_SOURCE):
+                sheet_upstream_datasources.add(field[c.DATA_SOURCE][c.ID])
 
         return sheet_upstream_datasources
 
@@ -1961,20 +1910,20 @@ def _create_datahub_chart_usage_stat(
     def _get_chart_stat_wu(
         self, sheet: dict, sheet_urn: str
     ) -> Optional[MetadataWorkUnit]:
-        luid: Optional[str] = sheet.get(tableau_constant.LUID)
+        luid: Optional[str] = sheet.get(c.LUID)
         if luid is None:
             logger.debug(
                 "stat:luid is none for sheet %s(id:%s)",
-                sheet.get(tableau_constant.NAME),
-                sheet.get(tableau_constant.ID),
+                sheet.get(c.NAME),
+                sheet.get(c.ID),
             )
             return None
         usage_stat: Optional[UsageStat] = self.tableau_stat_registry.get(luid)
         if usage_stat is None:
             logger.debug(
                 "stat:UsageStat is not available in tableau_stat_registry for sheet %s(id:%s)",
-                sheet.get(tableau_constant.NAME),
-                sheet.get(tableau_constant.ID),
+                sheet.get(c.NAME),
+                sheet.get(c.ID),
             )
             return None
 
@@ -1983,8 +1932,8 @@ def _get_chart_stat_wu(
         )
         logger.debug(
             "stat: Chart usage stat work unit is created for %s(id:%s)",
-            sheet.get(tableau_constant.NAME),
-            sheet.get(tableau_constant.ID),
+            sheet.get(c.NAME),
+            sheet.get(c.ID),
         )
         return MetadataChangeProposalWrapper(
             aspect=aspect,
@@ -1992,22 +1941,20 @@ def _get_chart_stat_wu(
         ).as_workunit()
 
     def emit_sheets(self) -> Iterable[MetadataWorkUnit]:
-        sheets_filter = f"{tableau_constant.ID_WITH_IN}: {json.dumps(self.sheet_ids)}"
+        sheets_filter = f"{c.ID_WITH_IN}: {json.dumps(self.sheet_ids)}"
 
         for sheet in self.get_connection_objects(
             sheet_graphql_query,
-            tableau_constant.SHEETS_CONNECTION,
+            c.SHEETS_CONNECTION,
             sheets_filter,
         ):
-            yield from self.emit_sheets_as_charts(
-                sheet, sheet.get(tableau_constant.WORKBOOK)
-            )
+            yield from self.emit_sheets_as_charts(sheet, sheet.get(c.WORKBOOK))
 
     def emit_sheets_as_charts(
         self, sheet: dict, workbook: Optional[Dict]
     ) -> Iterable[MetadataWorkUnit]:
         sheet_urn: str = builder.make_chart_urn(
-            self.platform, sheet[tableau_constant.ID], self.config.platform_instance
+            self.platform, sheet[c.ID], self.config.platform_instance
         )
         chart_snapshot = ChartSnapshot(
             urn=sheet_urn,
@@ -2015,34 +1962,32 @@ def emit_sheets_as_charts(
         )
 
         creator: Optional[str] = None
-        if workbook is not None and workbook.get(tableau_constant.OWNER) is not None:
-            creator = workbook[tableau_constant.OWNER].get(tableau_constant.USERNAME)
-        created_at = sheet.get(tableau_constant.CREATED_AT, datetime.now())
-        updated_at = sheet.get(tableau_constant.UPDATED_AT, datetime.now())
+        if workbook is not None and workbook.get(c.OWNER) is not None:
+            creator = workbook[c.OWNER].get(c.USERNAME)
+        created_at = sheet.get(c.CREATED_AT, datetime.now())
+        updated_at = sheet.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
 
-        if sheet.get(tableau_constant.PATH):
+        if sheet.get(c.PATH):
             site_part = f"/site/{self.config.site}" if self.config.site else ""
-            sheet_external_url = f"{self.config.connect_uri}/#{site_part}/views/{sheet.get(tableau_constant.PATH)}"
-        elif (
-            sheet.get(tableau_constant.CONTAINED_IN_DASHBOARDS) is not None
-            and len(sheet[tableau_constant.CONTAINED_IN_DASHBOARDS]) > 0
-            and sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0] is not None
-            and sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0].get(
-                tableau_constant.PATH
+            sheet_external_url = (
+                f"{self.config.connect_uri}/#{site_part}/views/{sheet.get(c.PATH)}"
             )
+        elif (
+            sheet.get(c.CONTAINED_IN_DASHBOARDS) is not None
+            and len(sheet[c.CONTAINED_IN_DASHBOARDS]) > 0
+            and sheet[c.CONTAINED_IN_DASHBOARDS][0] is not None
+            and sheet[c.CONTAINED_IN_DASHBOARDS][0].get(c.PATH)
         ):
             # sheet contained in dashboard
             site_part = f"/t/{self.config.site}" if self.config.site else ""
-            dashboard_path = sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0][
-                tableau_constant.PATH
-            ]
-            sheet_external_url = f"{self.config.connect_uri}{site_part}/authoring/{dashboard_path}/{sheet.get(tableau_constant.NAME, '')}"
+            dashboard_path = sheet[c.CONTAINED_IN_DASHBOARDS][0][c.PATH]
+            sheet_external_url = f"{self.config.connect_uri}{site_part}/authoring/{dashboard_path}/{sheet.get(c.NAME, '')}"
         else:
             # hidden or viz-in-tooltip sheet
             sheet_external_url = None
         input_fields: List[InputField] = []
-        if sheet.get(tableau_constant.DATA_SOURCE_FIELDS):
+        if sheet.get(c.DATA_SOURCE_FIELDS):
             self.populate_sheet_upstream_fields(sheet, input_fields)
 
         # datasource urn
@@ -2060,15 +2005,13 @@ def emit_sheets_as_charts(
         # Chart Info
         chart_info = ChartInfoClass(
             description="",
-            title=sheet.get(tableau_constant.NAME) or "",
+            title=sheet.get(c.NAME) or "",
             lastModified=last_modified,
             externalUrl=sheet_external_url
             if self.config.ingest_external_links_for_charts
             else None,
             inputs=sorted(datasource_urn),
-            customProperties=self.get_custom_props_from_dict(
-                sheet, [tableau_constant.LUID]
-            ),
+            customProperties=self.get_custom_props_from_dict(sheet, [c.LUID]),
         )
         chart_snapshot.aspects.append(chart_info)
         # chart_snapshot doesn't support the stat aspect as list element and hence need to emit MCP
@@ -2083,7 +2026,7 @@ def emit_sheets_as_charts(
             chart_snapshot.aspects.append(browse_paths)
         else:
             logger.warning(
-                f"Could not set browse path for workbook {sheet[tableau_constant.ID]}. Please check permissions."
+                f"Could not set browse path for workbook {sheet[c.ID]}. Please check permissions."
             )
 
         # Ownership
@@ -2107,9 +2050,7 @@ def emit_sheets_as_charts(
             )
         if workbook is not None:
             yield from add_entity_to_container(
-                self.gen_workbook_key(workbook[tableau_constant.ID]),
-                tableau_constant.CHART,
-                chart_snapshot.urn,
+                self.gen_workbook_key(workbook[c.ID]), c.CHART, chart_snapshot.urn
             )
 
         if input_fields:
@@ -2134,14 +2075,12 @@ def _get_project_path(self, project: TableauProject) -> str:
     def populate_sheet_upstream_fields(
         self, sheet: dict, input_fields: List[InputField]
     ) -> None:
-        for field in sheet.get(tableau_constant.DATA_SOURCE_FIELDS):  # type: ignore
+        for field in sheet.get(c.DATA_SOURCE_FIELDS):  # type: ignore
             if not field:
                 continue
-            name = field.get(tableau_constant.NAME)
+            name = field.get(c.NAME)
             upstream_ds_id = (
-                field.get(tableau_constant.DATA_SOURCE)[tableau_constant.ID]
-                if field.get(tableau_constant.DATA_SOURCE)
-                else None
+                field.get(c.DATA_SOURCE)[c.ID] if field.get(c.DATA_SOURCE) else None
             )
             if name and upstream_ds_id:
                 input_fields.append(
@@ -2162,10 +2101,8 @@ def populate_sheet_upstream_fields(
                 )
 
     def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUnit]:
-        workbook_container_key = self.gen_workbook_key(workbook[tableau_constant.ID])
-        creator = workbook.get(tableau_constant.OWNER, {}).get(
-            tableau_constant.USERNAME
-        )
+        workbook_container_key = self.gen_workbook_key(workbook[c.ID])
+        creator = workbook.get(c.OWNER, {}).get(c.USERNAME)
 
         owner_urn = (
             builder.make_user_urn(creator)
@@ -2191,17 +2128,17 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni
         if project_luid and project_luid in self.tableau_project_registry.keys():
             parent_key = self.gen_project_key(project_luid)
         else:
-            workbook_id: Optional[str] = workbook.get(tableau_constant.ID)
-            workbook_name: Optional[str] = workbook.get(tableau_constant.NAME)
+            workbook_id: Optional[str] = workbook.get(c.ID)
+            workbook_name: Optional[str] = workbook.get(c.NAME)
             logger.warning(
                 f"Could not load project hierarchy for workbook {workbook_name}({workbook_id}). Please check permissions."
             )
 
         yield from gen_containers(
             container_key=workbook_container_key,
-            name=workbook.get(tableau_constant.NAME) or "",
+            name=workbook.get(c.NAME) or "",
             parent_container_key=parent_key,
-            description=workbook.get(tableau_constant.DESCRIPTION),
+            description=workbook.get(c.DESCRIPTION),
             sub_types=[BIContainerSubTypes.TABLEAU_WORKBOOK],
             owner_urn=owner_urn,
             external_url=workbook_external_url,
@@ -2237,20 +2174,20 @@ def _create_datahub_dashboard_usage_stat(
     def _get_dashboard_stat_wu(
         self, dashboard: dict, dashboard_urn: str
     ) -> Optional[MetadataWorkUnit]:
-        luid: Optional[str] = dashboard.get(tableau_constant.LUID)
+        luid: Optional[str] = dashboard.get(c.LUID)
         if luid is None:
             logger.debug(
                 "stat:luid is none for dashboard %s(id:%s)",
-                dashboard.get(tableau_constant.NAME),
-                dashboard.get(tableau_constant.ID),
+                dashboard.get(c.NAME),
+                dashboard.get(c.ID),
             )
             return None
         usage_stat: Optional[UsageStat] = self.tableau_stat_registry.get(luid)
         if usage_stat is None:
             logger.debug(
                 "stat:UsageStat is not available in tableau_stat_registry for dashboard %s(id:%s)",
-                dashboard.get(tableau_constant.NAME),
-                dashboard.get(tableau_constant.ID),
+                dashboard.get(c.NAME),
+                dashboard.get(c.ID),
             )
             return None
 
@@ -2259,8 +2196,8 @@ def _get_dashboard_stat_wu(
         )
         logger.debug(
             "stat: Dashboard usage stat is created for %s(id:%s)",
-            dashboard.get(tableau_constant.NAME),
-            dashboard.get(tableau_constant.ID),
+            dashboard.get(c.NAME),
+            dashboard.get(c.ID),
         )
 
         return MetadataChangeProposalWrapper(
@@ -2288,26 +2225,20 @@ def new_work_unit(self, mcp: MetadataChangeProposalWrapper) -> MetadataWorkUnit:
         )
 
     def emit_dashboards(self) -> Iterable[MetadataWorkUnit]:
-        dashboards_filter = (
-            f"{tableau_constant.ID_WITH_IN}: {json.dumps(self.dashboard_ids)}"
-        )
+        dashboards_filter = f"{c.ID_WITH_IN}: {json.dumps(self.dashboard_ids)}"
 
         for dashboard in self.get_connection_objects(
             dashboard_graphql_query,
-            tableau_constant.DASHBOARDS_CONNECTION,
+            c.DASHBOARDS_CONNECTION,
             dashboards_filter,
         ):
-            yield from self.emit_dashboard(
-                dashboard, dashboard.get(tableau_constant.WORKBOOK)
-            )
+            yield from self.emit_dashboard(dashboard, dashboard.get(c.WORKBOOK))
 
     def get_tags(self, obj: dict) -> Optional[List[str]]:
-        tag_list = obj.get(tableau_constant.TAGS, [])
+        tag_list = obj.get(c.TAGS, [])
         if tag_list and self.config.ingest_tags:
             tag_list_str = [
-                t[tableau_constant.NAME]
-                for t in tag_list
-                if t is not None and t.get(tableau_constant.NAME)
+                t[c.NAME] for t in tag_list if t is not None and t.get(c.NAME)
             ]
 
             return tag_list_str
@@ -2317,7 +2248,7 @@ def emit_dashboard(
         self, dashboard: dict, workbook: Optional[Dict]
     ) -> Iterable[MetadataWorkUnit]:
         dashboard_urn: str = builder.make_dashboard_urn(
-            self.platform, dashboard[tableau_constant.ID], self.config.platform_instance
+            self.platform, dashboard[c.ID], self.config.platform_instance
         )
         dashboard_snapshot = DashboardSnapshot(
             urn=dashboard_urn,
@@ -2325,26 +2256,28 @@ def emit_dashboard(
         )
 
         creator: Optional[str] = None
-        if workbook is not None and workbook.get(tableau_constant.OWNER) is not None:
-            creator = workbook[tableau_constant.OWNER].get(tableau_constant.USERNAME)
-        created_at = dashboard.get(tableau_constant.CREATED_AT, datetime.now())
-        updated_at = dashboard.get(tableau_constant.UPDATED_AT, datetime.now())
+        if workbook is not None and workbook.get(c.OWNER) is not None:
+            creator = workbook[c.OWNER].get(c.USERNAME)
+        created_at = dashboard.get(c.CREATED_AT, datetime.now())
+        updated_at = dashboard.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
 
         site_part = f"/site/{self.config.site}" if self.config.site else ""
-        dashboard_external_url = f"{self.config.connect_uri}/#{site_part}/views/{dashboard.get(tableau_constant.PATH, '')}"
+        dashboard_external_url = (
+            f"{self.config.connect_uri}/#{site_part}/views/{dashboard.get(c.PATH, '')}"
+        )
         title = (
-            dashboard[tableau_constant.NAME].replace("/", REPLACE_SLASH_CHAR)
-            if dashboard.get(tableau_constant.NAME)
+            dashboard[c.NAME].replace("/", REPLACE_SLASH_CHAR)
+            if dashboard.get(c.NAME)
             else ""
         )
         chart_urns = [
             builder.make_chart_urn(
                 self.platform,
-                sheet.get(tableau_constant.ID),
+                sheet.get(c.ID),
                 self.config.platform_instance,
             )
-            for sheet in dashboard.get(tableau_constant.SHEETS, [])
+            for sheet in dashboard.get(c.SHEETS, [])
         ]
         dashboard_info_class = DashboardInfoClass(
             description="",
@@ -2354,9 +2287,7 @@ def emit_dashboard(
             dashboardUrl=dashboard_external_url
             if self.config.ingest_external_links_for_dashboards
             else None,
-            customProperties=self.get_custom_props_from_dict(
-                dashboard, [tableau_constant.LUID]
-            ),
+            customProperties=self.get_custom_props_from_dict(dashboard, [c.LUID]),
         )
         dashboard_snapshot.aspects.append(dashboard_info_class)
 
@@ -2377,7 +2308,7 @@ def emit_dashboard(
             dashboard_snapshot.aspects.append(browse_paths)
         else:
             logger.warning(
-                f"Could not set browse path for dashboard {dashboard[tableau_constant.ID]}. Please check permissions."
+                f"Could not set browse path for dashboard {dashboard[c.ID]}. Please check permissions."
             )
 
         # Ownership
@@ -2397,8 +2328,8 @@ def emit_dashboard(
 
         if workbook is not None:
             yield from add_entity_to_container(
-                self.gen_workbook_key(workbook[tableau_constant.ID]),
-                tableau_constant.DASHBOARD,
+                self.gen_workbook_key(workbook[c.ID]),
+                c.DASHBOARD,
                 dashboard_snapshot.urn,
             )
 
@@ -2406,38 +2337,40 @@ def get_browse_paths_aspect(
         self, workbook: Optional[Dict]
     ) -> Optional[BrowsePathsClass]:
         browse_paths: Optional[BrowsePathsClass] = None
-        if workbook and workbook.get(tableau_constant.NAME):
+        if workbook and workbook.get(c.NAME):
             project_luid: Optional[str] = self._get_workbook_project_luid(workbook)
             if project_luid in self.tableau_project_registry:
                 browse_paths = BrowsePathsClass(
                     paths=[
                         f"/{self.platform}/{self._project_luid_to_browse_path_name(project_luid)}"
-                        f"/{workbook[tableau_constant.NAME].replace('/', REPLACE_SLASH_CHAR)}"
+                        f"/{workbook[c.NAME].replace('/', REPLACE_SLASH_CHAR)}"
                     ]
                 )
 
-            elif workbook.get(tableau_constant.PROJECT_NAME):
+            elif workbook.get(c.PROJECT_NAME):
                 # browse path
                 browse_paths = BrowsePathsClass(
                     paths=[
-                        f"/{self.platform}/{workbook[tableau_constant.PROJECT_NAME].replace('/', REPLACE_SLASH_CHAR)}"
-                        f"/{workbook[tableau_constant.NAME].replace('/', REPLACE_SLASH_CHAR)}"
+                        f"/{self.platform}/{workbook[c.PROJECT_NAME].replace('/', REPLACE_SLASH_CHAR)}"
+                        f"/{workbook[c.NAME].replace('/', REPLACE_SLASH_CHAR)}"
                     ]
                 )
 
         return browse_paths
 
     def emit_embedded_datasources(self) -> Iterable[MetadataWorkUnit]:
-        datasource_filter = f"{tableau_constant.ID_WITH_IN}: {json.dumps(self.embedded_datasource_ids_being_used)}"
+        datasource_filter = (
+            f"{c.ID_WITH_IN}: {json.dumps(self.embedded_datasource_ids_being_used)}"
+        )
 
         for datasource in self.get_connection_objects(
             embedded_datasource_graphql_query,
-            tableau_constant.EMBEDDED_DATA_SOURCES_CONNECTION,
+            c.EMBEDDED_DATA_SOURCES_CONNECTION,
             datasource_filter,
         ):
             yield from self.emit_datasource(
                 datasource,
-                datasource.get(tableau_constant.WORKBOOK),
+                datasource.get(c.WORKBOOK),
                 is_embedded_ds=True,
             )
 
@@ -2483,7 +2416,7 @@ def emit_project_containers(self) -> Iterable[MetadataWorkUnit]:
                 container_key=self.gen_project_key(_id),
                 name=project.name,
                 description=project.description,
-                sub_types=[tableau_constant.PROJECT],
+                sub_types=[c.PROJECT],
                 parent_container_key=self.gen_project_key(project.parent_id)
                 if project.parent_id
                 else None,
@@ -2498,7 +2431,7 @@ def emit_project_containers(self) -> Iterable[MetadataWorkUnit]:
                 yield from gen_containers(
                     container_key=self.gen_project_key(project.parent_id),
                     name=cast(str, project.parent_name),
-                    sub_types=[tableau_constant.PROJECT],
+                    sub_types=[c.PROJECT],
                 )
 
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
index 7c4852042ce7c..65d779b7f4516 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
@@ -8,7 +8,7 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import ConfigModel
-from datahub.ingestion.source import tableau_constant as tc
+from datahub.ingestion.source import tableau_constant as c
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageType,
     FineGrainedLineage,
@@ -591,12 +591,12 @@ def create(
         cls, d: dict, default_schema_map: Optional[Dict[str, str]] = None
     ) -> "TableauUpstreamReference":
         # Values directly from `table` object from Tableau
-        database = t_database = d.get(tc.DATABASE, {}).get(tc.NAME)
-        schema = t_schema = d.get(tc.SCHEMA)
-        table = t_table = d.get(tc.NAME) or ""
-        t_full_name = d.get(tc.FULL_NAME)
-        t_connection_type = d[tc.CONNECTION_TYPE]  # required to generate urn
-        t_id = d[tc.ID]
+        database = t_database = d.get(c.DATABASE, {}).get(c.NAME)
+        schema = t_schema = d.get(c.SCHEMA)
+        table = t_table = d.get(c.NAME) or ""
+        t_full_name = d.get(c.FULL_NAME)
+        t_connection_type = d[c.CONNECTION_TYPE]  # required to generate urn
+        t_id = d[c.ID]
 
         parsed_full_name = cls.parse_full_name(t_full_name)
         if parsed_full_name and len(parsed_full_name) == 3:

From 9174301719122c2597db75c8bb6b60c4d1a74f77 Mon Sep 17 00:00:00 2001
From: sachinsaju <33017477+sachinsaju@users.noreply.github.com>
Date: Thu, 9 Nov 2023 10:37:09 +0530
Subject: [PATCH 183/259] docs: update broken link in metadata-modelling
 (#9184)

Co-authored-by: Hyejin Yoon <0327jane@gmail.com>
Co-authored-by: John Joyce <john@acryl.io>
---
 docs/modeling/metadata-model.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/modeling/metadata-model.md b/docs/modeling/metadata-model.md
index a8958985a0a72..4c97cadc88417 100644
--- a/docs/modeling/metadata-model.md
+++ b/docs/modeling/metadata-model.md
@@ -625,7 +625,7 @@ curl --location --request POST 'http://localhost:8080/analytics?action=getTimese
     }
 }
 ```
-For more examples on the complex types of group-by/aggregations, refer to the tests in the group `getAggregatedStats` of [ElasticSearchTimeseriesAspectServiceTest.java](https://github.com/datahub-project/datahub/blob/master/metadata-io/src/test/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectServiceTest.java).
+For more examples on the complex types of group-by/aggregations, refer to the tests in the group `getAggregatedStats` of [TimeseriesAspectServiceTestBase.java](https://github.com/datahub-project/datahub/blob/master/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java).
 
 
 

From e494a9cc102f863bc51fcf80674bd6d3d36d726c Mon Sep 17 00:00:00 2001
From: Kos Korchak <97058061+kkorchak@users.noreply.github.com>
Date: Thu, 9 Nov 2023 00:23:17 -0500
Subject: [PATCH 184/259] test(): Test policy to create and manage privileges
 (#9173)

---
 .../tests/privileges/test_privileges.py       | 112 +++++++++++++++++-
 1 file changed, 111 insertions(+), 1 deletion(-)

diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py
index 740311754678e..d0f00734ae9f3 100644
--- a/smoke-test/tests/privileges/test_privileges.py
+++ b/smoke-test/tests/privileges/test_privileges.py
@@ -114,6 +114,21 @@ def _ensure_can_create_access_token(session, json):
     assert ingestion_data["data"]["createAccessToken"]["__typename"] == "AccessToken"
 
 
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(sleep_sec)
+)
+def _ensure_can_create_user_policy(session, json):
+    response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
+    response.raise_for_status()
+    res_data = response.json()
+
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["createPolicy"] is not None
+
+    return res_data["data"]["createPolicy"] 
+
+
 @pytest.mark.dependency(depends=["test_healthchecks"])
 def test_privilege_to_create_and_manage_secrets():
 
@@ -337,4 +352,99 @@ def test_privilege_to_create_and_manage_access_tokens():
 
 
     # Ensure that user can't create access token after policy is removed
-    _ensure_cant_perform_action(user_session, create_access_token,"createAccessToken")
\ No newline at end of file
+    _ensure_cant_perform_action(user_session, create_access_token,"createAccessToken")
+
+
+@pytest.mark.dependency(depends=["test_healthchecks"])
+def test_privilege_to_create_and_manage_policies():
+
+    (admin_user, admin_pass) = get_admin_credentials()
+    admin_session = login_as(admin_user, admin_pass)
+    user_session = login_as("user", "user")
+  
+
+    # Verify new user can't create a policy
+    create_policy = {
+        "query": """mutation createPolicy($input: PolicyUpdateInput!) {\n
+            createPolicy(input: $input) }""",
+        "variables": {
+            "input": {
+                "type": "PLATFORM",
+                "name": "Policy Name",
+                "description": "Policy Description",
+                "state": "ACTIVE",
+                "resources": {"filter":{"criteria":[]}},
+                "privileges": ["MANAGE_POLICIES"],
+                "actors": {
+                    "users": [],
+                    "resourceOwners": False,
+                    "allUsers": True,
+                    "allGroups": False,
+                },
+            }
+        },
+    }
+
+    _ensure_cant_perform_action(user_session, create_policy,"createPolicy")
+
+
+    # Assign privileges to the new user to create and manage policies
+    admin_policy_urn = create_user_policy("urn:li:corpuser:user", ["MANAGE_POLICIES"], admin_session)
+   
+
+    # Verify new user can create and manage policy(create, edit, delete)
+    # Create a policy
+    user_policy_urn = _ensure_can_create_user_policy(user_session, create_policy)
+
+    # Edit a policy
+    edit_policy = { 
+        "query": """mutation updatePolicy($urn: String!, $input: PolicyUpdateInput!) {\n
+            updatePolicy(urn: $urn, input: $input) }""",
+        "variables": {
+            "urn": user_policy_urn,
+            "input": {
+                "type": "PLATFORM",
+                "state": "INACTIVE",
+                "name": "Policy Name test",
+                "description": "Policy Description updated",
+                "privileges": ["MANAGE_POLICIES"],
+                "actors": {
+                    "users": [],
+                    "groups": None,
+                    "resourceOwners": False,
+                    "allUsers": True,
+                    "allGroups": False,
+                    "resourceOwnersTypes": None,
+                },
+            },
+        },
+    }
+    edit_policy_response = user_session.post(f"{get_frontend_url()}/api/v2/graphql", json=edit_policy)
+    edit_policy_response.raise_for_status()
+    res_data = edit_policy_response.json()
+
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["updatePolicy"] == user_policy_urn
+
+    # Delete a policy
+    remove_user_policy = { 
+        "query": "mutation deletePolicy($urn: String!) {\n  deletePolicy(urn: $urn)\n}\n",
+        "variables":{"urn":user_policy_urn}
+    }
+
+    remove_policy_response = user_session.post(f"{get_frontend_url()}/api/v2/graphql", json=remove_user_policy)
+    remove_policy_response.raise_for_status()
+    res_data = remove_policy_response.json()
+
+    assert res_data
+    assert res_data["data"]
+    assert res_data["data"]["deletePolicy"] == user_policy_urn
+
+
+    # Remove the user privilege by admin
+    remove_policy(admin_policy_urn, admin_session)
+
+
+    # Ensure that user can't create a policy after privilege is removed by admin
+    _ensure_cant_perform_action(user_session, create_policy,"createPolicy")
\ No newline at end of file

From 2187d24b54493953ab66b70f9a4b4fe0fd8841e1 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Thu, 9 Nov 2023 13:58:12 -0600
Subject: [PATCH 185/259] docs(security): add security doc to website (#9209)

---
 docs-website/generateDocsDir.ts | 1 -
 docs-website/sidebars.js        | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs-website/generateDocsDir.ts b/docs-website/generateDocsDir.ts
index a321146e10efa..e19f09530665a 100644
--- a/docs-website/generateDocsDir.ts
+++ b/docs-website/generateDocsDir.ts
@@ -125,7 +125,6 @@ function list_markdown_files(): string[] {
     /^docker\/(?!README|datahub-upgrade|airflow\/local_airflow)/, // Drop all but a few docker docs.
     /^docs\/docker\/README\.md/, // This one is just a pointer to another file.
     /^docs\/README\.md/, // This one is just a pointer to the hosted docs site.
-    /^SECURITY\.md$/,
     /^\s*$/, //Empty string
   ];
 
diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 9cc035f3e29e0..4d2420256ebff 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -546,6 +546,7 @@ module.exports = {
         "docs/CONTRIBUTING",
         "docs/links",
         "docs/rfc",
+        "SECURITY",
       ],
     },
     {

From 5911a7b45ed726292b2aa77c9e307d0e8683603a Mon Sep 17 00:00:00 2001
From: sachinsaju <33017477+sachinsaju@users.noreply.github.com>
Date: Fri, 10 Nov 2023 01:54:53 +0530
Subject: [PATCH 186/259] docs(java-sdk-dataset): add dataset via java sdk
 example (#9136)

Co-authored-by: Hyejin Yoon <0327jane@gmail.com>
---
 docs/api/tutorials/datasets.md                |  7 ++
 .../datahubproject/examples/DatasetAdd.java   | 84 +++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetAdd.java

diff --git a/docs/api/tutorials/datasets.md b/docs/api/tutorials/datasets.md
index 7c6d4a88d4190..39b0fdce1bdb5 100644
--- a/docs/api/tutorials/datasets.md
+++ b/docs/api/tutorials/datasets.md
@@ -28,6 +28,13 @@ For detailed steps, please refer to [Datahub Quickstart Guide](/docs/quickstart.
 > 🚫 Creating a dataset via `graphql` is currently not supported.
 > Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information.
 
+</TabItem>
+<TabItem value="java" label="Java">
+
+```java
+{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetAdd.java show_path_as_comment }}
+```
+
 </TabItem>
 <TabItem value="python" label="Python" default>
 
diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetAdd.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetAdd.java
new file mode 100644
index 0000000000000..ac368972e8dc9
--- /dev/null
+++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetAdd.java
@@ -0,0 +1,84 @@
+package io.datahubproject.examples;
+
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.CorpuserUrn;
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.common.urn.DatasetUrn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.schema.DateType;
+import com.linkedin.schema.OtherSchema;
+import com.linkedin.schema.SchemaField;
+import com.linkedin.schema.SchemaFieldArray;
+import com.linkedin.schema.SchemaFieldDataType;
+import com.linkedin.schema.SchemaMetadata;
+import com.linkedin.schema.StringType;
+import datahub.client.MetadataWriteResponse;
+import datahub.client.rest.RestEmitter;
+import datahub.event.MetadataChangeProposalWrapper;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+public class DatasetAdd {
+    
+    private DatasetAdd() {
+        
+    }
+
+    public static void main(String[] args) throws IOException, ExecutionException, InterruptedException {
+        DatasetUrn datasetUrn = UrnUtils.toDatasetUrn("hive", "fct_users_deleted", "PROD");
+        CorpuserUrn userUrn = new CorpuserUrn("ingestion");
+        AuditStamp lastModified = new AuditStamp().setTime(1640692800000L).setActor(userUrn);
+
+        SchemaMetadata schemaMetadata = new SchemaMetadata()
+                .setSchemaName("customer")
+                .setPlatform(new DataPlatformUrn("hive"))
+                .setVersion(0L)
+                .setHash("")
+                .setPlatformSchema(SchemaMetadata.PlatformSchema.create(new OtherSchema().setRawSchema("__insert raw schema here__")))
+                .setLastModified(lastModified);
+
+        SchemaFieldArray fields = new SchemaFieldArray();
+
+        SchemaField field1 = new SchemaField()
+                .setFieldPath("address.zipcode")
+                .setType(new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())))
+                .setNativeDataType("VARCHAR(50)")
+                .setDescription("This is the zipcode of the address. Specified using extended form and limited to addresses in the United States")
+                .setLastModified(lastModified);
+        fields.add(field1);
+
+        SchemaField field2 = new SchemaField().setFieldPath("address.street")
+                .setType(new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())))
+                .setNativeDataType("VARCHAR(100)")
+                .setDescription("Street corresponding to the address")
+                .setLastModified(lastModified);
+        fields.add(field2);
+
+        SchemaField field3 = new SchemaField().setFieldPath("last_sold_date")
+                .setType(new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new DateType())))
+                .setNativeDataType("Date")
+                .setDescription("Date of the last sale date for this property")
+                .setLastModified(lastModified);
+        fields.add(field3);
+
+        schemaMetadata.setFields(fields);
+
+        MetadataChangeProposalWrapper mcpw = MetadataChangeProposalWrapper.builder()
+                .entityType("dataset")
+                .entityUrn(datasetUrn)
+                .upsert()
+                .aspect(schemaMetadata)
+                .build();
+
+        String token = "";
+        RestEmitter emitter = RestEmitter.create(
+                b -> b.server("http://localhost:8080")
+                        .token(token)
+        );
+        Future<MetadataWriteResponse> response = emitter.emit(mcpw, null);
+        System.out.println(response.get().getResponseContent());
+    }
+
+}
\ No newline at end of file

From d6cb106fab4a4d49193afd0efd8ff7d90a8d3fa8 Mon Sep 17 00:00:00 2001
From: sachinsaju <33017477+sachinsaju@users.noreply.github.com>
Date: Fri, 10 Nov 2023 02:10:55 +0530
Subject: [PATCH 187/259] doc(java-sdk-example):example to create tag via
 java-sdk (#9151)

---
 docs/api/tutorials/tags.md                    |  8 ++++
 .../io/datahubproject/examples/TagCreate.java | 40 +++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 metadata-integration/java/examples/src/main/java/io/datahubproject/examples/TagCreate.java

diff --git a/docs/api/tutorials/tags.md b/docs/api/tutorials/tags.md
index b2234bf00bcb9..24d583dc26dac 100644
--- a/docs/api/tutorials/tags.md
+++ b/docs/api/tutorials/tags.md
@@ -78,6 +78,14 @@ Expected Response:
 
 </TabItem>
 
+<TabItem value="java" label="Java">
+
+```java
+{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/TagCreate.java show_path_as_comment }}
+```
+
+</TabItem>
+
 <TabItem value="python" label="Python">
 
 ```python
diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/TagCreate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/TagCreate.java
new file mode 100644
index 0000000000000..077489a9e02d9
--- /dev/null
+++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/TagCreate.java
@@ -0,0 +1,40 @@
+package io.datahubproject.examples;
+
+import com.linkedin.tag.TagProperties;
+import datahub.client.MetadataWriteResponse;
+import datahub.client.rest.RestEmitter;
+import datahub.event.MetadataChangeProposalWrapper;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+public class TagCreate {
+    
+    private TagCreate() {
+
+    }
+
+    public static void main(String[] args) throws IOException, ExecutionException, InterruptedException {
+        TagProperties tagProperties = new TagProperties()
+                .setName("Deprecated")
+                .setDescription("Having this tag means this column or table is deprecated.");
+
+        MetadataChangeProposalWrapper mcpw = MetadataChangeProposalWrapper.builder()
+                .entityType("tag")
+                .entityUrn("urn:li:tag:deprecated")
+                .upsert()
+                .aspect(tagProperties)
+                .build();
+
+        String token = "";
+        RestEmitter emitter = RestEmitter.create(
+                b -> b.server("http://localhost:8080")
+                        .token(token)
+        );
+        Future<MetadataWriteResponse> response = emitter.emit(mcpw, null);
+        System.out.println(response.get().getResponseContent());
+
+
+    }
+}

From 107713846f56e761011fd811fd8ac3b0b87a40bd Mon Sep 17 00:00:00 2001
From: Teppo Naakka <teppo.naakka@gmail.com>
Date: Fri, 10 Nov 2023 02:48:06 +0200
Subject: [PATCH 188/259] fix(ingest/powerbi): use dataset workspace id as key
 for parent container (#8994)

---
 .../ingestion/source/powerbi/powerbi.py       |   42 +-
 .../powerbi/golden_test_container.json        | 1089 +++++++++++++----
 ..._config_and_modified_since_admin_only.json |  210 +++-
 .../tests/integration/powerbi/test_powerbi.py |    4 +
 4 files changed, 1004 insertions(+), 341 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
index 4611a8eed4782..dc4394efcf245 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -4,7 +4,7 @@
 #
 #########################################################
 import logging
-from typing import Iterable, List, Optional, Set, Tuple, Union
+from typing import Iterable, List, Optional, Tuple, Union
 
 import datahub.emitter.mce_builder as builder
 import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes
@@ -110,8 +110,7 @@ def __init__(
         self.__config = config
         self.__reporter = reporter
         self.__dataplatform_instance_resolver = dataplatform_instance_resolver
-        self.processed_datasets: Set[powerbi_data_classes.PowerBIDataset] = set()
-        self.workspace_key: ContainerKey
+        self.workspace_key: Optional[ContainerKey] = None
 
     @staticmethod
     def urn_to_lowercase(value: str, flag: bool) -> str:
@@ -374,6 +373,9 @@ def to_datahub_dataset(
             f"Mapping dataset={dataset.name}(id={dataset.id}) to datahub dataset"
         )
 
+        if self.__config.extract_datasets_to_containers:
+            dataset_mcps.extend(self.generate_container_for_dataset(dataset))
+
         for table in dataset.tables:
             # Create a URN for dataset
             ds_urn = builder.make_dataset_urn_with_platform_instance(
@@ -461,7 +463,6 @@ def to_datahub_dataset(
 
             self.append_container_mcp(
                 dataset_mcps,
-                workspace,
                 ds_urn,
                 dataset,
             )
@@ -473,8 +474,6 @@ def to_datahub_dataset(
                 dataset.tags,
             )
 
-        self.processed_datasets.add(dataset)
-
         return dataset_mcps
 
     @staticmethod
@@ -572,7 +571,6 @@ def tile_custom_properties(tile: powerbi_data_classes.Tile) -> dict:
 
         self.append_container_mcp(
             result_mcps,
-            workspace,
             chart_urn,
         )
 
@@ -695,7 +693,6 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict:
 
         self.append_container_mcp(
             list_of_mcps,
-            workspace,
             dashboard_urn,
         )
 
@@ -711,7 +708,6 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict:
     def append_container_mcp(
         self,
         list_of_mcps: List[MetadataChangeProposalWrapper],
-        workspace: powerbi_data_classes.Workspace,
         entity_urn: str,
         dataset: Optional[powerbi_data_classes.PowerBIDataset] = None,
     ) -> None:
@@ -719,12 +715,8 @@ def append_container_mcp(
             dataset, powerbi_data_classes.PowerBIDataset
         ):
             container_key = dataset.get_dataset_key(self.__config.platform_name)
-        elif self.__config.extract_workspaces_to_containers:
-            container_key = workspace.get_workspace_key(
-                platform_name=self.__config.platform_name,
-                platform_instance=self.__config.platform_instance,
-                workspace_id_as_urn_part=self.__config.workspace_id_as_urn_part,
-            )
+        elif self.__config.extract_workspaces_to_containers and self.workspace_key:
+            container_key = self.workspace_key
         else:
             return None
 
@@ -743,6 +735,7 @@ def generate_container_for_workspace(
     ) -> Iterable[MetadataWorkUnit]:
         self.workspace_key = workspace.get_workspace_key(
             platform_name=self.__config.platform_name,
+            platform_instance=self.__config.platform_instance,
             workspace_id_as_urn_part=self.__config.workspace_id_as_urn_part,
         )
         container_work_units = gen_containers(
@@ -754,7 +747,7 @@ def generate_container_for_workspace(
 
     def generate_container_for_dataset(
         self, dataset: powerbi_data_classes.PowerBIDataset
-    ) -> Iterable[MetadataWorkUnit]:
+    ) -> Iterable[MetadataChangeProposalWrapper]:
         dataset_key = dataset.get_dataset_key(self.__config.platform_name)
         container_work_units = gen_containers(
             container_key=dataset_key,
@@ -762,7 +755,13 @@ def generate_container_for_dataset(
             parent_container_key=self.workspace_key,
             sub_types=[BIContainerSubTypes.POWERBI_DATASET],
         )
-        return container_work_units
+
+        # The if statement here is just to satisfy mypy
+        return [
+            wu.metadata
+            for wu in container_work_units
+            if isinstance(wu.metadata, MetadataChangeProposalWrapper)
+        ]
 
     def append_tag_mcp(
         self,
@@ -965,7 +964,6 @@ def to_chart_mcps(
 
             self.append_container_mcp(
                 list_of_mcps,
-                workspace,
                 chart_urn,
             )
 
@@ -1086,7 +1084,6 @@ def report_to_dashboard(
 
         self.append_container_mcp(
             list_of_mcps,
-            workspace,
             dashboard_urn,
         )
 
@@ -1220,10 +1217,6 @@ def validate_dataset_type_mapping(self):
             f"Dataset lineage would get ingested for data-platform = {self.source_config.dataset_type_mapping}"
         )
 
-    def extract_datasets_as_containers(self):
-        for dataset in self.mapper.processed_datasets:
-            yield from self.mapper.generate_container_for_dataset(dataset)
-
     def extract_independent_datasets(
         self, workspace: powerbi_data_classes.Workspace
     ) -> Iterable[MetadataWorkUnit]:
@@ -1270,9 +1263,6 @@ def get_workspace_workunit(
             ):
                 yield work_unit
 
-        if self.source_config.extract_datasets_to_containers:
-            yield from self.extract_datasets_as_containers()
-
         yield from self.extract_independent_datasets(workspace)
 
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
index 850816bf80807..91b5499eaadcb 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
@@ -15,7 +15,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -30,7 +31,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -45,7 +47,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -62,7 +65,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -77,7 +81,44 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "powerbi",
+                "dataset": "05169CD2-E713-41E6-9600-1D8066D95445"
+            },
+            "name": "library-dataset"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -94,7 +135,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -115,7 +157,79 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:powerbi"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Dataset"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
+                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -130,7 +244,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -148,7 +263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -158,12 +274,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -177,13 +294,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -200,7 +322,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -221,7 +344,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -236,7 +360,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -254,7 +379,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -264,12 +390,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -283,13 +410,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -306,7 +438,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -327,7 +460,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -342,7 +476,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -360,7 +495,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -370,12 +506,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -389,13 +526,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -412,7 +554,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -433,7 +576,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -448,7 +592,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -466,7 +611,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -476,12 +622,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -495,13 +642,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -518,7 +670,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -539,7 +692,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -554,7 +708,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -572,7 +727,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -582,12 +738,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -601,13 +758,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -624,7 +786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -645,7 +808,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -660,7 +824,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +843,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -688,12 +854,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -707,13 +874,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -730,7 +902,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -751,7 +924,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -766,7 +940,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -784,7 +959,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -794,12 +970,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -813,13 +990,54 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2",
+                    "urn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "powerbi",
+                "dataset": "ba0130a1-5b03-40de-9535-b34e778ea6ed"
+            },
+            "name": "hr_pbi_test"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -836,7 +1054,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -857,7 +1076,79 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:powerbi"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "PowerBI Dataset"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
+                    "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -872,7 +1163,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -890,7 +1182,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -900,12 +1193,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -919,13 +1213,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -942,7 +1241,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -963,7 +1263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -978,7 +1279,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -996,7 +1298,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1006,12 +1309,13 @@
     "aspectName": "container",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1025,13 +1329,18 @@
                 {
                     "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9",
                     "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+                },
+                {
+                    "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
+                    "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
                 }
             ]
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1046,7 +1355,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1061,7 +1371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1089,6 +1400,9 @@
                 }
             },
             "inputs": [
+                {
+                    "string": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
                 },
@@ -1115,7 +1429,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1130,7 +1445,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1146,7 +1462,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1163,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1178,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1198,7 +1517,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1226,6 +1546,9 @@
                 }
             },
             "inputs": [
+                {
+                    "string": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
                 },
@@ -1237,7 +1560,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1252,7 +1576,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1268,7 +1593,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1285,7 +1611,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1627,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1320,7 +1648,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1337,7 +1666,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1374,7 +1704,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1389,7 +1720,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1405,7 +1737,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1433,7 +1766,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1448,7 +1782,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1468,7 +1803,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1485,7 +1821,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1506,7 +1843,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1521,7 +1859,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1539,22 +1878,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1571,7 +1896,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1592,7 +1918,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1607,7 +1934,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1625,22 +1953,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1657,7 +1971,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1678,7 +1993,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1693,7 +2009,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1711,22 +2028,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1743,7 +2046,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1764,7 +2068,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1779,7 +2084,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1797,46 +2103,33 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "container",
+    "aspectName": "viewProperties",
     "aspect": {
         "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
+            "materialized": false,
+            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
+            "viewLanguage": "m_query"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
     "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "let\n    Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n    Source",
-            "viewLanguage": "m_query"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "datasetProperties",
+    "aspectName": "datasetProperties",
     "aspect": {
         "json": {
             "customProperties": {
@@ -1850,7 +2143,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1865,7 +2159,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1883,22 +2178,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1915,7 +2196,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1936,7 +2218,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1951,7 +2234,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1969,22 +2253,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2001,7 +2271,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2022,7 +2293,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2037,7 +2309,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2055,22 +2328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)",
-    "changeType": "UPSERT",
-    "aspectName": "container",
-    "aspect": {
-        "json": {
-            "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2085,7 +2344,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2100,7 +2360,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2126,6 +2387,9 @@
                 }
             },
             "inputs": [
+                {
+                    "string": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
                 },
@@ -2152,7 +2416,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2167,7 +2432,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2184,7 +2450,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2199,7 +2466,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2219,7 +2487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2245,6 +2514,9 @@
                 }
             },
             "inputs": [
+                {
+                    "string": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2"
+                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)"
                 },
@@ -2271,7 +2543,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2286,7 +2559,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2303,7 +2577,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2318,7 +2593,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2338,7 +2614,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2355,7 +2632,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2388,7 +2666,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2403,7 +2682,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2419,7 +2699,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2436,7 +2717,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2464,7 +2746,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2479,7 +2762,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2499,7 +2783,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2514,7 +2799,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2529,7 +2815,310 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:users.User4@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "corpUserKey",
+    "aspect": {
+        "json": {
+            "username": "User4@foo.com"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:users.User4@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:users.User3@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "corpUserKey",
+    "aspect": {
+        "json": {
+            "username": "User3@foo.com"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "corpuser",
+    "entityUrn": "urn:li:corpuser:users.User3@foo.com",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+    "changeType": "UPSERT",
+    "aspectName": "containerProperties",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "platform": "powerbi",
+                "workspace": "second-demo-workspace"
+            },
+            "name": "second-demo-workspace"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": []
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+    "changeType": "UPSERT",
+    "aspectName": "dataPlatformInstance",
+    "aspect": {
+        "json": {
+            "platform": "urn:li:dataPlatform:powerbi"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "container",
+    "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "Workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePaths",
+    "aspect": {
+        "json": {
+            "paths": [
+                "/powerbi/second-demo-workspace"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e",
+                    "urn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": {
+            "customProperties": {
+                "chartCount": "0",
+                "workspaceName": "second-demo-workspace",
+                "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108"
+            },
+            "title": "test_dashboard2",
+            "description": "",
+            "charts": [],
+            "datasets": [],
+            "lastModified": {
+                "created": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                },
+                "lastModified": {
+                    "time": 0,
+                    "actor": "urn:li:corpuser:unknown"
+                }
+            },
+            "dashboardUrl": "https://localhost/dashboards/web/1"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "dashboardKey",
+    "aspect": {
+        "json": {
+            "dashboardTool": "powerbi",
+            "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)",
+    "changeType": "UPSERT",
+    "aspectName": "ownership",
+    "aspect": {
+        "json": {
+            "owners": [
+                {
+                    "owner": "urn:li:corpuser:users.User3@foo.com",
+                    "type": "NONE"
+                },
+                {
+                    "owner": "urn:li:corpuser:users.User4@foo.com",
+                    "type": "NONE"
+                }
+            ],
+            "lastModified": {
+                "time": 0,
+                "actor": "urn:li:corpuser:unknown"
+            }
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
index a4527b9715704..b301ca1c1b988 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
@@ -15,7 +15,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -30,7 +31,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -45,7 +47,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -62,7 +65,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -77,7 +81,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -94,7 +99,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -126,7 +132,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -147,7 +154,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -162,7 +170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -180,7 +189,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -204,7 +214,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -219,7 +230,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -238,7 +250,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -249,6 +262,10 @@
     "aspect": {
         "json": {
             "path": [
+                {
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                },
                 {
                     "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
                     "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
@@ -258,7 +275,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -275,7 +293,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -307,7 +326,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -328,7 +348,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -343,7 +364,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -361,7 +383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -385,7 +408,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -400,7 +424,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -419,7 +444,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -430,6 +456,10 @@
     "aspect": {
         "json": {
             "path": [
+                {
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                },
                 {
                     "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
                     "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
@@ -439,7 +469,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -456,7 +487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -540,7 +572,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -561,7 +594,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -576,7 +610,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -594,7 +629,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -618,7 +654,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -633,7 +670,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -652,7 +690,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -663,6 +702,10 @@
     "aspect": {
         "json": {
             "path": [
+                {
+                    "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3",
+                    "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3"
+                },
                 {
                     "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc",
                     "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
@@ -672,7 +715,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -704,7 +748,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -719,7 +764,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -735,7 +781,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -752,7 +799,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -767,7 +815,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -787,7 +836,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -815,6 +865,9 @@
                 }
             },
             "inputs": [
+                {
+                    "string": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc"
+                },
                 {
                     "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)"
                 },
@@ -829,7 +882,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -844,7 +898,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -860,7 +915,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -877,7 +933,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -892,7 +949,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -912,7 +970,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -929,7 +988,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -966,7 +1026,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -981,7 +1042,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -997,7 +1059,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1012,7 +1075,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1032,7 +1096,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1049,7 +1114,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1079,7 +1145,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1094,7 +1161,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1110,7 +1178,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1127,7 +1196,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1151,7 +1221,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1166,7 +1237,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1186,7 +1258,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1205,7 +1278,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1220,7 +1294,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1235,7 +1310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1252,7 +1328,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1267,7 +1344,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1287,7 +1365,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1302,7 +1381,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "powerbi-test"
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
index 7232d2a38da1d..c9b0ded433749 100644
--- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
+++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py
@@ -1039,7 +1039,11 @@ def test_workspace_container(
                 "type": "powerbi",
                 "config": {
                     **default_source_config(),
+                    "workspace_id_pattern": {
+                        "deny": ["64ED5CAD-7322-4684-8180-826122881108"],
+                    },
                     "extract_workspaces_to_containers": True,
+                    "extract_datasets_to_containers": True,
                     "extract_reports": True,
                 },
             },

From bfa1769d4dd4f5281d751c6998c586e4e021897d Mon Sep 17 00:00:00 2001
From: John Joyce <john@acryl.io>
Date: Thu, 9 Nov 2023 17:56:33 -0800
Subject: [PATCH 189/259] refactor(schema tab): Remove last observed timestamps
 from schema tab (#9188)

---
 .../schema/SchemaTimeStamps.test.tsx          | 23 -------
 .../schema/components/SchemaHeader.tsx        |  6 --
 .../schema/components/SchemaTimeStamps.tsx    | 64 -------------------
 .../shared/tabs/Dataset/Schema/SchemaTab.tsx  |  5 --
 4 files changed, 98 deletions(-)
 delete mode 100644 datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx
 delete mode 100644 datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx

diff --git a/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx b/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx
deleted file mode 100644
index c8bb5d8100f2a..0000000000000
--- a/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx
+++ /dev/null
@@ -1,23 +0,0 @@
-import { render } from '@testing-library/react';
-import React from 'react';
-import { toRelativeTimeString } from '../../../../../shared/time/timeUtils';
-import SchemaTimeStamps from '../../schema/components/SchemaTimeStamps';
-
-describe('SchemaTimeStamps', () => {
-    it('should render last observed text if lastObserved is not null', () => {
-        const { getByText, queryByText } = render(<SchemaTimeStamps lastUpdated={123} lastObserved={123} />);
-        expect(getByText(`Last observed ${toRelativeTimeString(123)}`)).toBeInTheDocument();
-        expect(queryByText(`Reported ${toRelativeTimeString(123)}`)).toBeNull();
-    });
-
-    it('should render last updated text if lastObserved is null', () => {
-        const { getByText, queryByText } = render(<SchemaTimeStamps lastUpdated={123} lastObserved={null} />);
-        expect(queryByText(`Last observed ${toRelativeTimeString(123)}`)).toBeNull();
-        expect(getByText(`Reported ${toRelativeTimeString(123)}`)).toBeInTheDocument();
-    });
-
-    it('should return null if lastUpdated and lastObserved are both null', () => {
-        const { container } = render(<SchemaTimeStamps lastUpdated={null} lastObserved={null} />);
-        expect(container.firstChild).toBeNull();
-    });
-});
diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx
index 9e9e0ede2a1ce..2fc8fc11cd1b2 100644
--- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx
+++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx
@@ -17,7 +17,6 @@ import { SemanticVersionStruct } from '../../../../../../types.generated';
 import { toRelativeTimeString } from '../../../../../shared/time/timeUtils';
 import { ANTD_GRAY, REDESIGN_COLORS } from '../../../../shared/constants';
 import { navigateToVersionedDatasetUrl } from '../../../../shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl';
-import SchemaTimeStamps from './SchemaTimeStamps';
 import getSchemaFilterFromQueryString from '../../../../shared/tabs/Dataset/Schema/utils/getSchemaFilterFromQueryString';
 
 const SchemaHeaderContainer = styled.div`
@@ -137,8 +136,6 @@ type Props = {
     hasKeySchema: boolean;
     showKeySchema: boolean;
     setShowKeySchema: (show: boolean) => void;
-    lastUpdated?: number | null;
-    lastObserved?: number | null;
     selectedVersion: string;
     versionList: Array<SemanticVersionStruct>;
     showSchemaAuditView: boolean;
@@ -158,8 +155,6 @@ export default function SchemaHeader({
     hasKeySchema,
     showKeySchema,
     setShowKeySchema,
-    lastUpdated,
-    lastObserved,
     selectedVersion,
     versionList,
     showSchemaAuditView,
@@ -255,7 +250,6 @@ export default function SchemaHeader({
                     )}
                 </LeftButtonsGroup>
                 <RightButtonsGroup>
-                    <SchemaTimeStamps lastObserved={lastObserved} lastUpdated={lastUpdated} />
                     <Tooltip title={schemaAuditToggleText}>
                         <SchemaAuditButton
                             type="text"
diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx
deleted file mode 100644
index 42eae019b947a..0000000000000
--- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx
+++ /dev/null
@@ -1,64 +0,0 @@
-import { ClockCircleOutlined } from '@ant-design/icons';
-import { Popover, Typography } from 'antd';
-import React from 'react';
-import styled from 'styled-components/macro';
-import { toLocalDateTimeString, toRelativeTimeString } from '../../../../../shared/time/timeUtils';
-import { ANTD_GRAY } from '../../../../shared/constants';
-
-const CurrentVersionTimestampText = styled(Typography.Text)`
-    &&& {
-        line-height: 22px;
-        margin-top: 10px;
-        margin-right: 10px;
-        color: ${ANTD_GRAY[7]};
-        width: max-content;
-    }
-`;
-
-const TimeStampWrapper = styled.div`
-    margin-bottom: 5px;
-`;
-
-const StyledClockIcon = styled(ClockCircleOutlined)`
-    margin-right: 5px;
-`;
-
-interface Props {
-    lastUpdated?: number | null;
-    lastObserved?: number | null;
-}
-
-function SchemaTimeStamps(props: Props) {
-    const { lastUpdated, lastObserved } = props;
-
-    if (!lastUpdated && !lastObserved) return null;
-
-    return (
-        <Popover
-            content={
-                <>
-                    {lastObserved && (
-                        <TimeStampWrapper>Last observed on {toLocalDateTimeString(lastObserved)}.</TimeStampWrapper>
-                    )}
-                    {lastUpdated && <div>First reported on {toLocalDateTimeString(lastUpdated)}.</div>}
-                </>
-            }
-        >
-            <CurrentVersionTimestampText>
-                {lastObserved && (
-                    <span>
-                        <StyledClockIcon /> Last observed {toRelativeTimeString(lastObserved)}
-                    </span>
-                )}
-                {!lastObserved && lastUpdated && (
-                    <span>
-                        <StyledClockIcon />
-                        Reported {toRelativeTimeString(lastUpdated)}
-                    </span>
-                )}
-            </CurrentVersionTimestampText>
-        </Popover>
-    );
-}
-
-export default SchemaTimeStamps;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx
index 4bdb2dac033e7..75027e17b6d0c 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx
@@ -151,9 +151,6 @@ export const SchemaTab = ({ properties }: { properties?: any }) => {
         return groupByFieldPath(filteredRows, { showKeySchema });
     }, [showKeySchema, filteredRows]);
 
-    const lastUpdated = getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp;
-    const lastObserved = versionedDatasetData.data?.versionedDataset?.schema?.lastObserved;
-
     const schemaFieldBlameList: Array<SchemaFieldBlame> =
         (getSchemaBlameData?.getSchemaBlame?.schemaFieldBlameList as Array<SchemaFieldBlame>) || [];
 
@@ -167,8 +164,6 @@ export const SchemaTab = ({ properties }: { properties?: any }) => {
                 hasKeySchema={hasKeySchema}
                 showKeySchema={showKeySchema}
                 setShowKeySchema={setShowKeySchema}
-                lastObserved={lastObserved}
-                lastUpdated={lastUpdated}
                 selectedVersion={selectedVersion}
                 versionList={versionList}
                 showSchemaAuditView={showSchemaAuditView}

From 9c0f4de38241477524682943c815d5c03259e1a5 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Fri, 10 Nov 2023 16:06:06 +0900
Subject: [PATCH 190/259] docs: adjust sidebar & create new admin section
 (#9064)

---
 docs-website/sidebars.js | 213 ++++++++++++++++++++------------------
 docs/CODE_OF_CONDUCT.md  |   2 +-
 docs/saas.md             |  14 ---
 docs/townhall-history.md | 216 +++++++++++++++++++++++----------------
 docs/townhalls.md        |  11 +-
 5 files changed, 253 insertions(+), 203 deletions(-)
 delete mode 100644 docs/saas.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 4d2420256ebff..f15f2927379c5 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -9,17 +9,13 @@ module.exports = {
 
   overviewSidebar: [
     {
-      label: "Getting Started",
+      label: "What Is DataHub?",
       type: "category",
       collapsed: true,
+      link: { type: "doc", id: "docs/features" },
       items: [
         // By the end of this section, readers should understand the core use cases that DataHub addresses,
         // target end-users, high-level architecture, & hosting options
-        {
-          type: "doc",
-          label: "Introduction",
-          id: "docs/features",
-        },
         {
           type: "doc",
           label: "Quickstart",
@@ -31,7 +27,6 @@ module.exports = {
           href: "https://demo.datahubproject.io/",
         },
         "docs/what-is-datahub/datahub-concepts",
-        "docs/saas",
       ],
     },
     {
@@ -161,7 +156,15 @@ module.exports = {
         "docs/deploy/azure",
         "docker/README",
         "docs/deploy/kubernetes",
+        "docs/deploy/confluent-cloud",
         "docs/deploy/environment-vars",
+        "docs/how/extract-container-logs",
+      ],
+    },
+    {
+      type: "category",
+      label: "Admin",
+      items: [
         {
           Authentication: [
             "docs/authentication/README",
@@ -195,20 +198,91 @@ module.exports = {
             "docs/how/restore-indices",
             "docs/advanced/db-retention",
             "docs/advanced/monitoring",
-            "docs/how/extract-container-logs",
             "docs/deploy/telemetry",
             "docs/how/kafka-config",
-            "docs/deploy/confluent-cloud",
             "docs/advanced/no-code-upgrade",
             "docs/how/jattach-guide",
           ],
         },
-        "docs/how/updating-datahub",
       ],
     },
     {
-      API: [
-        "docs/api/datahub-apis",
+      Developers: [
+        // The purpose of this section is to provide developers & technical users with
+        // concrete tutorials for how to work with the DataHub CLI & APIs
+        {
+          Architecture: [
+            "docs/architecture/architecture",
+            "docs/components",
+            "docs/architecture/metadata-ingestion",
+            "docs/architecture/metadata-serving",
+            "docs/architecture/docker-containers",
+          ],
+        },
+        {
+          "Metadata Model": [
+            "docs/modeling/metadata-model",
+            "docs/modeling/extending-the-metadata-model",
+            "docs/what/mxe",
+            {
+              Entities: [
+                {
+                  type: "autogenerated",
+                  dirName: "docs/generated/metamodel/entities", // '.' means the current docs folder
+                },
+              ],
+            },
+          ],
+        },
+        {
+          "Developing on DataHub": [
+            "docs/developers",
+            "docs/docker/development",
+            "metadata-ingestion/developing",
+            "docs/api/graphql/graphql-endpoint-development",
+            {
+              Modules: [
+                "datahub-web-react/README",
+                "datahub-frontend/README",
+                "datahub-graphql-core/README",
+                "metadata-service/README",
+                "metadata-jobs/mae-consumer-job/README",
+                "metadata-jobs/mce-consumer-job/README",
+              ],
+            },
+          ],
+        },
+        "docs/plugins",
+        {
+          Troubleshooting: [
+            "docs/troubleshooting/quickstart",
+            "docs/troubleshooting/build",
+            "docs/troubleshooting/general",
+          ],
+        },
+        {
+          Advanced: [
+            "metadata-ingestion/docs/dev_guides/reporting_telemetry",
+            "docs/advanced/mcp-mcl",
+            "docker/datahub-upgrade/README",
+            "docs/advanced/no-code-modeling",
+            "datahub-web-react/src/app/analytics/README",
+            "docs/how/migrating-graph-service-implementation",
+            "docs/advanced/field-path-spec-v2",
+            "metadata-ingestion/adding-source",
+            "docs/how/add-custom-ingestion-source",
+            "docs/how/add-custom-data-platform",
+            "docs/advanced/browse-paths-upgrade",
+            "docs/browseV2/browse-paths-v2",
+          ],
+        },
+      ],
+    },
+    {
+      type: "category",
+      label: "API",
+      link: { type: "doc", id: "docs/api/datahub-apis" },
+      items: [
         {
           "GraphQL API": [
             {
@@ -466,92 +540,14 @@ module.exports = {
       ],
     },
     {
-      Develop: [
-        // The purpose of this section is to provide developers & technical users with
-        // concrete tutorials for how to work with the DataHub CLI & APIs
-        {
-          "DataHub Metadata Model": [
-            "docs/modeling/metadata-model",
-            "docs/modeling/extending-the-metadata-model",
-            "docs/what/mxe",
-            {
-              Entities: [
-                {
-                  type: "autogenerated",
-                  dirName: "docs/generated/metamodel/entities", // '.' means the current docs folder
-                },
-              ],
-            },
-          ],
-        },
-        {
-          Architecture: [
-            "docs/architecture/architecture",
-            "docs/components",
-            "docs/architecture/metadata-ingestion",
-            "docs/architecture/metadata-serving",
-            "docs/architecture/docker-containers",
-          ],
-        },
-        {
-          "Developing on DataHub": [
-            "docs/developers",
-            "docs/docker/development",
-            "metadata-ingestion/developing",
-            "docs/api/graphql/graphql-endpoint-development",
-            {
-              Modules: [
-                "datahub-web-react/README",
-                "datahub-frontend/README",
-                "datahub-graphql-core/README",
-                "metadata-service/README",
-                "metadata-jobs/mae-consumer-job/README",
-                "metadata-jobs/mce-consumer-job/README",
-              ],
-            },
-          ],
-        },
-        "docs/plugins",
-        {
-          Troubleshooting: [
-            "docs/troubleshooting/quickstart",
-            "docs/troubleshooting/build",
-            "docs/troubleshooting/general",
-          ],
-        },
-        {
-          Advanced: [
-            "metadata-ingestion/docs/dev_guides/reporting_telemetry",
-            "docs/advanced/mcp-mcl",
-            "docker/datahub-upgrade/README",
-            "docs/advanced/no-code-modeling",
-            "datahub-web-react/src/app/analytics/README",
-            "docs/how/migrating-graph-service-implementation",
-            "docs/advanced/field-path-spec-v2",
-            "metadata-ingestion/adding-source",
-            "docs/how/add-custom-ingestion-source",
-            "docs/how/add-custom-data-platform",
-            "docs/advanced/browse-paths-upgrade",
-            "docs/browseV2/browse-paths-v2",
-          ],
-        },
-      ],
-    },
-    {
-      Community: [
-        "docs/slack",
-        "docs/townhalls",
-        "docs/townhall-history",
-        "docs/CODE_OF_CONDUCT",
-        "docs/CONTRIBUTING",
-        "docs/links",
-        "docs/rfc",
-        "SECURITY",
-      ],
-    },
-    {
-      "Managed DataHub": [
-        "docs/managed-datahub/managed-datahub-overview",
+      label: "Managed DataHub",
+      type: "category",
+      collapsed: true,
+      link: {
+        type: "doc",
+        id: "docs/managed-datahub/managed-datahub-overview",
+      },
+      items: [
         "docs/managed-datahub/welcome-acryl",
         {
           type: "doc",
@@ -648,7 +644,26 @@ module.exports = {
       ],
     },
     {
-      "Release History": ["releases"],
+      label: "Community",
+      type: "category",
+      collapsed: true,
+      link: {
+        type: "generated-index",
+        title: "Community",
+        description: "Learn about DataHub community.",
+      },
+      items: [
+        "docs/slack",
+        "docs/townhalls",
+        //        "docs/townhall-history",
+        "docs/CODE_OF_CONDUCT",
+        "docs/CONTRIBUTING",
+        "docs/links",
+        "docs/rfc",
+      ],
+    },
+    {
+      "Release History": ["releases", "docs/how/updating-datahub"],
     },
 
     // "Candidates for Deprecation": [
diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md
index 1c4fd659f14e0..ca899dc26d5f7 100644
--- a/docs/CODE_OF_CONDUCT.md
+++ b/docs/CODE_OF_CONDUCT.md
@@ -1,4 +1,4 @@
-# Contributor Covenant Code of Conduct
+# Code of Conduct
 
 ## Our Pledge
 
diff --git a/docs/saas.md b/docs/saas.md
deleted file mode 100644
index de57b5617e062..0000000000000
--- a/docs/saas.md
+++ /dev/null
@@ -1,14 +0,0 @@
-# DataHub SaaS
-
-Sign up for fully managed, hassle-free and secure SaaS service for DataHub, provided by [Acryl Data](https://www.acryl.io/).
-
-<p>
-<a
-    className="button button--primary button--lg"
-    href="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup"
-    target="_blank" >
-    Sign up
-</a>
-</p>
-
-Refer to [Managed Datahub Exclusives](/docs/managed-datahub/managed-datahub-overview.md) for more information.
diff --git a/docs/townhall-history.md b/docs/townhall-history.md
index d92905af0cd72..0242e4ec2cee1 100644
--- a/docs/townhall-history.md
+++ b/docs/townhall-history.md
@@ -1,22 +1,55 @@
-# Town Hall History
+# Town Hall History 
 
-A list of previous Town Halls, their planned schedule, and the recording of the meeting.
+:::note 
+For the Town Hall meetings after June 2023, please refer to our [LinkedIn Live event history](https://www.linkedin.com/company/acryl-data/events/). 
+::: 
 
-## 03/23/2023
-[Full YouTube video](https://youtu.be/BTX8rIBe0yo)
+### June 2023
+[Full YouTube video](https://www.youtube.com/watch?v=1QVcUmRQK5E)
+
+- Community & Project Updates - Maggie Hays & Shirshanka Das (Acryl Data)
+- Community Case Study: Dataset Joins - Raj Tekal & Bobbie-Jean Nowak (Optum)
+- DataHub 201: Column-Level Lineage - Hyejin Yoon (Acryl Data)
+- Sneak Peek:  BigQuery Column-Level Lineage with SQL Parsing - Harshal Sheth (Acryl Data)
+- DataHub Performance Tuning – Indy Prentice (Acryl Data)
+
+
+### May 2023
+[Full YouTube video](https://www.youtube.com/watch?v=KHNPjSbbZR8)
+
+**Agenda**
+- Community - Maggie Hays & Shirshanka Das (Acryl Data)
+- Community Case Study: Jira + DataHub for Access Requests - Joshua Garza (Sharp Healthcare)
+- Sneak Peek: Use your own ownership types - Pedro Silva (Acryl Data)
+- Sneak Peek: Data Contracts are coming! – John Joyce, Shirshanka (Acryl Data)
+- Bring DataHub into your BI Tools — Chris Collins (Acryl Data)
+
+### Apr 2023
+[Full YouTube video](https://www.youtube.com/watch?v=D5YYGu-ZIBo)
+
+**Agenda**
+- Community & Roadmap Updates - Maggie Hays & Shirshanka Das (Acryl Data)
+- DataHub 201: Python SDK - Hyejin Yoon (Acryl Data)
+- Streamlined Search & Browse Experience - Chris Collins (Acryl Data)
+- Acryl's DataHub GitHub Actions - Harshal Sheth (Acryl Data)
+- Data Products in DataHub - Shirshanka Das & Chris Collins (Acryl Data)
+- DataHub Docs Bot - Maggie Hays (Acryl Data)
+
+###  Mar 2023 
 
-### Agenda
+[Full YouTube video](https://youtu.be/BTX8rIBe0yo)
 
+**Agenda**
 - Community & Roadmap Update
 - Recent Releases
 - Community Case Study — Jumio’s DataHub adoption journey
 - DataHub 201: Data Debugging
 - Sneak Peek: Streamlined Filtering Experience 
 
-## 02/23/2023
+### Feb 2023 
 [Full YouTube video](https://youtu.be/UItt4ppJSFc)
 
-### Agenda
+**Agenda**
 
 - Community & Roadmap Update
 - Recent Releases
@@ -27,20 +60,20 @@ A list of previous Town Halls, their planned schedule, and the recording of the
 - Simplifying Metadata Ingestion
 - DataHub 201: Rolling Out DataHub
 
-## 01/26/2023
+### Jan 2023 (26th)
 [Full YouTube video](https://youtu.be/A3mSiGHZ6Rc)
 
-### Agenda
+**Agenda**
 
 - What’s to Come - Q1 2023 Roadmap: Data Products, Data Contracts and more
 - Community Case Study - Notion: Automating annotations and metadata propagation
 - Community Contribution - Grab: Improvements to documentation editing
 - Simplifying DataHub - Removing Schema Registry requirement and introducing DataHub Lite
 
-## 01/05/2023
+### Jan 2023 (5th)
 [Full YouTube video](https://youtu.be/ECxIMbKwuOY)
 
-### Agenda
+**Agenda**
 
 - DataHub Community: 2022 in Review - Our Community of Data Practitioners is one of a kind. We’ll take the time to celebrate who we are, what we’ve built, and how we’ve collaborated in the past 12 months.
 - Search Improvements - Learn how we’re making the Search experience smarter and faster to connect you with the most relevant resources during data discovery.
@@ -49,13 +82,12 @@ A list of previous Town Halls, their planned schedule, and the recording of the
 - Sneak Peek: Time-based Lineage - Get a preview of how you’ll soon be able to trace lineage between datasets across different points in time to understand how interdependencies have evolved.
 - Sneak Peek: Chrome Extension - Soon, you’ll be able to quickly access rich metadata from DataHub while exploring resources in Looker via our upcoming Chrome Extension.
 
-## 12/01/2022
+### Dec 2023 
 [Full YouTube video](https://youtu.be/BlCLhG8lGoY)
 
-### Agenda
+**Agenda**
 
 November Town Hall (in December!)
-
 - Community Case Study - The Pinterest Team will share how they have integrated DataHub + Thrift and extended the Metadata Model with a Data Element entity to capture semantic types.
 - NEW! Ingestion Quickstart Guides - DataHub newbies, this one is for you! We’re rolling out ingestion quickstart guides to help you quickly get up and running with DataHub + Snowflake, BigQuery, and more!
 - NEW! In-App Product Tours - We’re making it easier than ever for end-users to get familiar with all that DataHub has to offer - hear all about the in-product onboarding resources we’re rolling out soon!
@@ -64,10 +96,10 @@ November Town Hall (in December!)
 - NEW! Slack + Microsoft Teams Integrations - Send automated alerts to Slack and/or Teams to keep track of critical events and changes within DataHub.
 - Hacktoberfest Winners Announced - We’ll recap this year’s Hacktoberfest and announce three winners of a $250 Amazon gift card & DataHub Swag.
 
-## 10/27/2022
+### Oct 2022
 [Full YouTube video](https://youtu.be/B74WHxX5EMk)
 
-### Agenda
+**Agenda**
 
 - Conquer Data Governance with Acryl Data’s Metadata Tests - Learn how to tackle Data Governance with incremental, automation-driven governance using Metadata Tests provided in Acryl Data’s managed DataHub offering
 - Community Case Study - The Grab Team shares how they are using DataHub for data discoverability, automated classification and governance workflows, data quality observability, and beyond!
@@ -75,20 +107,19 @@ November Town Hall (in December!)
 - Sneak Peek! Saved Views - Learn how you can soon use Saved Views to help end-users navigate entities in DataHub with more precision and focus
 - Performance Improvements - Hear about the latest upgrades to DataHub performance
 
-## 9/29/2022
+### Sep 2022 
 [Full YouTube video](https://youtu.be/FjkNySWkghY)
 
-### Agenda
-
+**Agenda**
 - Column Level Lineage is here! - Demo of column-level lineage and impact analysis in the DataHub UI
 - Community Case Study - The Stripe Team shares how they leverage DataHub to power observability within their Airflow-based ecosystem
 - Sneak Peek! Automated PII Classification - Preview upcoming functionality to automatically identify data fields that likely contain sensitive data
 - Ingestion Improvements Galore - Improved performance and functionality for dbt, Looker, Tableau, and Presto ingestion sources
 
-## 8/25/2022
+### Aug 2022 
 [Full YouTube video](https://youtu.be/EJCKxKBvCwo)
 
-### Agenda
+**Agenda**
 
 - Community Case Study - The Etsy Team shares their journey of adopting DataHub
 - Looker & DataHub Improvements - surface the most relevant Looks and Dashboards
@@ -97,10 +128,11 @@ November Town Hall (in December!)
 - Patch Support - Native support for PATCH in the metadata protocol to support efficient updates to add & remove owners, lineage, tags and more
 - Sneak Peek! Advanced Search
 
-## 7/28/2022
+### Jul 2022 
+
 [Full YouTube video](https://youtu.be/Zrkf3Mzcvc4)
 
-### Agenda
+**Agenda**
 
 - Community Updates
 - Project Updates
@@ -109,21 +141,20 @@ November Town Hall (in December!)
 - Streamlined Metadata Ingestion
 - DataHub 201: Metadata Enrichment
 
-## 6/30/2022
+### Jun 2022
 [Full YouTube video](https://youtu.be/fAD53fEJ6m0)
 
-### Agenda
-
+**Agenda**
 - Community Updates
 - Project Updates
 - dbt Integration Updates
 - CSV Ingestion Support
 - DataHub 201 - Glossary Term Deep Dive
 
-## 5/26/2022
+### May 2022
 [Full YouTube video](https://youtu.be/taKb_zyowEE)
 
-### Agenda
+**Agenda**
 
 - Community Case Study: Hear how the G-Research team is using Cassandra as DataHub’s Backend
 - Creating & Editing Glossary Terms from the DataHub UI
@@ -132,20 +163,22 @@ November Town Hall (in December!)
 - Sneak Peek: Data Reliability with DataHub
 - Metadata Day Hackathon Winners
 
-## 4/28/2022
+### Apr 2022 
 [Full YouTube video](https://www.youtube.com/watch?v=7iwNxHgqxtg)
 
-### Agenda
+**Agenda**
+
 - Community Case Study: Hear from Included Health about how they are embedding external tools into the DataHub UI
 - New! Actions Framework: run custom code when changes happen within DataHub
 - UI Refresh for ML Entities
 - Improved deletion support for time-series aspects, tags, terms, & more
 - OpenAPI Improvements
 
-## 3/31/2022
+### Mar 2022
 [Full YouTube video](https://www.youtube.com/watch?v=IVazVgcNRdw)
 
-### Agenda
+**Agenda**
+
 - Community Case Study: Hear from Zendesk about how they are applying “shift left” principles by authoring metadata in their Protobuf schemas
 - RBAC Functionality: View-Based Policies
 - Schema Version History - surfacing the history of schema changes in DataHub's UI
@@ -154,20 +187,22 @@ November Town Hall (in December!)
 - Delete API
 
 
-## 2/25/2022
+### Feb 2022 
 [Full YouTube video](https://www.youtube.com/watch?v=enBqB2Dbuv4)
 
-### Agenda
+**Agenda**
+
 
 - Lineage Impact Analysis - using DataHub to understand the impact of changes on downstream dependencies
 - Displaying Data Quality Checks in the UI
 - Roadmap update: Schema Version History & Column-Level Lineage
 - Community Case Study: Managing Lineage via YAML
 
-## 1/28/2022
+### Jan 2022 
 [Full YouTube video](https://youtu.be/ShlSR3dMUnE)
 
-### Agenda
+**Agenda**
+
 
 - Community & Roadmap Updates by Maggie Hays (Acryl Data)
 - Project Updates by Shirshanka Das  (Acryl Data)
@@ -176,10 +211,11 @@ November Town Hall (in December!)
 - DataHub Basics — Data Profiling & Usage Stats 101 by Maggie Hays & Tamás Németh (Acryl Data)
 - Demo: Spark Lineage by Mugdha Hardikar (GS Lab) & Shirshanka Das
 
-## 12/17/2021
+### Dec 2021
 [Full YouTube video](https://youtu.be/rYInKCwxu7o)
 
-### Agenda
+**Agenda**
+
 - Community & Roadmap Updates by Maggie Hays (Acryl Data)
 - Project Updates by Shirshanka Das (Acryl Data)
 - 2021 DataHub Community in Review by Maggie Hays
@@ -189,10 +225,11 @@ November Town Hall (in December!)
 - Top DataHub Contributors of 2021 - Maggie Hays
 - Final Surprise! We Interviewed a 10yo and a 70yo about DataHub
 
-## 11/19/2021
+### Nov 2021
 [Full YouTube video](https://youtu.be/to80sEDZz7k)
 
-### Agenda
+**Agenda**
+
 - Community & Roadmap Updates by Maggie Hays (Acryl Data)
 - Project Updates by Shirshanka Das (Acryl Data)
 - DataHub Basics -- Lineage 101 by John Joyce & Surya Lanka (Acryl Data)
@@ -200,10 +237,11 @@ November Town Hall (in December!)
 - DataHub API Authentication by John Joyce (Acryl Data)
 - Case Study: LinkedIn pilot to extend the OSS UI by Aikepaer Abuduweili & Joshua Shinavier
 
-## 10/29/2021
+### Oct 2021
 [Full YouTube video](https://youtu.be/GrS_uZhYNm0)
 
-### Agenda
+**Agenda**
+
 - DataHub Community & Roadmap Update - Maggie Hays (Acryl Data)
 - October Project Updates - Shirshanka Das (Acryl Data)
 - Introducing Recommendations - John Joyce & Dexter Lee (Acryl Data)
@@ -211,10 +249,11 @@ November Town Hall (in December!)
 - Data Profiling Improvements - Surya Lanka & Harshal Sheth (Acryl Data)
 - Lineage Improvements & BigQuery Dataset Lineage by Gabe Lyons & Varun Bharill (Acryl Data)
 
-## 9/24/2021
+### Sep 2021 
 [Full YouTube video](https://youtu.be/nQDiKPKnLLQ)
 
-### Agenda
+**Agenda**
+
 - Project Updates and Callouts by Shirshanka
    - GraphQL Public API Annoucement
 - Demo: Faceted Search by Gabe Lyons (Acryl Data)
@@ -224,10 +263,11 @@ November Town Hall (in December!)
 - Offline
    - Foreign Key and Related Term Mapping by Gabe Lyons (Acryl Data) [video](https://www.loom.com/share/79f27c2d9f6c4a3b8aacbc48c19add18)
 
-## 8/27/2021
+### Aug 2021 
 [Full YouTube video](https://youtu.be/3joZINi3ti4)
 
-### Agenda
+**Agenda**
+
 - Project Updates and Callouts by Shirshanka
    - Business Glossary Demo
    - 0.8.12 Upcoming Release Highlights 
@@ -239,12 +279,13 @@ November Town Hall (in December!)
    - Performance Monitoring by Dexter Lee (Acryl Data) [video](https://youtu.be/6Xfr_Y9abZo)
 
 
-## 7/23/2021
+### Jul 2021 
 [Full YouTube video](https://www.youtube.com/watch?v=rZsiB8z5rG4)
 
 [Medium Post](https://medium.com/datahub-project/datahub-project-updates-f4299cd3602e?source=friends_link&sk=27af7637f7ae44786ede694c3af512a5)
 
-### Agenda
+**Agenda**
+
 
 - Project Updates by Shirshanka
    - Release highlights
@@ -253,12 +294,13 @@ November Town Hall (in December!)
 - Demo: AWS SageMaker integration for Models and Features by Kevin Hu (Acryl Data)
 
 
-## 6/25/2021
+### Jun 2021 
 [Full YouTube video](https://www.youtube.com/watch?v=xUHOdDfdFpY)
 
 [Medium Post](https://medium.com/datahub-project/datahub-project-updates-ed3155476408?source=friends_link&sk=02816a16ff2acd688e6db8eb55808d31)
 
-#### Agenda
+**Agenda**
+
 
 - Project Updates by Shirshanka
     - Release notes
@@ -269,12 +311,13 @@ November Town Hall (in December!)
 - Developer Session: Simplified Deployment for DataHub by John Joyce, Gabe Lyons (Acryl Data)
 
 
-## 5/27/2021
+### May 2021 
 [Full YouTube video](https://www.youtube.com/watch?v=qgW_xpIr1Ho)
 
 [Medium Post](https://medium.com/datahub-project/linkedin-datahub-project-updates-ed98cdf913c1?source=friends_link&sk=9930ec5579299b155ea87c747683d1ad)
 
-#### Agenda
+**Agenda**
+
 
 - Project Updates by Shirshanka - 10 mins
    - 0.8.0 Release
@@ -284,12 +327,13 @@ November Town Hall (in December!)
 - Deep Dive: No Code Metadata Engine by John Joyce (Acryl Data) - 20 mins
 - General Q&A and closing remarks
 
-## 4/23/2021
+### Apr 2021 
 [Full YouTube video](https://www.youtube.com/watch?v=dlFa4ubJ9ho)
 
 [Medium Digest](https://medium.com/datahub-project/linkedin-datahub-project-updates-2b0d26066b8f?source=friends_link&sk=686c47219ed294e0838ae3e2fe29084d)
 
-#### Agenda
+**Agenda**
+
 
 - Welcome - 5 mins
 - Project Updates by Shirshanka - 10 mins
@@ -302,12 +346,13 @@ November Town Hall (in December!)
 - General Q&A and closing remarks - 5 mins
 
 
-## 3/19/2021
+### Mar 2021 
  [YouTube video](https://www.youtube.com/watch?v=xE8Uc27VTG4) 
- 
+
  [Medium Digest](https://medium.com/datahub-project/linkedin-datahub-project-updates-697f0faddd10?source=friends_link&sk=9888633c5c7219b875125e87a703ec4d)
 
-#### Agenda
+**Agenda**
+
 
 * Welcome - 5 mins
 * Project Updates ([slides](https://drive.google.com/file/d/1c3BTP3oDAzJr07l6pY6CkDZi5nT0cLRs/view?usp=sharing)) by [Shirshanka](https://www.linkedin.com/in/shirshankadas/) - 10 mins
@@ -320,11 +365,11 @@ November Town Hall (in December!)
 * Closing remarks - 5 mins
 
 
-## 2/19/2021
+### Feb 2021 
  [YouTube video](https://www.youtube.com/watch?v=Z9ImbcsAVl0) 
- 
+
  [Medium Digest](https://medium.com/datahub-project/linkedin-datahub-project-updates-february-2021-edition-338d2c6021f0)
-#### Agenda
+**Agenda**
 
 * Welcome - 5 mins
 * Latest React App Demo! ([video](https://www.youtube.com/watch?v=RQBEJhcen5E)) by John Joyce and Gabe Lyons - 5 mins
@@ -334,12 +379,12 @@ November Town Hall (in December!)
 * Closing remarks - 5 mins
 
 
-## 1/15/2021
+### Jan 2021 
 [Full Recording](https://youtu.be/r862MZTLAJ0)
 
 [Slide-deck](https://docs.google.com/presentation/d/e/2PACX-1vQ2B0iHb2uwege1wlkXHOgQer0myOMEE5EGnzRjyqw0xxS5SaAc8VMZ_1XVOHuTZCJYzZZW4i9YnzSN/pub?start=false&loop=false&delayms=3000)
 
-Agenda
+**Agenda**
 
 - Announcements - 2 mins
 - Community Updates ([video](https://youtu.be/r862MZTLAJ0?t=99)) - 10 mins
@@ -349,10 +394,10 @@ Agenda
 - General Q&A from sign up sheet, slack, and participants - 15 mins
 - Closing remarks - 5 minutes
 
-## 12/04/2020
+### Dec 2020 
 [Recording](https://linkedin.zoom.us/rec/share/8E7-lFnCi_kQ8OvXR9kW6fn-AjvV8VlqOO2xYR8b5Y_UeWI_ODcKFlxlHqYgBP7j.S-c8C1YMrz7d3Mjq)
 
-Agenda
+**Agenda**
 
 - Quick intro - 5 mins
 - [Why did Grofers choose DataHub for their data catalog?](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Datahub_at_Grofers.pdf) by [Shubham Gupta](https://www.linkedin.com/in/shubhamg931/) - 15 minutes
@@ -360,11 +405,11 @@ Agenda
 - General Q&A from sign up sheet, slack, and participants - 15 mins
 - Closing remarks - 5 minutes
 
-## 11/06/2020
+### Nov 2020
 
 [Recording](https://linkedin.zoom.us/rec/share/0yvjZ2fOzVmD8aaDo3lC59fXivmYG3EnF0U9tMVgKs827595usvSoIhtFUPjZCsU.b915nLRkw6iQlnoD)
 
-Agenda
+**Agenda**
 
 - Quick intro - 5 mins
 - [Lightning talk on Metadata use-cases at LinkedIn](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Metadata_Use-Cases_at_LinkedIn_-_Lightning_Talk.pdf) by [Shirshanka Das](https://www.linkedin.com/in/shirshankadas/) (LinkedIn) - 5 mins
@@ -374,11 +419,11 @@ Agenda
 - Closing remarks - 5 minutes
 
 
-## 09/25/2020
+### Sep 2020
 
 [Recording](https://linkedin.zoom.us/rec/share/uEQ2pRY0BHbVqk_sOTVRm05VXJ0xM_zKJ26yzfCBqNZItiBht__k_juCCahJ37QK.IKAU9qA_0qdURX4_)
 
-Agenda
+**Agenda**
 
 - Quick intro - 5 mins
 - [Data Discoverability at SpotHero](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Data_Discoverability_at_SpotHero.pdf) by [Maggie Hays](https://www.linkedin.com/in/maggie-hays/) (SpotHero) - 20 mins
@@ -386,23 +431,23 @@ Agenda
 - General Q&A from sign up sheet, slack, and participants - 15 mins
 - Closing remarks - 5 mins
 
-## 08/28/2020
+### Aug 2020 
 
 [Recording](https://linkedin.zoom.us/rec/share/vMBfcb31825IBZ3T71_wffM_GNv3T6a8hicf8_dcfzQlhfFxl5i_CPVKcmYaZA)
 
 
-Agenda
+**Agenda**
 - Quick intro - 5 mins
 - [Data Governance look for a Digital Bank](https://www.slideshare.net/SheetalPratik/linkedinsaxobankdataworkbench) by [Sheetal Pratik](https://www.linkedin.com/in/sheetalpratik/) (Saxo Bank) - 20 mins
 - Column level lineage for datasets demo by [Nagarjuna Kanamarlapudi](https://www.linkedin.com/in/nagarjunak/) (LinkedIn) - 15 mins
 - General Q&A from sign up sheet and participants - 15 mins
 - Closing remarks - 5 mins
 
-## 07/31/20
+### Jul 2020 
 
 [Recording](https://bluejeans.com/s/wjnDRJevi5z/)
 
-Agenda
+**Agenda**
   * Quick intro - 5 mins
   * Showcasing new entities onboarded to internal LinkedIn DataHub (Data Concepts, Schemas) by [Nagarjuna Kanamarlapudi](https://www.linkedin.com/in/nagarjunak) (LinkedIn) - 15 mins
   * Showcasing new Lineage UI in internal LinkedIn DataHub By [Ignacio Bona](https://www.linkedin.com/in/ignaciobona) (LinkedIn) - 10 mins
@@ -410,12 +455,12 @@ Agenda
   * Answering questions from the signup sheet - 13 mins
   * Questions from the participants - 10 mins
   * Closing remarks - 5 mins
-  
-## 06/26/20
+
+###  June 2020
 
 [Recording](https://bluejeans.com/s/yILyR/)
 
-Agenda
+**Agenda**
   * Quick intro - 5 mins
   * Onboarding Data Process entity by [Liangjun Jiang](https://github.com/liangjun-jiang) (Expedia) - 15 mins
   * How to onboard a new relationship to metadata graph by [Kerem Sahin](https://github.com/keremsahin1) (Linkedin) - 15 mins
@@ -423,11 +468,11 @@ Agenda
   * Questions from the participants - 10 mins
   * Closing remarks - 5 mins
 
-## 05/29/20
+### May 2020
 
 [Recording](https://bluejeans.com/s/GCAzY)
 
-Agenda
+**Agenda**
   * Quick intro - 5 mins
   * How to add a new aspect/feature for an existing entity in UI by [Charlie Tran](https://www.linkedin.com/in/charlie-tran/) (LinkedIn) - 10 mins
   * How to search over a new field by [Jyoti Wadhwani](https://www.linkedin.com/in/jyotiwadhwani/) (LinkedIn) - 10 mins
@@ -435,11 +480,11 @@ Agenda
   * Questions from the participants - 10 mins
   * Closing remarks - 5 mins
 
-## 04/17/20
+### Apr 2020 (17th)
 
 [Recording](https://bluejeans.com/s/eYRD4)
 
-Agenda
+**Agenda**
   * Quick intro - 5 mins
   * [DataHub Journey with Expedia Group](https://www.youtube.com/watch?v=ajcRdB22s5o&ab_channel=ArunVasudevan) by [Arun Vasudevan](https://www.linkedin.com/in/arun-vasudevan-55117368/) (Expedia) - 10 mins
   * Deploying DataHub using Nix by [Larry Luo](https://github.com/clojurians-org) (Shanghai HuaRui Bank) - 10 mins
@@ -447,13 +492,13 @@ Agenda
   * Questions from the participants - 10 mins
   * Closing remarks - 5 mins
 
-## 04/03/20
+### Apr 2020 (3rd)
 
 [Recording](https://bluejeans.com/s/vzYpa)
 
 [Q&A](https://docs.google.com/document/d/1ChF9jiJWv9wj3HLLkFYRg7NSYg8Kb0PT7COd7Hf9Zpk/edit?usp=sharing)
 
-- Agenda
+- **Agenda**
   * Quick intro - 5 mins
   * Creating Helm charts for deploying DataHub on Kubernetes by [Bharat Akkinepalli](https://www.linkedin.com/in/bharat-akkinepalli-ba0b7223/) (ThoughtWorks) - 10 mins
   * How to onboard a new metadata aspect by [Mars Lan](https://www.linkedin.com/in/marslan) (LinkedIn) - 10 mins
@@ -461,13 +506,13 @@ Agenda
   * Questions from the participants - 10 mins
   * Closing remarks - 5 mins
 
-## 03/20/20
+### Mar 2020 (20th)
 
 [Recording](https://bluejeans.com/s/FSKEF)
 
 [Q&A](https://docs.google.com/document/d/1vQ6tAGXsVafnPIcZv1GSYgnTJJXFOACa1aWzOQjiGHI/edit)
 
-Agenda
+**Agenda**
   * Quick intro - 5 mins
   * Internal DataHub demo - 10 mins
   * What's coming up next for DataHub (what roadmap items we are working on) - 10 mins
@@ -475,9 +520,8 @@ Agenda
   * Questions from the participants - 10 mins
   * Closing remarks - 5 mins
 
-## 03/06/20
+### Mar 2020 (6th)
 
 [Recording](https://bluejeans.com/s/vULMG)
 
-[Q&A](https://docs.google.com/document/d/1N_VGqlH9CD-54LBsVlpcK2Cf2Mgmuzq79EvN9qgBqtQ/edit)
-
+[Q&A](https://docs.google.com/document/d/1N_VGqlH9CD-54LBsVlpcK2Cf2Mgmuzq79EvN9qgBqtQ/edit)
\ No newline at end of file
diff --git a/docs/townhalls.md b/docs/townhalls.md
index f9c3bb16150cd..c80d198e5184c 100644
--- a/docs/townhalls.md
+++ b/docs/townhalls.md
@@ -7,8 +7,13 @@ From time to time we also use the opportunity to showcase upcoming features.
 
 ## Meeting Invite & Agenda
 
-You can join with this link https://zoom.datahubproject.io, or [RSVP](https://rsvp.datahubproject.io/) to get a calendar invite - this will always have the most up-to-date agenda for upcoming sessions.
+You can join with [this link](https://zoom.datahubproject.io) or [RSVP](https://rsvp.datahubproject.io/) to get a calendar invite - this will always have the most up-to-date agenda for upcoming sessions.
+
+## Town Hall History
+
+See our Town Hall history for the recordings and summaries of the past town halls.
+
+* [Town Hall Events (July 2023~)](https://www.linkedin.com/company/acryl-data/events/)
+* [Town Hall Events (~June 2023)](townhall-history.md)
 
-## Past Meetings
 
-See [Town Hall History](townhall-history.md) for recordings of past town halls.

From 179f103412d036212a1155d436a507def4f4928f Mon Sep 17 00:00:00 2001
From: Xuelei Li <115022112+lix-mms@users.noreply.github.com>
Date: Fri, 10 Nov 2023 17:58:38 +0100
Subject: [PATCH 191/259] fix(metadata-io): in Neo4j service use proper
 algorithm to get lineage (#8687)

Co-authored-by: RyanHolstien <RyanHolstien@users.noreply.github.com>
Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
---
 build.gradle                                  |   8 +-
 docker/neo4j/env/docker.env                   |   1 +
 .../docker-compose-m1.quickstart.yml          |   1 +
 .../quickstart/docker-compose.quickstart.yml  |   1 +
 docs/how/updating-datahub.md                  |   1 +
 metadata-io/build.gradle                      |   3 +
 .../graph/neo4j/Neo4jGraphService.java        | 231 +++++++++---------
 .../graph/neo4j/Neo4jGraphServiceTest.java    | 130 ++++++++++
 .../graph/neo4j/Neo4jTestServerBuilder.java   |   6 +-
 9 files changed, 267 insertions(+), 115 deletions(-)

diff --git a/build.gradle b/build.gradle
index 31e005e001cf0..54802917d05a5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -7,6 +7,8 @@ buildscript {
   ext.springBootVersion = '2.7.14'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '4.4.9'
+  ext.neo4jTestVersion = '4.4.25'
+  ext.neo4jApocVersion = '4.4.0.20:all'
   ext.testContainersVersion = '1.17.4'
   ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.2'
@@ -154,8 +156,10 @@ project.ext.externalDependency = [
     'mockServer': 'org.mock-server:mockserver-netty:5.11.2',
     'mockServerClient': 'org.mock-server:mockserver-client-java:5.11.2',
     'mysqlConnector': 'mysql:mysql-connector-java:8.0.20',
-    'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jVersion,
+    'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jTestVersion,
     'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jVersion,
+    'neo4jTestJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jTestVersion,
+    'neo4jApoc': 'org.neo4j.procedure:apoc:' + neo4jApocVersion,
     'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:' + openTelemetryVersion,
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
@@ -218,7 +222,7 @@ project.ext.externalDependency = [
     'common': 'commons-io:commons-io:2.7',
     'jline':'jline:jline:1.4.1',
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0'
-    
+
 ]
 
 allprojects {
diff --git a/docker/neo4j/env/docker.env b/docker/neo4j/env/docker.env
index 961a5ffcf5483..c8f2a4878900f 100644
--- a/docker/neo4j/env/docker.env
+++ b/docker/neo4j/env/docker.env
@@ -1,3 +1,4 @@
 NEO4J_AUTH=neo4j/datahub
 NEO4J_dbms_default__database=graph.db
 NEO4J_dbms_allow__upgrade=true
+NEO4JLABS_PLUGINS="[\"apoc\"]"
diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml
index 613718306abef..4df32395cf82d 100644
--- a/docker/quickstart/docker-compose-m1.quickstart.yml
+++ b/docker/quickstart/docker-compose-m1.quickstart.yml
@@ -253,6 +253,7 @@ services:
     - NEO4J_AUTH=neo4j/datahub
     - NEO4J_dbms_default__database=graph.db
     - NEO4J_dbms_allow__upgrade=true
+    - NEO4JLABS_PLUGINS=["apoc"]
     healthcheck:
       interval: 1s
       retries: 5
diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
index 30ccbae59be74..29c980532d46f 100644
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@@ -253,6 +253,7 @@ services:
     - NEO4J_AUTH=neo4j/datahub
     - NEO4J_dbms_default__database=graph.db
     - NEO4J_dbms_allow__upgrade=true
+    - NEO4JLABS_PLUGINS=["apoc"]
     healthcheck:
       interval: 1s
       retries: 5
diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 28f11e4b6d707..90b53161950e8 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -16,6 +16,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
+- #8687 (datahub-helm #365 #353) - If Helm is used for installation and Neo4j is enabled, update the prerequisites Helm chart to version >=0.1.2 and adjust your value overrides in the `neo4j:` section according to the new structure.
 - #9044 - GraphQL APIs for adding ownership now expect either an `ownershipTypeUrn` referencing a customer ownership type or a (deprecated) `type`. Where before adding an ownership without a concrete type was allowed, this is no longer the case. For simplicity you can use the `type` parameter which will get translated to a custom ownership type internally if one exists for the type being added.
 - #9010 - In Redshift source's config `incremental_lineage` is set default to off.
 - #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now.
diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle
index 740fed61f13d5..4b36f533476f7 100644
--- a/metadata-io/build.gradle
+++ b/metadata-io/build.gradle
@@ -57,6 +57,9 @@ dependencies {
   testImplementation externalDependency.h2
   testImplementation externalDependency.mysqlConnector
   testImplementation externalDependency.neo4jHarness
+  testImplementation (externalDependency.neo4jApoc) {
+    exclude group: 'org.yaml', module: 'snakeyaml'
+  }
   testImplementation externalDependency.mockito
   testImplementation externalDependency.mockitoInline
   testImplementation externalDependency.iStackCommons
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java
index 41d39cca4edda..ac57fb7db2b78 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java
@@ -5,6 +5,7 @@
 import com.datahub.util.exception.RetryLimitReached;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.linkedin.common.UrnArray;
 import com.linkedin.common.UrnArrayArray;
 import com.linkedin.common.urn.Urn;
@@ -25,17 +26,20 @@
 import com.linkedin.metadata.query.filter.RelationshipDirection;
 import com.linkedin.metadata.query.filter.RelationshipFilter;
 import com.linkedin.metadata.utils.metrics.MetricUtils;
+import com.linkedin.util.Pair;
 import io.opentelemetry.extension.annotations.WithSpan;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.StringJoiner;
-import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.AllArgsConstructor;
@@ -50,8 +54,7 @@
 import org.neo4j.driver.Session;
 import org.neo4j.driver.SessionConfig;
 import org.neo4j.driver.exceptions.Neo4jException;
-import org.neo4j.driver.internal.InternalRelationship;
-import org.neo4j.driver.types.Node;
+import org.neo4j.driver.types.Relationship;
 
 
 @Slf4j
@@ -62,9 +65,6 @@ public class Neo4jGraphService implements GraphService {
   private final Driver _driver;
   private SessionConfig _sessionConfig;
 
-  private static final String SOURCE = "source";
-  private static final String UI = "UI";
-
   public Neo4jGraphService(@Nonnull LineageRegistry lineageRegistry, @Nonnull Driver driver) {
     this(lineageRegistry, driver, SessionConfig.defaultConfig());
   }
@@ -234,53 +234,36 @@ public EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageDi
       @Nullable Long endTimeMillis) {
     log.debug(String.format("Neo4j getLineage maxHops = %d", maxHops));
 
-    final String statement =
-        generateLineageStatement(entityUrn, direction, graphFilters, maxHops, startTimeMillis, endTimeMillis);
+    final var statementAndParams =
+        generateLineageStatementAndParameters(entityUrn, direction, graphFilters, maxHops, startTimeMillis, endTimeMillis);
+
+    final var statement = statementAndParams.getFirst();
+    final var parameters = statementAndParams.getSecond();
 
     List<Record> neo4jResult =
-        statement != null ? runQuery(buildStatement(statement, new HashMap<>())).list() : new ArrayList<>();
-
-    // It is possible to have more than 1 path from node A to node B in the graph and previous query returns all the paths.
-    // We convert the List into Map with only the shortest paths. "item.get(i).size()" is the path size between two nodes in relation.
-    // The key for mapping is the destination node as the source node is always the same, and it is defined by parameter.
-    neo4jResult = neo4jResult.stream()
-        .collect(Collectors.toMap(item -> item.values().get(2).asNode().get("urn").asString(), Function.identity(),
-            (item1, item2) -> item1.get(1).size() < item2.get(1).size() ? item1 : item2))
-        .values()
-        .stream()
-        .collect(Collectors.toList());
+        statement != null ? runQuery(buildStatement(statement, parameters)).list() : new ArrayList<>();
 
     LineageRelationshipArray relations = new LineageRelationshipArray();
     neo4jResult.stream().skip(offset).limit(count).forEach(item -> {
       String urn = item.values().get(2).asNode().get("urn").asString();
-      String relationType = ((InternalRelationship) item.get(1).asList().get(0)).type().split("r_")[1];
-      int numHops = item.get(1).size();
       try {
-        // Generate path from r in neo4jResult
-        List<Urn> pathFromRelationships =
-            item.values().get(1).asList(Collections.singletonList(new ArrayList<Node>())).stream().map(t -> createFromString(
-                  // Get real upstream node/downstream node by direction
-                  ((InternalRelationship) t).get(direction == LineageDirection.UPSTREAM ? "startUrn" : "endUrn")
-                      .asString())).collect(Collectors.toList());
-        if (direction == LineageDirection.UPSTREAM) {
-          // For ui to show path correctly, reverse path for UPSTREAM direction
-          Collections.reverse(pathFromRelationships);
-          // Add missing original node to the end since we generate path from relationships
-          pathFromRelationships.add(Urn.createFromString(item.values().get(0).asNode().get("urn").asString()));
-        } else {
-          // Add missing original node to the beginning since we generate path from relationships
-          pathFromRelationships.add(0, Urn.createFromString(item.values().get(0).asNode().get("urn").asString()));
-        }
+        final var path = item.get(1).asPath();
+        final List<Urn> nodeListAsPath = StreamSupport.stream(
+                path.nodes().spliterator(), false)
+            .map(node -> createFromString(node.get("urn").asString()))
+            .collect(Collectors.toList());
+
+        final var firstRelationship = Optional.ofNullable(Iterables.getFirst(path.relationships(), null));
 
         relations.add(new LineageRelationship().setEntity(Urn.createFromString(urn))
-            .setType(relationType)
-            .setDegree(numHops)
-            .setPaths(new UrnArrayArray(new UrnArray(pathFromRelationships))));
+            // although firstRelationship should never be absent, provide "" as fallback value
+            .setType(firstRelationship.map(Relationship::type).orElse(""))
+            .setDegree(path.length())
+            .setPaths(new UrnArrayArray(new UrnArray(nodeListAsPath))));
       } catch (URISyntaxException ignored) {
         log.warn(String.format("Can't convert urn = %s, Error = %s", urn, ignored.getMessage()));
       }
     });
-
     EntityLineageResult result = new EntityLineageResult().setStart(offset)
             .setCount(relations.size())
             .setRelationships(relations)
@@ -290,31 +273,104 @@ public EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageDi
     return result;
   }
 
-  private String generateLineageStatement(@Nonnull Urn entityUrn, @Nonnull LineageDirection direction,
-      GraphFilters graphFilters, int maxHops, @Nullable Long startTimeMillis, @Nullable Long endTimeMillis) {
-    String statement;
-    final String allowedEntityTypes = String.join(" OR b:", graphFilters.getAllowedEntityTypes());
-
-    final String multiHopMatchTemplateIndirect = "MATCH p = shortestPath((a {urn: '%s'})<-[r*1..%d]-(b)) ";
-    final String multiHopMatchTemplateDirect = "MATCH p = shortestPath((a {urn: '%s'})-[r*1..%d]->(b)) ";
-    // directionFilterTemplate should apply to all condition.
-    final String multiHopMatchTemplate =
-        direction == LineageDirection.UPSTREAM ? multiHopMatchTemplateIndirect : multiHopMatchTemplateDirect;
-    final String fullQueryTemplate = generateFullQueryTemplate(multiHopMatchTemplate, startTimeMillis, endTimeMillis);
-
-    if (startTimeMillis != null && endTimeMillis != null) {
-      statement =
-          String.format(fullQueryTemplate, startTimeMillis, endTimeMillis, entityUrn, maxHops, allowedEntityTypes,
-              entityUrn);
-    } else if (startTimeMillis != null) {
-      statement = String.format(fullQueryTemplate, startTimeMillis, entityUrn, maxHops, allowedEntityTypes, entityUrn);
-    } else if (endTimeMillis != null) {
-      statement = String.format(fullQueryTemplate, endTimeMillis, entityUrn, maxHops, allowedEntityTypes, entityUrn);
+  private String getPathFindingLabelFilter(List<String> entityNames) {
+    return entityNames.stream().map(x -> String.format("+%s", x)).collect(Collectors.joining("|"));
+  }
+
+  private String getPathFindingRelationshipFilter(@Nonnull List<String> entityNames, @Nullable LineageDirection direction) {
+    // relationshipFilter supports mixing different directions for various relation types,
+    // so simply transform entries lineage registry into format of filter
+    final var filterComponents = new HashSet<String>();
+    for (final var entityName : entityNames) {
+      if (direction != null) {
+        for (final var edgeInfo : _lineageRegistry.getLineageRelationships(entityName, direction)) {
+          final var type = edgeInfo.getType();
+          if (edgeInfo.getDirection() == RelationshipDirection.INCOMING) {
+            filterComponents.add("<" + type);
+          } else {
+            filterComponents.add(type + ">");
+          }
+        }
+      } else {
+        // return disjunctive combination of edge types regardless of direction
+        for (final var direction1 : List.of(LineageDirection.UPSTREAM, LineageDirection.DOWNSTREAM)) {
+          for (final var edgeInfo : _lineageRegistry.getLineageRelationships(entityName, direction1)) {
+            filterComponents.add(edgeInfo.getType());
+          }
+        }
+      }
+    }
+    return String.join("|", filterComponents);
+  }
+
+  private Pair<String, Map<String, Object>> generateLineageStatementAndParameters(
+      @Nonnull Urn entityUrn, @Nonnull LineageDirection direction,
+      GraphFilters graphFilters, int maxHops,
+      @Nullable Long startTimeMillis, @Nullable Long endTimeMillis) {
+
+    final var parameterMap = new HashMap<String, Object>(Map.of(
+        "urn", entityUrn.toString(),
+        "labelFilter", getPathFindingLabelFilter(graphFilters.getAllowedEntityTypes()),
+        "relationshipFilter", getPathFindingRelationshipFilter(graphFilters.getAllowedEntityTypes(), direction),
+        "maxHops", maxHops
+    ));
+
+    if (startTimeMillis == null && endTimeMillis == null) {
+      // if no time filtering required, simply find all expansion paths to other nodes
+      final var statement = "MATCH (a {urn: $urn}) "
+          + "CALL apoc.path.spanningTree(a, { "
+          + "  relationshipFilter: $relationshipFilter, "
+          + "  labelFilter: $labelFilter, "
+          + "  minLevel: 1, "
+          + "  maxLevel: $maxHops "
+          + "}) "
+          + "YIELD path "
+          + "WITH a, path AS path "
+          + "RETURN a, path, last(nodes(path));";
+      return Pair.of(statement, parameterMap);
     } else {
-      statement = String.format(fullQueryTemplate, entityUrn, maxHops, allowedEntityTypes, entityUrn);
+      // when needing time filtering, possibility on multiple paths between two
+      // nodes must be considered, and we need to construct more complex query
+
+      // use r_ edges until they are no longer useful
+      final var relationFilter = getPathFindingRelationshipFilter(graphFilters.getAllowedEntityTypes(), null)
+          .replaceAll("(\\w+)", "r_$1");
+      final var relationshipPattern =
+          String.format(
+              (direction == LineageDirection.UPSTREAM ? "<-[:%s*1..%d]-" : "-[:%s*1..%d]->"),
+              relationFilter, maxHops);
+
+      // two steps:
+      // 1. find list of nodes reachable within maxHops
+      // 2. find the shortest paths from start node to every other node in these nodes
+      //    (note: according to the docs of shortestPath, WHERE conditions are applied during path exploration, not
+      //     after path exploration is done)
+      final var statement = "MATCH (a {urn: $urn}) "
+          + "CALL apoc.path.subgraphNodes(a, { "
+          + "  relationshipFilter: $relationshipFilter, "
+          + "  labelFilter: $labelFilter, "
+          + "  minLevel: 1, "
+          + "  maxLevel: $maxHops "
+          + "}) "
+          + "YIELD node AS b "
+          + "WITH a, b "
+          + "MATCH path = shortestPath((a)" + relationshipPattern + "(b)) "
+          + "WHERE a <> b "
+          + "  AND ALL(rt IN relationships(path) WHERE "
+          + "    (EXISTS(rt.source) AND rt.source = 'UI') OR "
+          + "    (NOT EXISTS(rt.createdOn) AND NOT EXISTS(rt.updatedOn)) OR "
+          + "    ($startTimeMillis <= rt.createdOn <= $endTimeMillis OR "
+          + "     $startTimeMillis <= rt.updatedOn <= $endTimeMillis) "
+          + "  ) "
+          + "RETURN a, path, b;";
+
+      // provide dummy start/end time when not provided, so no need to
+      // format clause differently if either of them is missing
+      parameterMap.put("startTimeMillis", startTimeMillis == null ? 0 : startTimeMillis);
+      parameterMap.put("endTimeMillis", endTimeMillis == null ? System.currentTimeMillis() : endTimeMillis);
+
+      return Pair.of(statement, parameterMap);
     }
-
-    return statement;
   }
 
   @Nonnull
@@ -583,15 +639,6 @@ private Result runQuery(@Nonnull Statement statement) {
     }
   }
 
-  @Nonnull
-  private static String toCriterionWhereString(@Nonnull String key, @Nonnull Object value) {
-    if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) {
-      return key + " = " + value;
-    }
-
-    return key + " = \"" + value.toString() + "\"";
-  }
-
   // Returns "key:value" String, if value is not primitive, then use toString() and double quote it
   @Nonnull
   private static String toCriterionString(@Nonnull String key, @Nonnull Object value) {
@@ -715,44 +762,4 @@ Urn createFromString(@Nonnull String rawUrn) {
       return null;
     }
   }
-
-  private String generateFullQueryTemplate(@Nonnull String multiHopMatchTemplate, @Nullable Long startTimeMillis, @Nullable Long endTimeMillis) {
-    final String sourceUiCheck = String.format("(EXISTS(rt.%s) AND rt.%s = '%s') ", SOURCE, SOURCE, UI);
-    final String whereTemplate = "WHERE (b:%s) AND b.urn <> '%s' ";
-    final String returnTemplate = "RETURN a,r,b";
-    String withTimeTemplate = "";
-    String timeFilterConditionTemplate = "AND ALL(rt IN relationships(p) WHERE left(type(rt), 2)='r_')";
-
-    if (startTimeMillis != null && endTimeMillis != null) {
-      withTimeTemplate = "WITH %d as startTimeMillis, %d as endTimeMillis ";
-      timeFilterConditionTemplate =
-          "AND ALL(rt IN relationships(p) WHERE " + sourceUiCheck + "OR "
-              + "(NOT EXISTS(rt.createdOn) AND NOT EXISTS(rt.updatedOn)) OR "
-              + "((rt.createdOn >= startTimeMillis AND rt.createdOn <= endTimeMillis) OR "
-              + "(rt.updatedOn >= startTimeMillis AND rt.updatedOn <= endTimeMillis))) "
-              + "AND ALL(rt IN relationships(p) WHERE left(type(rt), 2)='r_')";
-    } else if (startTimeMillis != null) {
-      withTimeTemplate = "WITH %d as startTimeMillis ";
-      timeFilterConditionTemplate =
-          "AND ALL(rt IN relationships(p) WHERE " + sourceUiCheck + "OR "
-              + "(NOT EXISTS(rt.createdOn) AND NOT EXISTS(rt.updatedOn)) OR "
-              + "(rt.createdOn >= startTimeMillis OR rt.updatedOn >= startTimeMillis)) "
-              + "AND ALL(rt IN relationships(p) WHERE left(type(rt), 2)='r_')";
-    } else if (endTimeMillis != null) {
-      withTimeTemplate = "WITH %d as endTimeMillis ";
-      timeFilterConditionTemplate =
-          "AND ALL(rt IN relationships(p) WHERE " + sourceUiCheck + "OR "
-              + "(NOT EXISTS(rt.createdOn) AND NOT EXISTS(rt.updatedOn)) OR "
-              + "(rt.createdOn <= endTimeMillis OR rt.updatedOn <= endTimeMillis)) "
-              + "AND ALL(rt IN relationships(p) WHERE left(type(rt), 2)='r_')";
-    }
-    final StringJoiner fullQueryTemplateJoiner = new StringJoiner(" ");
-    fullQueryTemplateJoiner.add(withTimeTemplate);
-    fullQueryTemplateJoiner.add(multiHopMatchTemplate);
-    fullQueryTemplateJoiner.add(whereTemplate);
-    fullQueryTemplateJoiner.add(timeFilterConditionTemplate);
-    fullQueryTemplateJoiner.add(returnTemplate);
-
-    return fullQueryTemplateJoiner.toString();
-  }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java
index d36f05cfb039d..6f63209f9c380 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java
@@ -1,6 +1,7 @@
 package com.linkedin.metadata.graph.neo4j;
 
 import com.linkedin.common.FabricType;
+import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.DataPlatformUrn;
 import com.linkedin.common.urn.DatasetUrn;
 import com.linkedin.common.urn.TagUrn;
@@ -17,6 +18,7 @@
 import com.linkedin.metadata.query.filter.RelationshipFilter;
 import java.util.Arrays;
 import java.util.Collections;
+
 import org.neo4j.driver.Driver;
 import org.neo4j.driver.GraphDatabase;
 import org.testng.SkipException;
@@ -29,6 +31,8 @@
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static com.linkedin.metadata.search.utils.QueryUtils.*;
 import static org.testng.Assert.assertEquals;
@@ -194,11 +198,82 @@ public void testRemoveEdge() throws Exception {
     assertEquals(result.getTotal(), 0);
   }
 
+  private Set<UrnArray> getPathUrnArraysFromLineageResult(EntityLineageResult result) {
+    return result.getRelationships()
+        .stream()
+        .map(x -> x.getPaths().get(0))
+        .collect(Collectors.toSet());
+  }
+
+  @Test
+  public void testGetLineage() {
+    GraphService service = getGraphService();
+
+    List<Edge> edges = Arrays.asList(
+        // d1 <-Consumes- dj1 -Produces-> d2 <-DownstreamOf- d3 <-DownstreamOf- d5
+        new Edge(dataJobOneUrn, datasetOneUrn, consumes, 1L, null, 3L, null, null),
+        new Edge(dataJobOneUrn, datasetTwoUrn, produces, 5L, null, 7L, null, null),
+        new Edge(datasetThreeUrn, datasetTwoUrn, downstreamOf, 9L, null, null, null, null),
+        new Edge(datasetFiveUrn, datasetThreeUrn, downstreamOf, 11L, null, null, null, null),
+
+        // another path between d2 and d5 which is shorter
+        // d1 <-DownstreamOf- d4 <-DownstreamOf- d5
+        new Edge(datasetFourUrn, datasetOneUrn, downstreamOf, 13L, null, 13L, null, null),
+        new Edge(datasetFiveUrn, datasetFourUrn, downstreamOf, 13L, null, 13L, null, null)
+    );
+    edges.forEach(service::addEdge);
+
+    // simple path finding
+    final var upstreamLineageDataset3Hop3 = service.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 1000, 3);
+    assertEquals(upstreamLineageDataset3Hop3.getTotal().intValue(), 3);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageDataset3Hop3),
+        Set.of(
+            new UrnArray(datasetThreeUrn, datasetTwoUrn),
+            new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn),
+            new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn, datasetOneUrn)));
+
+    // simple path finding
+    final var upstreamLineageDatasetFiveHop2 = service.getLineage(datasetFiveUrn, LineageDirection.UPSTREAM, 0, 1000, 2);
+    assertEquals(upstreamLineageDatasetFiveHop2.getTotal().intValue(), 4);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageDatasetFiveHop2),
+        Set.of(
+            new UrnArray(datasetFiveUrn, datasetThreeUrn),
+            new UrnArray(datasetFiveUrn, datasetThreeUrn, datasetTwoUrn),
+            new UrnArray(datasetFiveUrn, datasetFourUrn),
+            new UrnArray(datasetFiveUrn, datasetFourUrn, datasetOneUrn)));
+
+    // there are two paths from p5 to p1, one longer and one shorter, and the longer one is discarded from result
+    final var upstreamLineageDataset5Hop5 = service.getLineage(datasetFiveUrn, LineageDirection.UPSTREAM, 0, 1000, 5);
+    assertEquals(upstreamLineageDataset5Hop5.getTotal().intValue(), 5);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageDataset5Hop5),
+        Set.of(
+            new UrnArray(datasetFiveUrn, datasetThreeUrn),
+            new UrnArray(datasetFiveUrn, datasetThreeUrn, datasetTwoUrn),
+            new UrnArray(datasetFiveUrn, datasetThreeUrn, datasetTwoUrn, dataJobOneUrn),
+            new UrnArray(datasetFiveUrn, datasetFourUrn),
+            new UrnArray(datasetFiveUrn, datasetFourUrn, datasetOneUrn)));
+
+    // downstream lookup
+    final var downstreamLineageDataset1Hop2 = service.getLineage(datasetOneUrn, LineageDirection.DOWNSTREAM, 0, 1000, 2);
+    assertEquals(downstreamLineageDataset1Hop2.getTotal().intValue(), 4);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(downstreamLineageDataset1Hop2),
+        Set.of(
+            new UrnArray(datasetOneUrn, dataJobOneUrn),
+            new UrnArray(datasetOneUrn, dataJobOneUrn, datasetTwoUrn),
+            new UrnArray(datasetOneUrn, datasetFourUrn),
+            new UrnArray(datasetOneUrn, datasetFourUrn, datasetFiveUrn)));
+  }
+
   @Test
   public void testGetLineageTimeFilterQuery() throws Exception {
     GraphService service = getGraphService();
 
     List<Edge> edges = Arrays.asList(
+        // d1 <-Consumes- dj1 -Produces-> d2 <-DownstreamOf- d3 <-DownstreamOf- d4
         new Edge(dataJobOneUrn, datasetOneUrn, consumes, 1L, null, 3L, null, null),
         new Edge(dataJobOneUrn, datasetTwoUrn, produces, 5L, null, 7L, null, null),
         new Edge(datasetThreeUrn, datasetTwoUrn, downstreamOf, 9L, null, null, null, null),
@@ -206,21 +281,76 @@ public void testGetLineageTimeFilterQuery() throws Exception {
     );
     edges.forEach(service::addEdge);
 
+    // no time filtering
     EntityLineageResult upstreamLineageTwoHops = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 1000, 2);
     assertEquals(upstreamLineageTwoHops.getTotal().intValue(), 2);
     assertEquals(upstreamLineageTwoHops.getRelationships().size(), 2);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageTwoHops),
+        Set.of(
+            new UrnArray(datasetFourUrn, datasetThreeUrn),
+            new UrnArray(datasetFourUrn, datasetThreeUrn, datasetTwoUrn)));
 
+    // with time filtering
     EntityLineageResult upstreamLineageTwoHopsWithTimeFilter = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 1000, 2, 10L, 12L);
     assertEquals(upstreamLineageTwoHopsWithTimeFilter.getTotal().intValue(), 1);
     assertEquals(upstreamLineageTwoHopsWithTimeFilter.getRelationships().size(), 1);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageTwoHopsWithTimeFilter),
+        Set.of(
+            new UrnArray(datasetFourUrn, datasetThreeUrn)));
 
+    // with time filtering
     EntityLineageResult upstreamLineageTimeFilter = service.getLineage(datasetTwoUrn, LineageDirection.UPSTREAM, 0, 1000, 4, 2L, 6L);
     assertEquals(upstreamLineageTimeFilter.getTotal().intValue(), 2);
     assertEquals(upstreamLineageTimeFilter.getRelationships().size(), 2);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageTimeFilter),
+        Set.of(
+            new UrnArray(datasetTwoUrn, dataJobOneUrn),
+            new UrnArray(datasetTwoUrn, dataJobOneUrn, datasetOneUrn)));
 
+    // with time filtering
     EntityLineageResult downstreamLineageTimeFilter = service.getLineage(datasetOneUrn, LineageDirection.DOWNSTREAM, 0, 1000, 4, 0L, 4L);
     assertEquals(downstreamLineageTimeFilter.getTotal().intValue(), 1);
     assertEquals(downstreamLineageTimeFilter.getRelationships().size(), 1);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(downstreamLineageTimeFilter),
+        Set.of(
+            new UrnArray(datasetOneUrn, dataJobOneUrn)));
+  }
+
+  @Test
+  public void testGetLineageTimeFilteringSkipsShorterButNonMatchingPaths() {
+    GraphService service = getGraphService();
 
+    List<Edge> edges = Arrays.asList(
+        // d1 <-Consumes- dj1 -Produces-> d2 <-DownstreamOf- d3
+        new Edge(dataJobOneUrn, datasetOneUrn, consumes, 5L, null, 5L, null, null),
+        new Edge(dataJobOneUrn, datasetTwoUrn, produces, 7L, null, 7L, null, null),
+        new Edge(datasetThreeUrn, datasetTwoUrn, downstreamOf, 9L, null, null, null, null),
+
+        // d1 <-DownstreamOf- d3 (shorter path from d3 to d1, but with very old time)
+        new Edge(datasetThreeUrn, datasetOneUrn, downstreamOf, 1L, null, 2L, null, null)
+    );
+    edges.forEach(service::addEdge);
+
+    // no time filtering, shorter path from d3 to d1 is returned
+    EntityLineageResult upstreamLineageNoTimeFiltering = service.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 1000, 3);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageNoTimeFiltering),
+        Set.of(
+            new UrnArray(datasetThreeUrn, datasetTwoUrn),
+            new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn),
+            new UrnArray(datasetThreeUrn, datasetOneUrn)));
+
+    // with time filtering, shorter path from d3 to d1 is excluded so longer path is returned
+    EntityLineageResult upstreamLineageTimeFiltering = service.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 1000, 3, 3L, 17L);
+    assertEquals(
+        getPathUrnArraysFromLineageResult(upstreamLineageTimeFiltering),
+        Set.of(
+            new UrnArray(datasetThreeUrn, datasetTwoUrn),
+            new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn),
+            new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn, datasetOneUrn)));
   }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jTestServerBuilder.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jTestServerBuilder.java
index 4d6d15255b922..ba4e4cec37914 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jTestServerBuilder.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jTestServerBuilder.java
@@ -2,6 +2,8 @@
 
 import java.io.File;
 import java.net.URI;
+
+import apoc.path.PathExplorer;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.harness.Neo4j;
 import org.neo4j.harness.Neo4jBuilder;
@@ -17,7 +19,9 @@ private Neo4jTestServerBuilder(Neo4jBuilder builder) {
   }
 
   public Neo4jTestServerBuilder() {
-    this(new InProcessNeo4jBuilder());
+    this(new InProcessNeo4jBuilder()
+        .withProcedure(PathExplorer.class)
+    );
   }
 
   public Neo4jTestServerBuilder(File workingDirectory) {

From b851d59e208d6f1f9c33f90d43d49933f6e557be Mon Sep 17 00:00:00 2001
From: purnimagarg1 <139125209+purnimagarg1@users.noreply.github.com>
Date: Fri, 10 Nov 2023 22:52:04 +0530
Subject: [PATCH 192/259] Managed Ingestion UX Improvements (#9216)

---
 .../source/IngestionSourceTableColumns.tsx    | 12 ++++--
 .../ExecutionRequestDetailsModal.tsx          | 38 ++++++++++++++-----
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
index c47d08d5b6003..155e75f1895f5 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx
@@ -61,6 +61,14 @@ const CliBadge = styled.span`
         margin-right: 5px;
     }
 `;
+const StatusText = styled(Typography.Text)`
+    font-weight: bold;
+    margin-left: 8px;
+    color: ${(props) => props.color};
+    &:hover {
+        text-decoration: underline;
+      },
+`;
 interface TypeColumnProps {
     type: string;
     record: any;
@@ -124,9 +132,7 @@ export function LastStatusColumn({ status, record, setFocusExecutionUrn }: LastS
                 type="link"
                 onClick={() => setFocusExecutionUrn(record.lastExecUrn)}
             >
-                <Typography.Text strong style={{ color, marginLeft: 8 }}>
-                    {text || 'Pending...'}
-                </Typography.Text>
+                <StatusText color={color}>{text || 'Pending...'}</StatusText>
             </StatusButton>
         </StatusContainer>
     );
diff --git a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
index 00fdc89964f88..96dfc05e39153 100644
--- a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
@@ -83,6 +83,17 @@ const ShowMoreButton = styled(Button)`
     padding: 0px;
 `;
 
+const LogsContainer = styled.div<LogsContainerProps>`
+    margin-bottom: -25px;
+    ${(props) =>
+        props.areLogsExpandable &&
+        !props.showExpandedLogs &&
+        `
+        -webkit-mask-image: linear-gradient(to bottom, rgba(0,0,0,1) 50%, rgba(255,0,0,0.5) 60%, rgba(255,0,0,0) 90% );
+        mask-image: linear-gradient(to bottom, rgba(0,0,0,1) 50%, rgba(255,0,0,0.5) 60%, rgba(255,0,0,0) 90%);
+    `}
+`;
+
 const modalStyle = {
     top: 100,
 };
@@ -91,6 +102,11 @@ const modalBodyStyle = {
     padding: 0,
 };
 
+type LogsContainerProps = {
+    showExpandedLogs: boolean;
+    areLogsExpandable: boolean;
+};
+
 type Props = {
     urn: string;
     visible: boolean;
@@ -108,7 +124,7 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
         downloadFile(output, `exec-${urn}.log`);
     };
 
-    const logs = (showExpandedLogs && output) || output.slice(0, 100);
+    const logs = (showExpandedLogs && output) || output.slice(0, 250);
     const result = data?.executionRequest?.result?.status;
 
     useEffect(() => {
@@ -140,7 +156,7 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     }
     const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n').slice(0, 1).join('\n');
 
-    const areLogsExpandable = output.length > 100;
+    const areLogsExpandable = output.length > 250;
     const isRecipeExpandable = recipeYaml?.includes('\n');
 
     return (
@@ -181,14 +197,16 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
                             Download
                         </Button>
                     </SectionSubHeader>
-                    <Typography.Paragraph ellipsis>
-                        <pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
-                        {areLogsExpandable && (
-                            <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
-                                {showExpandedLogs ? 'Hide' : 'Show More'}
-                            </ShowMoreButton>
-                        )}
-                    </Typography.Paragraph>
+                    <LogsContainer areLogsExpandable={areLogsExpandable} showExpandedLogs={showExpandedLogs}>
+                        <Typography.Paragraph ellipsis>
+                            <pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
+                        </Typography.Paragraph>
+                    </LogsContainer>
+                    {areLogsExpandable && (
+                        <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
+                            {showExpandedLogs ? 'Hide' : 'Show More'}
+                        </ShowMoreButton>
+                    )}
                 </LogsSection>
                 {recipe && (
                     <RecipeSection>

From 89dff8f7bddee15d578170f5c1db586c628cabf4 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Fri, 10 Nov 2023 09:34:08 -0800
Subject: [PATCH 193/259] chore(ingest): start working on pydantic v2 support
 (#9220)

---
 metadata-ingestion/scripts/avro_codegen.py    |  2 +-
 metadata-ingestion/setup.cfg                  |  1 +
 .../src/datahub/cli/cli_utils.py              |  2 +-
 .../src/datahub/cli/lite_cli.py               |  2 +-
 .../src/datahub/configuration/_config_enum.py | 26 ++++--
 .../src/datahub/configuration/common.py       | 27 ++++--
 .../src/datahub/configuration/oauth.py        |  6 +-
 .../pydantic_migration_helpers.py             | 30 +++++++
 .../src/datahub/emitter/mcp_builder.py        |  2 +-
 .../ingestion/extractor/mce_extractor.py      |  6 +-
 .../datahub/ingestion/run/pipeline_config.py  |  2 +-
 .../source/bigquery_v2/bigquery_config.py     |  2 +-
 .../source/data_lake_common/path_spec.py      |  2 +-
 .../ingestion/source/datahub/config.py        |  2 +-
 .../datahub/ingestion/source/dbt/dbt_cloud.py |  1 +
 .../ingestion/source/dbt/dbt_common.py        |  6 +-
 .../src/datahub/ingestion/source/kafka.py     |  4 +-
 .../ingestion/source/looker/lookml_source.py  |  7 +-
 .../src/datahub/ingestion/source/nifi.py      |  4 +-
 .../ingestion/source/powerbi/config.py        |  5 +-
 .../report_server_domain.py                   | 84 ++++++++++---------
 .../ingestion/source/redshift/config.py       |  6 +-
 .../src/datahub/ingestion/source/s3/config.py |  2 +-
 .../source/s3/datalake_profiler_config.py     |  2 +-
 .../datahub/ingestion/source/salesforce.py    |  2 +-
 .../source/snowflake/snowflake_usage_v2.py    | 20 ++---
 .../ingestion/source/sql/clickhouse.py        | 11 +--
 .../src/datahub/ingestion/source/sql/druid.py |  2 +-
 .../src/datahub/ingestion/source/sql/hive.py  |  2 +-
 .../src/datahub/ingestion/source/sql/mysql.py |  4 +-
 .../datahub/ingestion/source/sql/postgres.py  |  6 +-
 .../datahub/ingestion/source/sql/presto.py    |  2 +-
 .../datahub/ingestion/source/sql/redshift.py  |  2 +-
 .../ingestion/source/sql/sql_config.py        |  2 +-
 .../datahub/ingestion/source/sql/teradata.py  |  2 +-
 .../src/datahub/ingestion/source/sql/trino.py |  2 +-
 .../source/state/stateful_ingestion_base.py   | 10 +--
 .../src/datahub/ingestion/source/superset.py  |  2 +-
 .../datahub/ingestion/source/unity/config.py  |  2 +-
 .../ingestion/source_config/sql/snowflake.py  |  3 +-
 .../source_config/usage/bigquery_usage.py     |  2 +-
 .../src/datahub/upgrade/upgrade.py            |  8 +-
 .../datahub/utilities/lossy_collections.py    | 12 +++
 .../src/datahub/utilities/sqlglot_lineage.py  | 26 +++---
 44 files changed, 216 insertions(+), 139 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/configuration/pydantic_migration_helpers.py

diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py
index 021ebd4a31eb3..de8836559217b 100644
--- a/metadata-ingestion/scripts/avro_codegen.py
+++ b/metadata-ingestion/scripts/avro_codegen.py
@@ -192,7 +192,7 @@ def add_avro_python3_warning(filepath: Path) -> None:
 # This means that installation order matters, which is a pretty unintuitive outcome.
 # See https://github.com/pypa/pip/issues/4625 for details.
 try:
-    from avro.schema import SchemaFromJSONData
+    from avro.schema import SchemaFromJSONData  # type: ignore
     import warnings
 
     warnings.warn("It seems like 'avro-python3' is installed, which conflicts with the 'avro' package used by datahub. "
diff --git a/metadata-ingestion/setup.cfg b/metadata-ingestion/setup.cfg
index 8b78e4d3c9c6f..b3fc53ccfaf58 100644
--- a/metadata-ingestion/setup.cfg
+++ b/metadata-ingestion/setup.cfg
@@ -88,6 +88,7 @@ filterwarnings =
     ignore:Deprecated call to \`pkg_resources.declare_namespace:DeprecationWarning
     ignore:pkg_resources is deprecated as an API:DeprecationWarning
     ignore:Did not recognize type:sqlalchemy.exc.SAWarning
+    ignore::datahub.configuration.pydantic_migration_helpers.PydanticDeprecatedSince20
 
 [coverage:run]
 # Because of some quirks in the way setup.cfg, coverage.py, pytest-cov,
diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py
index a7ea5b4f65785..8ac9a101121be 100644
--- a/metadata-ingestion/src/datahub/cli/cli_utils.py
+++ b/metadata-ingestion/src/datahub/cli/cli_utils.py
@@ -47,7 +47,7 @@
 
 class GmsConfig(BaseModel):
     server: str
-    token: Optional[str]
+    token: Optional[str] = None
 
 
 class DatahubConfig(BaseModel):
diff --git a/metadata-ingestion/src/datahub/cli/lite_cli.py b/metadata-ingestion/src/datahub/cli/lite_cli.py
index b49284bb627f2..8636187a51d09 100644
--- a/metadata-ingestion/src/datahub/cli/lite_cli.py
+++ b/metadata-ingestion/src/datahub/cli/lite_cli.py
@@ -40,7 +40,7 @@ class DuckDBLiteConfigWrapper(DuckDBLiteConfig):
 
 class LiteCliConfig(DatahubConfig):
     lite: LiteLocalConfig = LiteLocalConfig(
-        type="duckdb", config=DuckDBLiteConfigWrapper()
+        type="duckdb", config=DuckDBLiteConfigWrapper().dict()
     )
 
 
diff --git a/metadata-ingestion/src/datahub/configuration/_config_enum.py b/metadata-ingestion/src/datahub/configuration/_config_enum.py
index b4fb93dae4439..190a006b077d9 100644
--- a/metadata-ingestion/src/datahub/configuration/_config_enum.py
+++ b/metadata-ingestion/src/datahub/configuration/_config_enum.py
@@ -4,6 +4,8 @@
 import pydantic.types
 import pydantic.validators
 
+from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2
+
 
 class ConfigEnum(Enum):
     # Ideally we would use @staticmethod here, but some versions of Python don't support it.
@@ -15,11 +17,25 @@ def _generate_next_value_(  # type: ignore
         # From https://stackoverflow.com/a/44785241/5004662.
         return name
 
-    @classmethod
-    def __get_validators__(cls) -> "pydantic.types.CallableGenerator":
-        # We convert the text to uppercase before attempting to match it to an enum value.
-        yield cls.validate
-        yield pydantic.validators.enum_member_validator
+    if PYDANTIC_VERSION_2:
+        # if TYPE_CHECKING:
+        #     from pydantic import GetCoreSchemaHandler
+
+        @classmethod
+        def __get_pydantic_core_schema__(cls, source_type, handler):  # type: ignore
+            from pydantic_core import core_schema
+
+            return core_schema.no_info_before_validator_function(
+                cls.validate, handler(source_type)
+            )
+
+    else:
+
+        @classmethod
+        def __get_validators__(cls) -> "pydantic.types.CallableGenerator":
+            # We convert the text to uppercase before attempting to match it to an enum value.
+            yield cls.validate
+            yield pydantic.validators.enum_member_validator
 
     @classmethod
     def validate(cls, v):  # type: ignore[no-untyped-def]
diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py
index 73ac4baac48c0..f225856ca43ce 100644
--- a/metadata-ingestion/src/datahub/configuration/common.py
+++ b/metadata-ingestion/src/datahub/configuration/common.py
@@ -11,6 +11,7 @@
 from typing_extensions import Protocol, runtime_checkable
 
 from datahub.configuration._config_enum import ConfigEnum
+from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2
 from datahub.utilities.dedup_list import deduplicate_list
 
 _ConfigSelf = TypeVar("_ConfigSelf", bound="ConfigModel")
@@ -71,14 +72,8 @@ def redact_raw_config(obj: Any) -> Any:
 
 class ConfigModel(BaseModel):
     class Config:
-        extra = Extra.forbid
-        underscore_attrs_are_private = True
-        keep_untouched = (
-            cached_property,
-        )  # needed to allow cached_property to work. See https://github.com/samuelcolvin/pydantic/issues/1241 for more info.
-
         @staticmethod
-        def schema_extra(schema: Dict[str, Any], model: Type["ConfigModel"]) -> None:
+        def _schema_extra(schema: Dict[str, Any], model: Type["ConfigModel"]) -> None:
             # We use the custom "hidden_from_docs" attribute to hide fields from the
             # autogenerated docs.
             remove_fields = []
@@ -89,6 +84,19 @@ def schema_extra(schema: Dict[str, Any], model: Type["ConfigModel"]) -> None:
             for key in remove_fields:
                 del schema["properties"][key]
 
+        # This is purely to suppress pydantic's warnings, since this class is used everywhere.
+        if PYDANTIC_VERSION_2:
+            extra = "forbid"
+            ignored_types = (cached_property,)
+            json_schema_extra = _schema_extra
+        else:
+            extra = Extra.forbid
+            underscore_attrs_are_private = True
+            keep_untouched = (
+                cached_property,
+            )  # needed to allow cached_property to work. See https://github.com/samuelcolvin/pydantic/issues/1241 for more info.
+            schema_extra = _schema_extra
+
     @classmethod
     def parse_obj_allow_extras(cls: Type[_ConfigSelf], obj: Any) -> _ConfigSelf:
         with unittest.mock.patch.object(cls.Config, "extra", pydantic.Extra.allow):
@@ -102,7 +110,10 @@ class PermissiveConfigModel(ConfigModel):
     # It is usually used for argument bags that are passed through to third-party libraries.
 
     class Config:
-        extra = Extra.allow
+        if PYDANTIC_VERSION_2:
+            extra = "allow"
+        else:
+            extra = Extra.allow
 
 
 class TransformerSemantics(ConfigEnum):
diff --git a/metadata-ingestion/src/datahub/configuration/oauth.py b/metadata-ingestion/src/datahub/configuration/oauth.py
index 9a1ddbf437913..61a06580299db 100644
--- a/metadata-ingestion/src/datahub/configuration/oauth.py
+++ b/metadata-ingestion/src/datahub/configuration/oauth.py
@@ -24,11 +24,11 @@ class OAuthConfiguration(ConfigModel):
         default=False,
     )
     client_secret: Optional[SecretStr] = Field(
-        description="client secret of the application if use_certificate = false"
+        None, description="client secret of the application if use_certificate = false"
     )
     encoded_oauth_public_key: Optional[str] = Field(
-        description="base64 encoded certificate content if use_certificate = true"
+        None, description="base64 encoded certificate content if use_certificate = true"
     )
     encoded_oauth_private_key: Optional[str] = Field(
-        description="base64 encoded private key content if use_certificate = true"
+        None, description="base64 encoded private key content if use_certificate = true"
     )
diff --git a/metadata-ingestion/src/datahub/configuration/pydantic_migration_helpers.py b/metadata-ingestion/src/datahub/configuration/pydantic_migration_helpers.py
new file mode 100644
index 0000000000000..f1876b500598b
--- /dev/null
+++ b/metadata-ingestion/src/datahub/configuration/pydantic_migration_helpers.py
@@ -0,0 +1,30 @@
+import pydantic.version
+from packaging.version import Version
+
+PYDANTIC_VERSION_2: bool
+if Version(pydantic.version.VERSION) >= Version("2.0"):
+    PYDANTIC_VERSION_2 = True
+else:
+    PYDANTIC_VERSION_2 = False
+
+
+# This can be used to silence deprecation warnings while we migrate.
+if PYDANTIC_VERSION_2:
+    from pydantic import PydanticDeprecatedSince20  # type: ignore
+else:
+
+    class PydanticDeprecatedSince20(Warning):  # type: ignore
+        pass
+
+
+if PYDANTIC_VERSION_2:
+    from pydantic import BaseModel as GenericModel
+else:
+    from pydantic.generics import GenericModel  # type: ignore
+
+
+__all__ = [
+    "PYDANTIC_VERSION_2",
+    "PydanticDeprecatedSince20",
+    "GenericModel",
+]
diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
index d50feba8b119c..a7578e39374ac 100644
--- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -127,7 +127,7 @@ class BucketKey(ContainerKey):
 class NotebookKey(DatahubKey):
     notebook_id: int
     platform: str
-    instance: Optional[str]
+    instance: Optional[str] = None
 
     def as_urn(self) -> str:
         return make_dataset_urn_with_platform_instance(
diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py b/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py
index 36450dda153d7..7ad68c0fcf8ea 100644
--- a/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py
+++ b/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py
@@ -26,11 +26,11 @@ def _try_reformat_with_black(code: str) -> str:
 
 
 class WorkUnitRecordExtractorConfig(ConfigModel):
-    set_system_metadata = True
-    set_system_metadata_pipeline_name = (
+    set_system_metadata: bool = True
+    set_system_metadata_pipeline_name: bool = (
         False  # false for now until the models are available in OSS
     )
-    unpack_mces_into_mcps = False
+    unpack_mces_into_mcps: bool = False
 
 
 class WorkUnitRecordExtractor(
diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
index da3cee8ad9c1b..f22f94c9e9351 100644
--- a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py
@@ -72,7 +72,7 @@ class PipelineConfig(ConfigModel):
 
     source: SourceConfig
     sink: DynamicTypedConfig
-    transformers: Optional[List[DynamicTypedConfig]]
+    transformers: Optional[List[DynamicTypedConfig]] = None
     flags: FlagsConfig = Field(default=FlagsConfig(), hidden_from_docs=True)
     reporting: List[ReporterConfig] = []
     run_id: str = DEFAULT_RUN_ID
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
index f762d451849ab..cbe68a454ea43 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -265,7 +265,7 @@ def validate_column_lineage(cls, v: bool, values: Dict[str, Any]) -> bool:
         description="Option to exclude empty projects from being ingested.",
     )
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def profile_default_settings(cls, values: Dict) -> Dict:
         # Extra default SQLAlchemy option for better connection pooling and threading.
         # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow
diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
index a35fb94614f72..05b1b6b7cc040 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
@@ -214,7 +214,7 @@ def glob_include(self):
         logger.debug(f"Setting _glob_include: {glob_include}")
         return glob_include
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def validate_path_spec(cls, values: Dict) -> Dict[str, Any]:
         # validate that main fields are populated
         required_fields = ["include", "file_types", "default_extension"]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
index 83958dc76754f..a2bd6fd1e5558 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
@@ -80,7 +80,7 @@ class DataHubSourceConfig(StatefulIngestionConfigBase):
         hidden_from_docs=True,
     )
 
-    @root_validator
+    @root_validator(skip_on_failure=True)
     def check_ingesting_data(cls, values):
         if (
             not values.get("database_connection")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
index da1ea8ecb4678..a9685b2554553 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
@@ -46,6 +46,7 @@ class DBTCloudConfig(DBTCommonConfig):
         description="The ID of the job to ingest metadata from.",
     )
     run_id: Optional[int] = Field(
+        None,
         description="The ID of the run to ingest metadata from. If not specified, we'll default to the latest run.",
     )
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index c4de24bf192f1..76cb82aaa5b4b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -150,7 +150,7 @@ class DBTEntitiesEnabled(ConfigModel):
         description="Emit metadata for test results when set to Yes or Only",
     )
 
-    @root_validator
+    @root_validator(skip_on_failure=True)
     def process_only_directive(cls, values):
         # Checks that at most one is set to ONLY, and then sets the others to NO.
 
@@ -229,7 +229,7 @@ class DBTCommonConfig(
         default={},
         description="mapping rules that will be executed against dbt column meta properties. Refer to the section below on dbt meta automated mappings.",
     )
-    enable_meta_mapping = Field(
+    enable_meta_mapping: bool = Field(
         default=True,
         description="When enabled, applies the mappings that are defined through the meta_mapping directives.",
     )
@@ -237,7 +237,7 @@ class DBTCommonConfig(
         default={},
         description="mapping rules that will be executed against dbt query_tag meta properties. Refer to the section below on dbt meta automated mappings.",
     )
-    enable_query_tag_mapping = Field(
+    enable_query_tag_mapping: bool = Field(
         default=True,
         description="When enabled, applies the mappings that are defined through the `query_tag_mapping` directives.",
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
index 23770ff3cf812..25520e7aa66ff 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
@@ -100,11 +100,11 @@ class KafkaSourceConfig(
         default="datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry",
         description="The fully qualified implementation class(custom) that implements the KafkaSchemaRegistryBase interface.",
     )
-    schema_tags_field = pydantic.Field(
+    schema_tags_field: str = pydantic.Field(
         default="tags",
         description="The field name in the schema metadata that contains the tags to be added to the dataset.",
     )
-    enable_meta_mapping = pydantic.Field(
+    enable_meta_mapping: bool = pydantic.Field(
         default=True,
         description="When enabled, applies the mappings that are defined through the meta_mapping directives.",
     )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
index e6b78cc7a7745..9d7c972612777 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
@@ -275,7 +275,7 @@ def convert_string_to_connection_def(cls, conn_map):
                     )
         return conn_map
 
-    @root_validator()
+    @root_validator(skip_on_failure=True)
     def check_either_connection_map_or_connection_provided(cls, values):
         """Validate that we must either have a connection map or an api credential"""
         if not values.get("connection_to_platform_map", {}) and not values.get(
@@ -286,7 +286,7 @@ def check_either_connection_map_or_connection_provided(cls, values):
             )
         return values
 
-    @root_validator()
+    @root_validator(skip_on_failure=True)
     def check_either_project_name_or_api_provided(cls, values):
         """Validate that we must either have a project name or an api credential to fetch project names"""
         if not values.get("project_name") and not values.get("api"):
@@ -1070,7 +1070,6 @@ def _get_fields(
     def determine_view_file_path(
         cls, base_folder_path: str, absolute_file_path: str
     ) -> str:
-
         splits: List[str] = absolute_file_path.split(base_folder_path, 1)
         if len(splits) != 2:
             logger.debug(
@@ -1104,7 +1103,6 @@ def from_looker_dict(
         populate_sql_logic_in_descriptions: bool = False,
         process_isolation_for_sql_parsing: bool = False,
     ) -> Optional["LookerView"]:
-
         view_name = looker_view["name"]
         logger.debug(f"Handling view {view_name} in model {model_name}")
         # The sql_table_name might be defined in another view and this view is extending that view,
@@ -2087,7 +2085,6 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                 )
 
                 if looker_viewfile is not None:
-
                     for raw_view in looker_viewfile.views:
                         raw_view_name = raw_view["name"]
                         if LookerRefinementResolver.is_refinement(raw_view_name):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
index bc05edbb3c623..ab418b1705956 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -126,7 +126,7 @@ class NifiSourceConfig(EnvConfigMixin):
         description="Path to PEM file containing certs for the root CA(s) for the NiFi",
     )
 
-    @root_validator
+    @root_validator(skip_on_failure=True)
     def validate_auth_params(cla, values):
         if values.get("auth") is NifiAuthType.CLIENT_CERT and not values.get(
             "client_cert_file"
@@ -143,7 +143,7 @@ def validate_auth_params(cla, values):
             )
         return values
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def validator_site_url_to_site_name(cls, values):
         site_url_to_site_name = values.get("site_url_to_site_name")
         site_url = values.get("site_url")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
index 96729f4c60c6c..b8cc34c234ffa 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
@@ -405,8 +405,7 @@ class PowerBiDashboardSourceConfig(
         "Works for M-Query where native SQL is used for transformation.",
     )
 
-    @root_validator
-    @classmethod
+    @root_validator(skip_on_failure=True)
     def validate_extract_column_level_lineage(cls, values: Dict) -> Dict:
         flags = [
             "native_query_parsing",
@@ -445,7 +444,7 @@ def map_data_platform(cls, value):
 
         return value
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def workspace_id_backward_compatibility(cls, values: Dict) -> Dict:
         workspace_id = values.get("workspace_id")
         workspace_id_pattern = values.get("workspace_id_pattern")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py
index 60426fc5bd660..ee87d93774b3d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py
@@ -12,21 +12,21 @@
 class CatalogItem(BaseModel):
     id: str = Field(alias="Id")
     name: str = Field(alias="Name")
-    description: Optional[str] = Field(alias="Description")
+    description: Optional[str] = Field(None, alias="Description")
     path: str = Field(alias="Path")
-    type: Any = Field(alias="Type")
+    type: Any = Field(None, alias="Type")
     hidden: bool = Field(alias="Hidden")
     size: int = Field(alias="Size")
-    modified_by: Optional[str] = Field(alias="ModifiedBy")
-    modified_date: Optional[datetime] = Field(alias="ModifiedDate")
-    created_by: Optional[str] = Field(alias="CreatedBy")
-    created_date: Optional[datetime] = Field(alias="CreatedDate")
-    parent_folder_id: Optional[str] = Field(alias="ParentFolderId")
-    content_type: Optional[str] = Field(alias="ContentType")
+    modified_by: Optional[str] = Field(None, alias="ModifiedBy")
+    modified_date: Optional[datetime] = Field(None, alias="ModifiedDate")
+    created_by: Optional[str] = Field(None, alias="CreatedBy")
+    created_date: Optional[datetime] = Field(None, alias="CreatedDate")
+    parent_folder_id: Optional[str] = Field(None, alias="ParentFolderId")
+    content_type: Optional[str] = Field(None, alias="ContentType")
     content: str = Field(alias="Content")
     is_favorite: bool = Field(alias="IsFavorite")
-    user_info: Any = Field(alias="UserInfo")
-    display_name: Optional[str] = Field(alias="DisplayName")
+    user_info: Any = Field(None, alias="UserInfo")
+    display_name: Optional[str] = Field(None, alias="DisplayName")
     has_data_sources: bool = Field(default=False, alias="HasDataSources")
     data_sources: Optional[List["DataSource"]] = Field(
         default_factory=list, alias="DataSources"
@@ -72,12 +72,12 @@ def __hash__(self):
 
 
 class DataModelDataSource(BaseModel):
-    auth_type: Optional[str] = Field(alias="AuthType")
+    auth_type: Optional[str] = Field(None, alias="AuthType")
     supported_auth_types: List[Optional[str]] = Field(alias="SupportedAuthTypes")
     kind: str = Field(alias="Kind")
     model_connection_name: str = Field(alias="ModelConnectionName")
     secret: str = Field(alias="Secret")
-    type: Optional[str] = Field(alias="Type")
+    type: Optional[str] = Field(None, alias="Type")
     username: str = Field(alias="Username")
 
 
@@ -135,21 +135,23 @@ class DataSource(CatalogItem):
     is_enabled: bool = Field(alias="IsEnabled")
     connection_string: str = Field(alias="ConnectionString")
     data_model_data_source: Optional[DataModelDataSource] = Field(
-        alias="DataModelDataSource"
+        None, alias="DataModelDataSource"
     )
-    data_source_sub_type: Optional[str] = Field(alias="DataSourceSubType")
-    data_source_type: Optional[str] = Field(alias="DataSourceType")
+    data_source_sub_type: Optional[str] = Field(None, alias="DataSourceSubType")
+    data_source_type: Optional[str] = Field(None, alias="DataSourceType")
     is_original_connection_string_expression_based: bool = Field(
         alias="IsOriginalConnectionStringExpressionBased"
     )
     is_connection_string_overridden: bool = Field(alias="IsConnectionStringOverridden")
-    credentials_by_user: Optional[CredentialsByUser] = Field(alias="CredentialsByUser")
+    credentials_by_user: Optional[CredentialsByUser] = Field(
+        None, alias="CredentialsByUser"
+    )
     credentials_in_server: Optional[CredentialsInServer] = Field(
-        alias="CredentialsInServer"
+        None, alias="CredentialsInServer"
     )
     is_reference: bool = Field(alias="IsReference")
-    subscriptions: Optional[Subscription] = Field(alias="Subscriptions")
-    meta_data: Optional[MetaData] = Field(alias="MetaData")
+    subscriptions: Optional[Subscription] = Field(None, alias="Subscriptions")
+    meta_data: Optional[MetaData] = Field(None, alias="MetaData")
 
     def __members(self):
         return (self.id,)
@@ -274,15 +276,15 @@ def __hash__(self):
 class CorpUserEditableInfo(BaseModel):
     display_name: str = Field(alias="displayName")
     title: str
-    about_me: Optional[str] = Field(alias="aboutMe")
-    teams: Optional[List[str]]
-    skills: Optional[List[str]]
-    picture_link: Optional[str] = Field(alias="pictureLink")
+    about_me: Optional[str] = Field(None, alias="aboutMe")
+    teams: Optional[List[str]] = None
+    skills: Optional[List[str]] = None
+    picture_link: Optional[str] = Field(None, alias="pictureLink")
 
 
 class CorpUserEditableProperties(CorpUserEditableInfo):
-    slack: Optional[str]
-    phone: Optional[str]
+    slack: Optional[str] = None
+    phone: Optional[str] = None
     email: str
 
 
@@ -305,21 +307,21 @@ class EntityRelationshipsResult(BaseModel):
     start: int
     count: int
     total: int
-    relationships: Optional[EntityRelationship]
+    relationships: Optional[EntityRelationship] = None
 
 
 class CorpUserProperties(BaseModel):
     active: bool
     display_name: str = Field(alias="displayName")
     email: str
-    title: Optional[str]
-    manager: Optional["CorpUser"]
-    department_id: Optional[int] = Field(alias="departmentId")
-    department_name: Optional[str] = Field(alias="departmentName")
-    first_name: Optional[str] = Field(alias="firstName")
-    last_name: Optional[str] = Field(alias="lastName")
-    full_name: Optional[str] = Field(alias="fullName")
-    country_code: Optional[str] = Field(alias="countryCode")
+    title: Optional[str] = None
+    manager: Optional["CorpUser"] = None
+    department_id: Optional[int] = Field(None, alias="departmentId")
+    department_name: Optional[str] = Field(None, alias="departmentName")
+    first_name: Optional[str] = Field(None, alias="firstName")
+    last_name: Optional[str] = Field(None, alias="lastName")
+    full_name: Optional[str] = Field(None, alias="fullName")
+    country_code: Optional[str] = Field(None, alias="countryCode")
 
 
 class CorpUser(BaseModel):
@@ -328,13 +330,13 @@ class CorpUser(BaseModel):
     username: str
     properties: CorpUserProperties
     editable_properties: Optional[CorpUserEditableProperties] = Field(
-        alias="editableProperties"
+        None, alias="editableProperties"
     )
-    status: Optional[CorpUserStatus]
-    tags: Optional[GlobalTags]
-    relationships: Optional[EntityRelationshipsResult]
-    editableInfo: Optional[CorpUserEditableInfo] = Field(alias="editableInfo")
-    global_tags: Optional[GlobalTags] = Field(alias="globalTags")
+    status: Optional[CorpUserStatus] = None
+    tags: Optional[GlobalTags] = None
+    relationships: Optional[EntityRelationshipsResult] = None
+    editableInfo: Optional[CorpUserEditableInfo] = Field(None, alias="editableInfo")
+    global_tags: Optional[GlobalTags] = Field(None, alias="globalTags")
 
     def get_urn_part(self):
         return "{}".format(self.username)
@@ -353,7 +355,7 @@ def __hash__(self):
 
 class OwnershipData(BaseModel):
     existing_owners: Optional[List[OwnerClass]] = []
-    owner_to_add: Optional[CorpUser]
+    owner_to_add: Optional[CorpUser] = None
 
     class Config:
         arbitrary_types_allowed = True
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
index 79b044841e054..9cbf1823db939 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -81,7 +81,7 @@ class RedshiftConfig(
     # Because of this behavior, it uses dramatically fewer round trips for
     # large Redshift warehouses. As an example, see this query for the columns:
     # https://github.com/sqlalchemy-redshift/sqlalchemy-redshift/blob/60b4db04c1d26071c291aeea52f1dcb5dd8b0eb0/sqlalchemy_redshift/dialect.py#L745.
-    scheme = Field(
+    scheme: str = Field(
         default="redshift+psycopg2",
         description="",
         hidden_from_schema=True,
@@ -150,14 +150,14 @@ def check_email_is_set_on_usage(cls, values):
             ), "email_domain needs to be set if usage is enabled"
         return values
 
-    @root_validator()
+    @root_validator(skip_on_failure=True)
     def check_database_or_database_alias_set(cls, values):
         assert values.get("database") or values.get(
             "database_alias"
         ), "either database or database_alias must be set"
         return values
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def backward_compatibility_configs_set(cls, values: Dict) -> Dict:
         match_fully_qualified_names = values.get("match_fully_qualified_names")
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
index 3ef6476078f6f..f752a33b42d9c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
@@ -144,7 +144,7 @@ def platform_not_empty(cls, platform: str, values: dict) -> str:
             raise ValueError("platform must not be empty")
         return platform
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def ensure_profiling_pattern_is_passed_to_profiling(
         cls, values: Dict[str, Any]
     ) -> Dict[str, Any]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py
index 9f6d13a08b182..89c092875e449 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py
@@ -72,7 +72,7 @@ class DataLakeProfilerConfig(ConfigModel):
         description="Whether to profile for the sample values for all columns.",
     )
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def ensure_field_level_settings_are_normalized(
         cls: "DataLakeProfilerConfig", values: Dict[str, Any]
     ) -> Dict[str, Any]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
index 3475c9f2881c1..6d52646f85d0a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
@@ -83,7 +83,7 @@ class SalesforceProfilingConfig(ConfigModel):
 
 
 class SalesforceConfig(DatasetSourceConfigMixin):
-    platform = "salesforce"
+    platform: str = "salesforce"
 
     auth: SalesforceAuthType = SalesforceAuthType.USERNAME_PASSWORD
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
index 1cbd4a3b3ea24..8f571313f1888 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
@@ -79,30 +79,30 @@ class SnowflakeColumnReference(PermissiveModel):
 
 
 class SnowflakeObjectAccessEntry(PermissiveModel):
-    columns: Optional[List[SnowflakeColumnReference]]
+    columns: Optional[List[SnowflakeColumnReference]] = None
     objectDomain: str
     objectName: str
     # Seems like it should never be null, but in practice have seen null objectIds
-    objectId: Optional[int]
-    stageKind: Optional[str]
+    objectId: Optional[int] = None
+    stageKind: Optional[str] = None
 
 
 class SnowflakeJoinedAccessEvent(PermissiveModel):
     query_start_time: datetime
     query_text: str
     query_type: str
-    rows_inserted: Optional[int]
-    rows_updated: Optional[int]
-    rows_deleted: Optional[int]
+    rows_inserted: Optional[int] = None
+    rows_updated: Optional[int] = None
+    rows_deleted: Optional[int] = None
     base_objects_accessed: List[SnowflakeObjectAccessEntry]
     direct_objects_accessed: List[SnowflakeObjectAccessEntry]
     objects_modified: List[SnowflakeObjectAccessEntry]
 
     user_name: str
-    first_name: Optional[str]
-    last_name: Optional[str]
-    display_name: Optional[str]
-    email: Optional[str]
+    first_name: Optional[str] = None
+    last_name: Optional[str] = None
+    display_name: Optional[str] = None
+    email: Optional[str] = None
     role_name: str
 
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
index 8873038079bad..30893fd03be22 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py
@@ -5,12 +5,11 @@
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
 
-import clickhouse_driver  # noqa: F401
+import clickhouse_driver
 import clickhouse_sqlalchemy.types as custom_types
 import pydantic
 from clickhouse_sqlalchemy.drivers import base
 from clickhouse_sqlalchemy.drivers.base import ClickHouseDialect
-from pydantic.class_validators import root_validator
 from pydantic.fields import Field
 from sqlalchemy import create_engine, text
 from sqlalchemy.engine import reflection
@@ -59,6 +58,8 @@
     UpstreamClass,
 )
 
+assert clickhouse_driver
+
 # adding extra types not handled by clickhouse-sqlalchemy 0.1.8
 base.ischema_names["DateTime64(0)"] = DATETIME
 base.ischema_names["DateTime64(1)"] = DATETIME
@@ -126,8 +127,8 @@ class ClickHouseConfig(
     TwoTierSQLAlchemyConfig, BaseTimeWindowConfig, DatasetLineageProviderConfigBase
 ):
     # defaults
-    host_port = Field(default="localhost:8123", description="ClickHouse host URL.")
-    scheme = Field(default="clickhouse", description="", hidden_from_docs=True)
+    host_port: str = Field(default="localhost:8123", description="ClickHouse host URL.")
+    scheme: str = Field(default="clickhouse", description="", hidden_from_docs=True)
     password: pydantic.SecretStr = Field(
         default=pydantic.SecretStr(""), description="password"
     )
@@ -165,7 +166,7 @@ def get_sql_alchemy_url(self, current_db=None):
         return str(url)
 
     # pre = True because we want to take some decision before pydantic initialize the configuration to default values
-    @root_validator(pre=True)
+    @pydantic.root_validator(pre=True)
     def projects_backward_compatibility(cls, values: Dict) -> Dict:
         secure = values.get("secure")
         protocol = values.get("protocol")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py b/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py
index 1dfa44f549135..3f20e0a0f18b6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py
@@ -32,7 +32,7 @@ def get_table_names(self, connection, schema=None, **kwargs):
 
 class DruidConfig(BasicSQLAlchemyConfig):
     # defaults
-    scheme = "druid"
+    scheme: str = "druid"
     schema_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern(deny=["^(lookup|sysgit|view).*"]),
         description="regex patterns for schemas to filter in ingestion.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
index d081acb6c1eff..003732236ba80 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
@@ -122,7 +122,7 @@ def get_view_definition_patched(self, connection, view_name, schema=None, **kw):
 
 class HiveConfig(TwoTierSQLAlchemyConfig):
     # defaults
-    scheme = Field(default="hive", hidden_from_docs=True)
+    scheme: str = Field(default="hive", hidden_from_docs=True)
 
     @validator("host_port")
     def clean_host_port(cls, v):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py
index e4969ce946f78..891b64066721b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py
@@ -48,8 +48,8 @@
 
 class MySQLConnectionConfig(SQLAlchemyConnectionConfig):
     # defaults
-    host_port = Field(default="localhost:3306", description="MySQL host URL.")
-    scheme = "mysql+pymysql"
+    host_port: str = Field(default="localhost:3306", description="MySQL host URL.")
+    scheme: str = "mysql+pymysql"
 
 
 class MySQLConfig(MySQLConnectionConfig, TwoTierSQLAlchemyConfig):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
index 4f133c6459a0f..c8418075928ef 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
@@ -98,8 +98,10 @@ class ViewLineageEntry(BaseModel):
 
 
 class BasePostgresConfig(BasicSQLAlchemyConfig):
-    scheme = Field(default="postgresql+psycopg2", description="database scheme")
-    schema_pattern = Field(default=AllowDenyPattern(deny=["information_schema"]))
+    scheme: str = Field(default="postgresql+psycopg2", description="database scheme")
+    schema_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern(deny=["information_schema"])
+    )
 
 
 class PostgresConfig(BasePostgresConfig):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto.py
index c7331b4e53e5e..9333c6edd1fa5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto.py
@@ -85,7 +85,7 @@ def _get_full_table(  # type: ignore
 
 class PrestoConfig(TrinoConfig):
     # defaults
-    scheme = Field(default="presto", description="", hidden_from_docs=True)
+    scheme: str = Field(default="presto", description="", hidden_from_docs=True)
 
 
 @platform_name("Presto", doc_order=1)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py
index cdab52ebc3935..33d517c8589e9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py
@@ -145,7 +145,7 @@ def get_identifier(self, schema: str, table: str) -> str:
     # Because of this behavior, it uses dramatically fewer round trips for
     # large Redshift warehouses. As an example, see this query for the columns:
     # https://github.com/sqlalchemy-redshift/sqlalchemy-redshift/blob/60b4db04c1d26071c291aeea52f1dcb5dd8b0eb0/sqlalchemy_redshift/dialect.py#L745.
-    scheme = Field(
+    scheme: str = Field(
         default="redshift+psycopg2",
         description="",
         hidden_from_docs=True,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
index 095b8e6443171..6a76ae847218d 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py
@@ -107,7 +107,7 @@ def view_pattern_is_table_pattern_unless_specified(
             values["view_pattern"] = table_pattern
         return values
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def ensure_profiling_pattern_is_passed_to_profiling(
         cls, values: Dict[str, Any]
     ) -> Dict[str, Any]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index 899a7b6697c0a..8aeb1e50cd0b3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -70,7 +70,7 @@ class TeradataReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowRep
 
 
 class BaseTeradataConfig(TwoTierSQLAlchemyConfig):
-    scheme = Field(default="teradatasql", description="database scheme")
+    scheme: str = Field(default="teradatasql", description="database scheme")
 
 
 class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py
index 3b80cbed86c02..2b693d9d80d91 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py
@@ -133,7 +133,7 @@ def _get_columns(self, connection, table_name, schema: str = None, **kw):  # typ
 
 class TrinoConfig(BasicSQLAlchemyConfig):
     # defaults
-    scheme = Field(default="trino", description="", hidden_from_docs=True)
+    scheme: str = Field(default="trino", description="", hidden_from_docs=True)
 
     def get_identifier(self: BasicSQLAlchemyConfig, schema: str, table: str) -> str:
         regular = f"{schema}.{table}"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
index d11b1f9ad6a53..b1d2b276130a9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
@@ -5,13 +5,13 @@
 import pydantic
 from pydantic import root_validator
 from pydantic.fields import Field
-from pydantic.generics import GenericModel
 
 from datahub.configuration.common import (
     ConfigModel,
     ConfigurationError,
     DynamicTypedConfig,
 )
+from datahub.configuration.pydantic_migration_helpers import GenericModel
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.api.common import PipelineContext
@@ -77,7 +77,7 @@ class StatefulIngestionConfig(ConfigModel):
         hidden_from_docs=True,
     )
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def validate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
         if values.get("enabled"):
             if values.get("state_provider") is None:
@@ -112,7 +112,7 @@ class StatefulLineageConfigMixin:
         "store_last_lineage_extraction_timestamp", "enable_stateful_lineage_ingestion"
     )
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def lineage_stateful_option_validator(cls, values: Dict) -> Dict:
         sti = values.get("stateful_ingestion")
         if not sti or not sti.enabled:
@@ -137,7 +137,7 @@ class StatefulProfilingConfigMixin(ConfigModel):
         "store_last_profiling_timestamps", "enable_stateful_profiling"
     )
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def profiling_stateful_option_validator(cls, values: Dict) -> Dict:
         sti = values.get("stateful_ingestion")
         if not sti or not sti.enabled:
@@ -161,7 +161,7 @@ class StatefulUsageConfigMixin(BaseTimeWindowConfig):
         "store_last_usage_extraction_timestamp", "enable_stateful_usage_ingestion"
     )
 
-    @root_validator(pre=False)
+    @root_validator(skip_on_failure=True)
     def last_usage_extraction_stateful_option_validator(cls, values: Dict) -> Dict:
         sti = values.get("stateful_ingestion")
         if not sti or not sti.enabled:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py
index e491a1e8b82fa..1ae971e4a82d0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/superset.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py
@@ -105,7 +105,7 @@ class SupersetConfig(StatefulIngestionConfigBase, ConfigModel):
     def remove_trailing_slash(cls, v):
         return config_clean.remove_trailing_slashes(v)
 
-    @root_validator
+    @root_validator(skip_on_failure=True)
     def default_display_uri_to_connect_uri(cls, values):
         base = values.get("display_uri")
         if base is None:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index 16820c37d546e..7073830318abe 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -76,7 +76,7 @@ class UnityCatalogProfilerConfig(ConfigModel):
         description="Number of worker threads to use for profiling. Set to 1 to disable.",
     )
 
-    @pydantic.root_validator
+    @pydantic.root_validator(skip_on_failure=True)
     def warehouse_id_required_for_profiling(
         cls, values: Dict[str, Any]
     ) -> Dict[str, Any]:
diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
index 9fc697018ecd6..2e9a15063661e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
@@ -340,7 +340,6 @@ def get_connection(self) -> snowflake.connector.SnowflakeConnection:
 
 
 class SnowflakeConfig(BaseSnowflakeConfig, BaseTimeWindowConfig, SQLCommonConfig):
-
     include_table_lineage: bool = pydantic.Field(
         default=True,
         description="If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role and Snowflake Enterprise Edition or above.",
@@ -357,7 +356,7 @@ class SnowflakeConfig(BaseSnowflakeConfig, BaseTimeWindowConfig, SQLCommonConfig
     ignore_start_time_lineage: bool = False
     upstream_lineage_in_report: bool = False
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def validate_include_view_lineage(cls, values):
         if (
             "include_table_lineage" in values
diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py
index 6037490acb267..5eb9c83236e4f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py
@@ -44,7 +44,7 @@ class BigQueryCredential(ConfigModel):
         description="If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email",
     )
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def validate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
         if values.get("client_x509_cert_url") is None:
             values[
diff --git a/metadata-ingestion/src/datahub/upgrade/upgrade.py b/metadata-ingestion/src/datahub/upgrade/upgrade.py
index acc7954ad25a6..075bfd29008f6 100644
--- a/metadata-ingestion/src/datahub/upgrade/upgrade.py
+++ b/metadata-ingestion/src/datahub/upgrade/upgrade.py
@@ -23,18 +23,18 @@
 
 class VersionStats(BaseModel, arbitrary_types_allowed=True):
     version: Version
-    release_date: Optional[datetime]
+    release_date: Optional[datetime] = None
 
 
 class ServerVersionStats(BaseModel):
     current: VersionStats
-    latest: Optional[VersionStats]
-    current_server_type: Optional[str]
+    latest: Optional[VersionStats] = None
+    current_server_type: Optional[str] = None
 
 
 class ClientVersionStats(BaseModel):
     current: VersionStats
-    latest: Optional[VersionStats]
+    latest: Optional[VersionStats] = None
 
 
 class DataHubVersionStats(BaseModel):
diff --git a/metadata-ingestion/src/datahub/utilities/lossy_collections.py b/metadata-ingestion/src/datahub/utilities/lossy_collections.py
index f0c1e0da40552..0542a9dfd51f9 100644
--- a/metadata-ingestion/src/datahub/utilities/lossy_collections.py
+++ b/metadata-ingestion/src/datahub/utilities/lossy_collections.py
@@ -1,6 +1,8 @@
 import random
 from typing import Dict, Iterator, List, Set, TypeVar, Union
 
+from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2
+
 T = TypeVar("T")
 _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
@@ -41,6 +43,16 @@ def __repr__(self) -> str:
     def __str__(self) -> str:
         return repr(self)
 
+    if PYDANTIC_VERSION_2:
+        # With pydantic 2, it doesn't recognize that this is a list subclass,
+        # so we need to make it explicit.
+
+        @classmethod
+        def __get_pydantic_core_schema__(cls, source_type, handler):  # type: ignore
+            from pydantic_core import core_schema
+
+            return core_schema.no_info_after_validator_function(cls, handler(list))
+
     def as_obj(self) -> List[Union[T, str]]:
         base_list: List[Union[T, str]] = list(self.__iter__())
         if self.sampled:
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index 6413275ac63a6..cdffb684d958e 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -17,6 +17,7 @@
 from pydantic import BaseModel
 from typing_extensions import TypedDict
 
+from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2
 from datahub.emitter.mce_builder import (
     DEFAULT_ENV,
     make_dataset_urn_with_platform_instance,
@@ -122,12 +123,17 @@ class _ParserBaseModel(
         SchemaFieldDataTypeClass: lambda v: v.to_obj(),
     },
 ):
-    pass
+    def json(self, *args: Any, **kwargs: Any) -> str:
+        if PYDANTIC_VERSION_2:
+            return super().model_dump_json(*args, **kwargs)  # type: ignore
+        else:
+            return super().json(*args, **kwargs)
 
 
 @functools.total_ordering
 class _FrozenModel(_ParserBaseModel, frozen=True):
     def __lt__(self, other: "_FrozenModel") -> bool:
+        # TODO: The __fields__ attribute is deprecated in Pydantic v2.
         for field in self.__fields__:
             self_v = getattr(self, field)
             other_v = getattr(other, field)
@@ -138,8 +144,8 @@ def __lt__(self, other: "_FrozenModel") -> bool:
 
 
 class _TableName(_FrozenModel):
-    database: Optional[str]
-    db_schema: Optional[str]
+    database: Optional[str] = None
+    db_schema: Optional[str] = None
     table: str
 
     def as_sqlglot_table(self) -> sqlglot.exp.Table:
@@ -187,16 +193,16 @@ class ColumnRef(_ParserBaseModel):
 
 
 class _DownstreamColumnRef(_ParserBaseModel):
-    table: Optional[_TableName]
+    table: Optional[_TableName] = None
     column: str
-    column_type: Optional[sqlglot.exp.DataType]
+    column_type: Optional[sqlglot.exp.DataType] = None
 
 
 class DownstreamColumnRef(_ParserBaseModel):
-    table: Optional[Urn]
+    table: Optional[Urn] = None
     column: str
-    column_type: Optional[SchemaFieldDataTypeClass]
-    native_column_type: Optional[str]
+    column_type: Optional[SchemaFieldDataTypeClass] = None
+    native_column_type: Optional[str] = None
 
     @pydantic.validator("column_type", pre=True)
     def _load_column_type(
@@ -213,7 +219,7 @@ class _ColumnLineageInfo(_ParserBaseModel):
     downstream: _DownstreamColumnRef
     upstreams: List[_ColumnRef]
 
-    logic: Optional[str]
+    logic: Optional[str] = None
 
 
 class ColumnLineageInfo(_ParserBaseModel):
@@ -244,7 +250,7 @@ class SqlParsingResult(_ParserBaseModel):
     in_tables: List[Urn]
     out_tables: List[Urn]
 
-    column_lineage: Optional[List[ColumnLineageInfo]]
+    column_lineage: Optional[List[ColumnLineageInfo]] = None
 
     # TODO include formatted original sql logic
     # TODO include list of referenced columns

From a187127ac5e5a3aebd9ef217e3facadc159f59fa Mon Sep 17 00:00:00 2001
From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com>
Date: Sat, 11 Nov 2023 04:06:00 +0530
Subject: [PATCH 194/259] feat(ingestion): file-based state checkpoint provider
 (#9029)

---
 metadata-ingestion/setup.py                   |   1 +
 ...gestion_job_checkpointing_provider_base.py |   2 +-
 .../src/datahub/ingestion/graph/client.py     |   2 +-
 .../source/state/stateful_ingestion_base.py   |  17 +-
 ...atahub_ingestion_checkpointing_provider.py |  27 +-
 .../file_ingestion_checkpointing_provider.py  | 108 +++
 .../integration/lookml/golden_test_state.json |  26 +
 .../lookml_mces_golden_deleted_stateful.json  | 650 ------------------
 .../tests/integration/lookml/test_lookml.py   | 116 +---
 ...atahub_ingestion_checkpointing_provider.py | 170 -----
 .../provider/test_provider.py                 | 183 +++++
 .../state/golden_test_checkpoint_state.json   |  26 +
 ...n_test_checkpoint_state_after_deleted.json |  26 +
 .../state/golden_test_stateful_ingestion.json |  50 ++
 ...test_stateful_ingestion_after_deleted.json |  50 ++
 .../state/test_stateful_ingestion.py          | 227 ++++++
 .../unit/stateful_ingestion/test_configs.py   |  15 +-
 17 files changed, 739 insertions(+), 957 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py
 create mode 100644 metadata-ingestion/tests/integration/lookml/golden_test_state.json
 delete mode 100644 metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json
 delete mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/provider/test_datahub_ingestion_checkpointing_provider.py
 create mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/provider/test_provider.py
 create mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json
 create mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json
 create mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json
 create mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json
 create mode 100644 metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 2392fce058061..5f44f14c3d74c 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -666,6 +666,7 @@
     ],
     "datahub.ingestion.checkpointing_provider.plugins": [
         "datahub = datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider:DatahubIngestionCheckpointingProvider",
+        "file = datahub.ingestion.source.state_provider.file_ingestion_checkpointing_provider:FileIngestionCheckpointingProvider",
     ],
     "datahub.ingestion.reporting_provider.plugins": [
         "datahub = datahub.ingestion.reporting.datahub_ingestion_run_summary_provider:DatahubIngestionRunSummaryProvider",
diff --git a/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py b/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py
index ca02b88ab6324..285ad9c088447 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py
@@ -35,7 +35,7 @@ def __init__(
     @classmethod
     @abstractmethod
     def create(
-        cls: Type[_Self], config_dict: Dict[str, Any], ctx: PipelineContext, name: str
+        cls: Type[_Self], config_dict: Dict[str, Any], ctx: PipelineContext
     ) -> "_Self":
         pass
 
diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py
index ccff677c3a471..d91165ac9777c 100644
--- a/metadata-ingestion/src/datahub/ingestion/graph/client.py
+++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py
@@ -756,7 +756,7 @@ def get_latest_pipeline_checkpoint(
             DatahubIngestionCheckpointingProvider,
         )
 
-        checkpoint_provider = DatahubIngestionCheckpointingProvider(self, "graph")
+        checkpoint_provider = DatahubIngestionCheckpointingProvider(self)
         job_name = StaleEntityRemovalHandler.compute_job_id(platform)
 
         raw_checkpoint = checkpoint_provider.get_latest_checkpoint(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
index b1d2b276130a9..8a448f40e95b4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py
@@ -1,6 +1,6 @@
 import logging
 from dataclasses import dataclass
-from typing import Any, Dict, Generic, Optional, Type, TypeVar, cast
+from typing import Any, Dict, Generic, Optional, Type, TypeVar
 
 import pydantic
 from pydantic import root_validator
@@ -39,10 +39,8 @@ class DynamicTypedStateProviderConfig(DynamicTypedConfig):
     type: str = Field(
         description="The type of the state provider to use. For DataHub use `datahub`",
     )
-    # This config type is declared Optional[Any] here. The eventual parser for the
-    # specified type is responsible for further validation.
-    config: Optional[Any] = Field(
-        default=None,
+    config: Dict[str, Any] = Field(
+        default={},
         description="The configuration required for initializing the state provider. Default: The datahub_api config if set at pipeline level. Otherwise, the default DatahubClientConfig. See the defaults (https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/graph/client.py#L19).",
     )
 
@@ -82,7 +80,7 @@ def validate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
         if values.get("enabled"):
             if values.get("state_provider") is None:
                 values["state_provider"] = DynamicTypedStateProviderConfig(
-                    type="datahub", config=None
+                    type="datahub", config={}
                 )
         return values
 
@@ -252,15 +250,10 @@ def _initialize_checkpointing_state_provider(self) -> None:
                     f"Cannot find checkpoint provider class of type={self.stateful_ingestion_config.state_provider.type} "
                     " in the registry! Please check the type of the checkpointing provider in your config."
                 )
-            config_dict: Dict[str, Any] = cast(
-                Dict[str, Any],
-                self.stateful_ingestion_config.state_provider.dict().get("config", {}),
-            )
             self.ingestion_checkpointing_state_provider = (
                 checkpointing_state_provider_class.create(
-                    config_dict=config_dict,
+                    config_dict=self.stateful_ingestion_config.state_provider.config,
                     ctx=self.ctx,
-                    name=checkpointing_state_provider_class.__name__,
                 )
             )
             assert self.ingestion_checkpointing_state_provider
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py
index d7ebcba2c6695..442abb3aaf4cf 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py
@@ -17,14 +17,17 @@
 
 
 class DatahubIngestionStateProviderConfig(IngestionCheckpointingProviderConfig):
-    datahub_api: Optional[DatahubClientConfig] = DatahubClientConfig()
+    datahub_api: DatahubClientConfig = DatahubClientConfig()
 
 
 class DatahubIngestionCheckpointingProvider(IngestionCheckpointingProviderBase):
     orchestrator_name: str = "datahub"
 
-    def __init__(self, graph: DataHubGraph, name: str):
-        super().__init__(name)
+    def __init__(
+        self,
+        graph: DataHubGraph,
+    ):
+        super().__init__(self.__class__.__name__)
         self.graph = graph
         if not self._is_server_stateful_ingestion_capable():
             raise ConfigurationError(
@@ -34,24 +37,14 @@ def __init__(self, graph: DataHubGraph, name: str):
 
     @classmethod
     def create(
-        cls, config_dict: Dict[str, Any], ctx: PipelineContext, name: str
+        cls, config_dict: Dict[str, Any], ctx: PipelineContext
     ) -> "DatahubIngestionCheckpointingProvider":
+        config = DatahubIngestionStateProviderConfig.parse_obj(config_dict)
         if ctx.graph:
             # Use the pipeline-level graph if set
-            return cls(ctx.graph, name)
-        elif config_dict is None:
-            raise ConfigurationError("Missing provider configuration.")
+            return cls(ctx.graph)
         else:
-            provider_config = (
-                DatahubIngestionStateProviderConfig.parse_obj_allow_extras(config_dict)
-            )
-            if provider_config.datahub_api:
-                graph = DataHubGraph(provider_config.datahub_api)
-                return cls(graph, name)
-            else:
-                raise ConfigurationError(
-                    "Missing datahub_api. Provide either a global one or under the state_provider."
-                )
+            return cls(DataHubGraph(config.datahub_api))
 
     def _is_server_stateful_ingestion_capable(self) -> bool:
         server_config = self.graph.get_config() if self.graph else None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py b/metadata-ingestion/src/datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py
new file mode 100644
index 0000000000000..a37774773b84d
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py
@@ -0,0 +1,108 @@
+import logging
+import pathlib
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import (
+    IngestionCheckpointingProviderBase,
+    IngestionCheckpointingProviderConfig,
+    JobId,
+)
+from datahub.ingestion.sink.file import write_metadata_file
+from datahub.ingestion.source.file import read_metadata_file
+from datahub.metadata.schema_classes import DatahubIngestionCheckpointClass
+
+logger = logging.getLogger(__name__)
+
+
+class FileIngestionStateProviderConfig(IngestionCheckpointingProviderConfig):
+    filename: str
+
+
+class FileIngestionCheckpointingProvider(IngestionCheckpointingProviderBase):
+    orchestrator_name: str = "file"
+
+    def __init__(self, config: FileIngestionStateProviderConfig):
+        super().__init__(self.__class__.__name__)
+        self.config = config
+
+    @classmethod
+    def create(
+        cls, config_dict: Dict[str, Any], ctx: PipelineContext
+    ) -> "FileIngestionCheckpointingProvider":
+        config = FileIngestionStateProviderConfig.parse_obj(config_dict)
+        return cls(config)
+
+    def get_latest_checkpoint(
+        self,
+        pipeline_name: str,
+        job_name: JobId,
+    ) -> Optional[DatahubIngestionCheckpointClass]:
+        logger.debug(
+            f"Querying for the latest ingestion checkpoint for pipelineName:'{pipeline_name}',"
+            f" job_name:'{job_name}'"
+        )
+
+        data_job_urn = self.get_data_job_urn(
+            self.orchestrator_name, pipeline_name, job_name
+        )
+        latest_checkpoint: Optional[DatahubIngestionCheckpointClass] = None
+        try:
+            for obj in read_metadata_file(pathlib.Path(self.config.filename)):
+                if (
+                    isinstance(obj, MetadataChangeProposalWrapper)
+                    and obj.entityUrn == data_job_urn
+                    and obj.aspect
+                    and isinstance(obj.aspect, DatahubIngestionCheckpointClass)
+                    and obj.aspect.get("pipelineName", "") == pipeline_name
+                ):
+                    latest_checkpoint = obj.aspect
+                    break
+        except FileNotFoundError:
+            logger.debug(f"File {self.config.filename} not found")
+
+        if latest_checkpoint:
+            logger.debug(
+                f"The last committed ingestion checkpoint for pipelineName:'{pipeline_name}',"
+                f" job_name:'{job_name}' found with start_time:"
+                f" {datetime.utcfromtimestamp(latest_checkpoint.timestampMillis/1000)}"
+            )
+            return latest_checkpoint
+        else:
+            logger.debug(
+                f"No committed ingestion checkpoint for pipelineName:'{pipeline_name}',"
+                f" job_name:'{job_name}' found"
+            )
+
+        return None
+
+    def commit(self) -> None:
+        if not self.state_to_commit:
+            logger.warning(f"No state available to commit for {self.name}")
+            return None
+
+        checkpoint_workunits: List[MetadataChangeProposalWrapper] = []
+        for job_name, checkpoint in self.state_to_commit.items():
+            # Emit the ingestion state for each job
+            logger.debug(
+                f"Committing ingestion checkpoint for pipeline:'{checkpoint.pipelineName}', "
+                f"job:'{job_name}'"
+            )
+            datajob_urn = self.get_data_job_urn(
+                self.orchestrator_name,
+                checkpoint.pipelineName,
+                job_name,
+            )
+            checkpoint_workunits.append(
+                MetadataChangeProposalWrapper(
+                    entityUrn=datajob_urn,
+                    aspect=checkpoint,
+                )
+            )
+        write_metadata_file(pathlib.Path(self.config.filename), checkpoint_workunits)
+        self.committed = True
+        logger.debug(
+            f"Committed all ingestion checkpoints for pipeline:'{checkpoint.pipelineName}'"
+        )
diff --git a/metadata-ingestion/tests/integration/lookml/golden_test_state.json b/metadata-ingestion/tests/integration/lookml/golden_test_state.json
new file mode 100644
index 0000000000000..c62106ac10089
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/golden_test_state.json
@@ -0,0 +1,26 @@
+[
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(file,lookml_stateful,prod),lookml_stale_entity_removal)",
+    "changeType": "UPSERT",
+    "aspectName": "datahubIngestionCheckpoint",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1586847600000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "pipelineName": "lookml_stateful",
+            "platformInstanceId": "",
+            "config": "",
+            "state": {
+                "formatVersion": "1.0",
+                "serde": "base85-bz2-json",
+                "payload": "LRx4!F+o`-Q(4)<4JiNuUmt)_WdINa0@Mn>@BivB0a-v1sF;Ar&}h&A0K-EjK*+=xnKU%Oib;?JVrrXB7?aRqCarWwpZm8v5Yh+DsN{|c*msMh9%WJXjKPvIPsDn^@g3;DD9Q9kBh?*|=8M4uRW$_0HKn3XhN;RhAcLIBhLnO2%UA@Ykl;h&Xx(^@2;Y9C#d4g3K_2CA-I*M)h{NMA8Nu4C3XjEQYdh{nR--&lfRUsTL}OOkOO435f=1nKzYJ^9)mbBljM0}gaqy26URw1=q<80Eb9y)y?Vl88kG;g~MToq#r%6tr<yx^i_E#v)8~7vUJum>K9U`U?k}RS<@^?i@<c?y}RaZG9JGf09m`0f!sz%!^wDYcoJR{ix%d2rWCL+XvG>1M1@9*%tk}1N3hRzUaNB"
+            },
+            "runId": "lookml-test"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json
deleted file mode 100644
index a323118666940..0000000000000
--- a/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json
+++ /dev/null
@@ -1,650 +0,0 @@
-[
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/prod/looker/lkml_samples/views"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..my_table,PROD)",
-                                "type": "VIEW"
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "my_view",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "country",
-                                "nullable": false,
-                                "description": "The country",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "city",
-                                "nullable": false,
-                                "description": "City",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "is_latest",
-                                "nullable": false,
-                                "description": "Is latest data",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
-                                    }
-                                },
-                                "nativeDataType": "yesno",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "timestamp",
-                                "nullable": false,
-                                "description": "Timestamp of measurement",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
-                                    }
-                                },
-                                "nativeDataType": "time",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        },
-                                        {
-                                            "tag": "urn:li:tag:Temporal"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            },
-                            {
-                                "fieldPath": "average_measurement",
-                                "nullable": false,
-                                "description": "My measurement",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
-                                    }
-                                },
-                                "nativeDataType": "average",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Measure"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": []
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "foo.view.lkml"
-                        },
-                        "name": "my_view",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "SELECT\n          is_latest,\n          country,\n          city,\n          timestamp,\n          measurement\n        FROM\n          my_table",
-            "viewLanguage": "sql"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "looker"
-                },
-                {
-                    "id": "lkml_samples"
-                },
-                {
-                    "id": "views"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "proposedSnapshot": {
-        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
-            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-            "aspects": [
-                {
-                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
-                        "paths": [
-                            "/prod/looker/lkml_samples/views"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.common.Status": {
-                        "removed": false
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
-                        "upstreams": [
-                            {
-                                "auditStamp": {
-                                    "time": 1586847600000,
-                                    "actor": "urn:li:corpuser:datahub"
-                                },
-                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD)",
-                                "type": "VIEW"
-                            }
-                        ],
-                        "fineGrainedLineages": [
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),id)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD),id)"
-                                ],
-                                "confidenceScore": 1.0
-                            },
-                            {
-                                "upstreamType": "FIELD_SET",
-                                "upstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),owner_name)"
-                                ],
-                                "downstreamType": "FIELD",
-                                "downstreams": [
-                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD),owner_name)"
-                                ],
-                                "confidenceScore": 1.0
-                            }
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
-                        "schemaName": "owners",
-                        "platform": "urn:li:dataPlatform:looker",
-                        "version": 0,
-                        "created": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "lastModified": {
-                            "time": 0,
-                            "actor": "urn:li:corpuser:unknown"
-                        },
-                        "hash": "",
-                        "platformSchema": {
-                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
-                                "rawSchema": ""
-                            }
-                        },
-                        "fields": [
-                            {
-                                "fieldPath": "id",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": true
-                            },
-                            {
-                                "fieldPath": "owner_name",
-                                "nullable": false,
-                                "description": "",
-                                "label": "",
-                                "type": {
-                                    "type": {
-                                        "com.linkedin.pegasus2avro.schema.StringType": {}
-                                    }
-                                },
-                                "nativeDataType": "string",
-                                "recursive": false,
-                                "globalTags": {
-                                    "tags": [
-                                        {
-                                            "tag": "urn:li:tag:Dimension"
-                                        }
-                                    ]
-                                },
-                                "isPartOfKey": false
-                            }
-                        ],
-                        "primaryKeys": [
-                            "id"
-                        ]
-                    }
-                },
-                {
-                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
-                        "customProperties": {
-                            "looker.file.path": "owners.view.lkml"
-                        },
-                        "name": "owners",
-                        "tags": []
-                    }
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "subTypes",
-    "aspect": {
-        "json": {
-            "typeNames": [
-                "View"
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "viewProperties",
-    "aspect": {
-        "json": {
-            "materialized": false,
-            "viewLogic": "view: owners {\n  dimension: id {\n    primary_key: yes\n    sql: ${TABLE}.id ;;\n  }\n  dimension: owner_name {\n    sql: ${TABLE}.owner_name ;;\n  }\n}",
-            "viewLanguage": "lookml"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "browsePathsV2",
-    "aspect": {
-        "json": {
-            "path": [
-                {
-                    "id": "looker"
-                },
-                {
-                    "id": "lkml_samples"
-                },
-                {
-                    "id": "views"
-                }
-            ]
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Dimension",
-    "changeType": "UPSERT",
-    "aspectName": "tagKey",
-    "aspect": {
-        "json": {
-            "name": "Dimension"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Measure",
-    "changeType": "UPSERT",
-    "aspectName": "tagKey",
-    "aspect": {
-        "json": {
-            "name": "Measure"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "tag",
-    "entityUrn": "urn:li:tag:Temporal",
-    "changeType": "UPSERT",
-    "aspectName": "tagKey",
-    "aspect": {
-        "json": {
-            "name": "Temporal"
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": true
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1586847600000,
-        "runId": "lookml-test"
-    }
-}
-]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py
index 21a0b19849d97..b1853cfa2b3c0 100644
--- a/metadata-ingestion/tests/integration/lookml/test_lookml.py
+++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py
@@ -1,6 +1,6 @@
 import logging
 import pathlib
-from typing import Any, Dict, List, cast
+from typing import Any, List
 from unittest import mock
 
 import pydantic
@@ -17,17 +17,13 @@
     LookerRefinementResolver,
     LookMLSourceConfig,
 )
-from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState
 from datahub.metadata.schema_classes import (
     DatasetSnapshotClass,
     MetadataChangeEventClass,
     UpstreamLineageClass,
 )
 from tests.test_helpers import mce_helpers
-from tests.test_helpers.state_helpers import (
-    get_current_checkpoint_from_pipeline,
-    validate_all_providers_have_committed_successfully,
-)
+from tests.test_helpers.state_helpers import get_current_checkpoint_from_pipeline
 
 logging.getLogger("lkml").setLevel(logging.INFO)
 
@@ -728,11 +724,10 @@ def test_hive_platform_drops_ids(pytestconfig, tmp_path, mock_time):
 
 
 @freeze_time(FROZEN_TIME)
-def test_lookml_ingest_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
+def test_lookml_stateful_ingestion(pytestconfig, tmp_path, mock_time):
     output_file_name: str = "lookml_mces.json"
-    golden_file_name: str = "expected_output.json"
-    output_file_deleted_name: str = "lookml_mces_deleted_stateful.json"
-    golden_file_deleted_name: str = "lookml_mces_golden_deleted_stateful.json"
+    state_file_name: str = "lookml_state_mces.json"
+    golden_file_name: str = "golden_test_state.json"
 
     test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
 
@@ -754,106 +749,37 @@ def test_lookml_ingest_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_
                     "remove_stale_metadata": True,
                     "fail_safe_threshold": 100.0,
                     "state_provider": {
-                        "type": "datahub",
-                        "config": {"datahub_api": {"server": GMS_SERVER}},
+                        "type": "file",
+                        "config": {
+                            "filename": f"{tmp_path}/{state_file_name}",
+                        },
                     },
                 },
             },
         },
         "sink": {
             "type": "file",
-            "config": {},
+            "config": {
+                "filename": f"{tmp_path}/{output_file_name}",
+            },
         },
     }
 
-    pipeline_run1 = None
-    with mock.patch(
-        "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
-        mock_datahub_graph,
-    ) as mock_checkpoint:
-        mock_checkpoint.return_value = mock_datahub_graph
-        pipeline_run1_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(  # type: ignore
-            base_pipeline_config  # type: ignore
-        )
-        # Set the special properties for this run
-        pipeline_run1_config["source"]["config"]["emit_reachable_views_only"] = False
-        pipeline_run1_config["sink"]["config"][
-            "filename"
-        ] = f"{tmp_path}/{output_file_name}"
-        pipeline_run1 = Pipeline.create(pipeline_run1_config)
-        pipeline_run1.run()
-        pipeline_run1.raise_from_status()
-        pipeline_run1.pretty_print_summary()
+    pipeline_run1 = Pipeline.create(base_pipeline_config)
+    pipeline_run1.run()
+    pipeline_run1.raise_from_status()
+    pipeline_run1.pretty_print_summary()
 
-        mce_helpers.check_golden_file(
-            pytestconfig,
-            output_path=tmp_path / output_file_name,
-            golden_path=f"{test_resources_dir}/{golden_file_name}",
-        )
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=f"{tmp_path}/{state_file_name}",
+        golden_path=f"{test_resources_dir}/{golden_file_name}",
+    )
 
     checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1)
     assert checkpoint1
     assert checkpoint1.state
 
-    pipeline_run2 = None
-    with mock.patch(
-        "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
-        mock_datahub_graph,
-    ) as mock_checkpoint:
-        mock_checkpoint.return_value = mock_datahub_graph
-        pipeline_run2_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(base_pipeline_config)  # type: ignore
-        # Set the special properties for this run
-        pipeline_run2_config["source"]["config"]["emit_reachable_views_only"] = True
-        pipeline_run2_config["sink"]["config"][
-            "filename"
-        ] = f"{tmp_path}/{output_file_deleted_name}"
-        pipeline_run2 = Pipeline.create(pipeline_run2_config)
-        pipeline_run2.run()
-        pipeline_run2.raise_from_status()
-        pipeline_run2.pretty_print_summary()
-
-        mce_helpers.check_golden_file(
-            pytestconfig,
-            output_path=tmp_path / output_file_deleted_name,
-            golden_path=f"{test_resources_dir}/{golden_file_deleted_name}",
-        )
-    checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2)
-    assert checkpoint2
-    assert checkpoint2.state
-
-    # Validate that all providers have committed successfully.
-    validate_all_providers_have_committed_successfully(
-        pipeline=pipeline_run1, expected_providers=1
-    )
-    validate_all_providers_have_committed_successfully(
-        pipeline=pipeline_run2, expected_providers=1
-    )
-
-    # Perform all assertions on the states. The deleted table should not be
-    # part of the second state
-    state1 = cast(GenericCheckpointState, checkpoint1.state)
-    state2 = cast(GenericCheckpointState, checkpoint2.state)
-
-    difference_dataset_urns = list(
-        state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2)
-    )
-    # the difference in dataset urns are all the views that are not reachable from the model file
-    assert len(difference_dataset_urns) == 11
-    deleted_dataset_urns: List[str] = [
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)",
-        "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)",
-    ]
-    assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns)
-
 
 def test_lookml_base_folder():
     fake_api = {
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/provider/test_datahub_ingestion_checkpointing_provider.py b/metadata-ingestion/tests/unit/stateful_ingestion/provider/test_datahub_ingestion_checkpointing_provider.py
deleted file mode 100644
index 600985266043b..0000000000000
--- a/metadata-ingestion/tests/unit/stateful_ingestion/provider/test_datahub_ingestion_checkpointing_provider.py
+++ /dev/null
@@ -1,170 +0,0 @@
-import types
-import unittest
-from typing import Dict, List, Optional, Type
-from unittest.mock import MagicMock, patch
-
-from avrogen.dict_wrapper import DictWrapper
-
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import (
-    CheckpointJobStateType,
-    JobId,
-)
-from datahub.ingestion.source.state.checkpoint import Checkpoint
-from datahub.ingestion.source.state.sql_common_state import (
-    BaseSQLAlchemyCheckpointState,
-)
-from datahub.ingestion.source.state.usage_common_state import (
-    BaseTimeWindowCheckpointState,
-)
-from datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider import (
-    DatahubIngestionCheckpointingProvider,
-)
-from tests.test_helpers.type_helpers import assert_not_null
-
-
-class TestDatahubIngestionCheckpointProvider(unittest.TestCase):
-    # Static members for the tests
-    pipeline_name: str = "test_pipeline"
-    job_names: List[JobId] = [JobId("job1"), JobId("job2")]
-    run_id: str = "test_run"
-
-    def setUp(self) -> None:
-        self._setup_mock_graph()
-        self.provider = self._create_provider()
-        assert self.provider
-
-    def _setup_mock_graph(self) -> None:
-        """
-        Setup monkey-patched graph client.
-        """
-        self.patcher = patch(
-            "datahub.ingestion.graph.client.DataHubGraph", autospec=True
-        )
-        self.addCleanup(self.patcher.stop)
-        self.mock_graph = self.patcher.start()
-        # Make server stateful ingestion capable
-        self.mock_graph.get_config.return_value = {"statefulIngestionCapable": True}
-        # Bind mock_graph's emit_mcp to testcase's monkey_patch_emit_mcp so that we can emulate emits.
-        self.mock_graph.emit_mcp = types.MethodType(
-            self.monkey_patch_emit_mcp, self.mock_graph
-        )
-        # Bind mock_graph's get_latest_timeseries_value to monkey_patch_get_latest_timeseries_value
-        self.mock_graph.get_latest_timeseries_value = types.MethodType(
-            self.monkey_patch_get_latest_timeseries_value, self.mock_graph
-        )
-        # Tracking for emitted mcps.
-        self.mcps_emitted: Dict[str, MetadataChangeProposalWrapper] = {}
-
-    def _create_provider(self) -> DatahubIngestionCheckpointingProvider:
-        ctx: PipelineContext = PipelineContext(
-            run_id=self.run_id, pipeline_name=self.pipeline_name
-        )
-        ctx.graph = self.mock_graph
-        return DatahubIngestionCheckpointingProvider.create(
-            {}, ctx, name=DatahubIngestionCheckpointingProvider.__name__
-        )
-
-    def monkey_patch_emit_mcp(
-        self, graph_ref: MagicMock, mcpw: MetadataChangeProposalWrapper
-    ) -> None:
-        """
-        Mockey patched implementation of DatahubGraph.emit_mcp that caches the mcp locally in memory.
-        """
-        self.assertIsNotNone(graph_ref)
-        if mcpw.aspectName != "status":
-            self.assertEqual(mcpw.entityType, "dataJob")
-            self.assertEqual(mcpw.aspectName, "datahubIngestionCheckpoint")
-        # Cache the mcpw against the entityUrn
-        assert mcpw.entityUrn is not None
-        self.mcps_emitted[mcpw.entityUrn] = mcpw
-
-    def monkey_patch_get_latest_timeseries_value(
-        self,
-        graph_ref: MagicMock,
-        entity_urn: str,
-        aspect_type: Type[DictWrapper],
-        filter_criteria_map: Dict[str, str],
-    ) -> Optional[DictWrapper]:
-        """
-        Monkey patched implementation of DatahubGraph.get_latest_timeseries_value that returns the latest cached aspect
-        for a given entity urn.
-        """
-        self.assertIsNotNone(graph_ref)
-        self.assertEqual(aspect_type, CheckpointJobStateType)
-        self.assertEqual(
-            filter_criteria_map,
-            {
-                "pipelineName": self.pipeline_name,
-            },
-        )
-        # Retrieve the cached mcpw and return its aspect value.
-        mcpw = self.mcps_emitted.get(entity_urn)
-        if mcpw:
-            return mcpw.aspect
-        return None
-
-    def test_provider(self):
-        # 1. Create the individual job checkpoints with appropriate states.
-        # Job1 - Checkpoint with a BaseSQLAlchemyCheckpointState state
-        job1_state_obj = BaseSQLAlchemyCheckpointState()
-        job1_checkpoint = Checkpoint(
-            job_name=self.job_names[0],
-            pipeline_name=self.pipeline_name,
-            run_id=self.run_id,
-            state=job1_state_obj,
-        )
-        # Job2 - Checkpoint with a BaseTimeWindowCheckpointState state
-        job2_state_obj = BaseTimeWindowCheckpointState(
-            begin_timestamp_millis=10, end_timestamp_millis=100
-        )
-        job2_checkpoint = Checkpoint(
-            job_name=self.job_names[1],
-            pipeline_name=self.pipeline_name,
-            run_id=self.run_id,
-            state=job2_state_obj,
-        )
-
-        # 2. Set the provider's state_to_commit.
-        self.provider.state_to_commit = {
-            # NOTE: state_to_commit accepts only the aspect version of the checkpoint.
-            self.job_names[0]: assert_not_null(
-                job1_checkpoint.to_checkpoint_aspect(max_allowed_state_size=2**20)
-            ),
-            self.job_names[1]: assert_not_null(
-                job2_checkpoint.to_checkpoint_aspect(max_allowed_state_size=2**20)
-            ),
-        }
-
-        # 3. Perform the commit
-        # NOTE: This will commit the state to the in-memory self.mcps_emitted because of the monkey-patching.
-        self.provider.commit()
-        self.assertTrue(self.provider.committed)
-
-        # 4. Get last committed state. This must match what has been committed earlier.
-        # NOTE: This will retrieve from in-memory self.mcps_emitted because of the monkey-patching.
-        job1_last_state = self.provider.get_latest_checkpoint(
-            self.pipeline_name, self.job_names[0]
-        )
-        job2_last_state = self.provider.get_latest_checkpoint(
-            self.pipeline_name, self.job_names[1]
-        )
-
-        # 5. Validate individual job checkpoint state values that have been committed and retrieved
-        # against the original values.
-        self.assertIsNotNone(job1_last_state)
-        job1_last_checkpoint = Checkpoint.create_from_checkpoint_aspect(
-            job_name=self.job_names[0],
-            checkpoint_aspect=job1_last_state,
-            state_class=type(job1_state_obj),
-        )
-        self.assertEqual(job1_last_checkpoint, job1_checkpoint)
-
-        self.assertIsNotNone(job2_last_state)
-        job2_last_checkpoint = Checkpoint.create_from_checkpoint_aspect(
-            job_name=self.job_names[1],
-            checkpoint_aspect=job2_last_state,
-            state_class=type(job2_state_obj),
-        )
-        self.assertEqual(job2_last_checkpoint, job2_checkpoint)
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/provider/test_provider.py b/metadata-ingestion/tests/unit/stateful_ingestion/provider/test_provider.py
new file mode 100644
index 0000000000000..4387e5a17790f
--- /dev/null
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/provider/test_provider.py
@@ -0,0 +1,183 @@
+import tempfile
+import types
+import unittest
+from typing import Dict, List, Optional, Type
+from unittest.mock import MagicMock, patch
+
+from avrogen.dict_wrapper import DictWrapper
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import (
+    CheckpointJobStateType,
+    IngestionCheckpointingProviderBase,
+    JobId,
+)
+from datahub.ingestion.source.state.checkpoint import Checkpoint
+from datahub.ingestion.source.state.sql_common_state import (
+    BaseSQLAlchemyCheckpointState,
+)
+from datahub.ingestion.source.state.usage_common_state import (
+    BaseTimeWindowCheckpointState,
+)
+from datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider import (
+    DatahubIngestionCheckpointingProvider,
+)
+from datahub.ingestion.source.state_provider.file_ingestion_checkpointing_provider import (
+    FileIngestionCheckpointingProvider,
+)
+from tests.test_helpers.type_helpers import assert_not_null
+
+
+class TestIngestionCheckpointProviders(unittest.TestCase):
+    # Static members for the tests
+    pipeline_name: str = "test_pipeline"
+    job_names: List[JobId] = [JobId("job1"), JobId("job2")]
+    run_id: str = "test_run"
+
+    def setUp(self) -> None:
+        self._setup_mock_graph()
+        self._create_providers()
+
+    def _setup_mock_graph(self) -> None:
+        """
+        Setup monkey-patched graph client.
+        """
+        self.patcher = patch(
+            "datahub.ingestion.graph.client.DataHubGraph", autospec=True
+        )
+        self.addCleanup(self.patcher.stop)
+        self.mock_graph = self.patcher.start()
+        # Make server stateful ingestion capable
+        self.mock_graph.get_config.return_value = {"statefulIngestionCapable": True}
+        # Bind mock_graph's emit_mcp to testcase's monkey_patch_emit_mcp so that we can emulate emits.
+        self.mock_graph.emit_mcp = types.MethodType(
+            self.monkey_patch_emit_mcp, self.mock_graph
+        )
+        # Bind mock_graph's get_latest_timeseries_value to monkey_patch_get_latest_timeseries_value
+        self.mock_graph.get_latest_timeseries_value = types.MethodType(
+            self.monkey_patch_get_latest_timeseries_value, self.mock_graph
+        )
+        # Tracking for emitted mcps.
+        self.mcps_emitted: Dict[str, MetadataChangeProposalWrapper] = {}
+
+    def _create_providers(self) -> None:
+        ctx: PipelineContext = PipelineContext(
+            run_id=self.run_id, pipeline_name=self.pipeline_name
+        )
+        ctx.graph = self.mock_graph
+        self.providers: List[IngestionCheckpointingProviderBase] = [
+            DatahubIngestionCheckpointingProvider.create({}, ctx),
+            FileIngestionCheckpointingProvider.create(
+                {"filename": f"{tempfile.mkdtemp()}/checkpoint_mces.json"},
+                ctx,
+            ),
+        ]
+
+    def monkey_patch_emit_mcp(
+        self, graph_ref: MagicMock, mcpw: MetadataChangeProposalWrapper
+    ) -> None:
+        """
+        Mockey patched implementation of DatahubGraph.emit_mcp that caches the mcp locally in memory.
+        """
+        self.assertIsNotNone(graph_ref)
+        if mcpw.aspectName != "status":
+            self.assertEqual(mcpw.entityType, "dataJob")
+            self.assertEqual(mcpw.aspectName, "datahubIngestionCheckpoint")
+        # Cache the mcpw against the entityUrn
+        assert mcpw.entityUrn is not None
+        self.mcps_emitted[mcpw.entityUrn] = mcpw
+
+    def monkey_patch_get_latest_timeseries_value(
+        self,
+        graph_ref: MagicMock,
+        entity_urn: str,
+        aspect_type: Type[DictWrapper],
+        filter_criteria_map: Dict[str, str],
+    ) -> Optional[DictWrapper]:
+        """
+        Monkey patched implementation of DatahubGraph.get_latest_timeseries_value that returns the latest cached aspect
+        for a given entity urn.
+        """
+        self.assertIsNotNone(graph_ref)
+        self.assertEqual(aspect_type, CheckpointJobStateType)
+        self.assertEqual(
+            filter_criteria_map,
+            {
+                "pipelineName": self.pipeline_name,
+            },
+        )
+        # Retrieve the cached mcpw and return its aspect value.
+        mcpw = self.mcps_emitted.get(entity_urn)
+        if mcpw:
+            return mcpw.aspect
+        return None
+
+    def test_providers(self):
+        self.assertEqual(len(self.providers), 2)
+        for provider in self.providers:
+            assert provider
+            # 1. Create the individual job checkpoints with appropriate states.
+            # Job1 - Checkpoint with a BaseSQLAlchemyCheckpointState state
+            job1_state_obj = BaseSQLAlchemyCheckpointState()
+            job1_checkpoint = Checkpoint(
+                job_name=self.job_names[0],
+                pipeline_name=self.pipeline_name,
+                run_id=self.run_id,
+                state=job1_state_obj,
+            )
+            # Job2 - Checkpoint with a BaseTimeWindowCheckpointState state
+            job2_state_obj = BaseTimeWindowCheckpointState(
+                begin_timestamp_millis=10, end_timestamp_millis=100
+            )
+            job2_checkpoint = Checkpoint(
+                job_name=self.job_names[1],
+                pipeline_name=self.pipeline_name,
+                run_id=self.run_id,
+                state=job2_state_obj,
+            )
+
+            # 2. Set the provider's state_to_commit.
+            provider.state_to_commit = {
+                # NOTE: state_to_commit accepts only the aspect version of the checkpoint.
+                self.job_names[0]: assert_not_null(
+                    job1_checkpoint.to_checkpoint_aspect(max_allowed_state_size=2**20)
+                ),
+                self.job_names[1]: assert_not_null(
+                    job2_checkpoint.to_checkpoint_aspect(max_allowed_state_size=2**20)
+                ),
+            }
+
+            # 3. Perform the commit
+            # NOTE: This will commit the state to
+            # In-memory self.mcps_emitted because of the monkey-patching for datahub ingestion checkpointer provider.
+            # And to temp directory json file for file ingestion checkpointer provider.
+            provider.commit()
+            self.assertTrue(provider.committed)
+
+            # 4. Get last committed state. This must match what has been committed earlier.
+            # NOTE: This will retrieve the state form where it is committed.
+            job1_last_state = provider.get_latest_checkpoint(
+                self.pipeline_name, self.job_names[0]
+            )
+            job2_last_state = provider.get_latest_checkpoint(
+                self.pipeline_name, self.job_names[1]
+            )
+
+            # 5. Validate individual job checkpoint state values that have been committed and retrieved
+            # against the original values.
+            self.assertIsNotNone(job1_last_state)
+            job1_last_checkpoint = Checkpoint.create_from_checkpoint_aspect(
+                job_name=self.job_names[0],
+                checkpoint_aspect=job1_last_state,
+                state_class=type(job1_state_obj),
+            )
+            self.assertEqual(job1_last_checkpoint, job1_checkpoint)
+
+            self.assertIsNotNone(job2_last_state)
+            job2_last_checkpoint = Checkpoint.create_from_checkpoint_aspect(
+                job_name=self.job_names[1],
+                checkpoint_aspect=job2_last_state,
+                state_class=type(job2_state_obj),
+            )
+            self.assertEqual(job2_last_checkpoint, job2_checkpoint)
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json
new file mode 100644
index 0000000000000..4e62492918bfb
--- /dev/null
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json
@@ -0,0 +1,26 @@
+[
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(file,dummy_stateful,prod),default_stale_entity_removal)",
+    "changeType": "UPSERT",
+    "aspectName": "datahubIngestionCheckpoint",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1586847600000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "pipelineName": "dummy_stateful",
+            "platformInstanceId": "",
+            "config": "",
+            "state": {
+                "formatVersion": "1.0",
+                "serde": "base85-bz2-json",
+                "payload": "LRx4!F+o`-Q(1w>5G4QrYoCBnWH=B60MH7jr`{?c0BA?5L)2-AGyu>6y;V<9hz%Mv0Bt1*)lOMzr>a0|Iq-4VtTsYONQsFPLn1EpdQS;HIy|&CvSAlRvAJwmtCEM+Rx(v_)~sVvkx3V@WX4O`=losC6yZWb2OL0@"
+            },
+            "runId": "dummy-test-stateful-ingestion"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json
new file mode 100644
index 0000000000000..6ecd43483d948
--- /dev/null
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json
@@ -0,0 +1,26 @@
+[
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(file,dummy_stateful,prod),default_stale_entity_removal)",
+    "changeType": "UPSERT",
+    "aspectName": "datahubIngestionCheckpoint",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1586847600000,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "pipelineName": "dummy_stateful",
+            "platformInstanceId": "",
+            "config": "",
+            "state": {
+                "formatVersion": "1.0",
+                "serde": "base85-bz2-json",
+                "payload": "LRx4!F+o`-Q(317h`0a%NgsevWH1l}0MH7jr`{?c0B9vdZ9%mLfYG4P6;f$2G%+v`9z&~6n|e(JEPC2_Iix~CA_im)jR-zsjEK*yo|HQz#IUUHtf@DYVEme-lUW9{Xmmt~y^2jCdyY95az!{$kf#WUxB"
+            },
+            "runId": "dummy-test-stateful-ingestion"
+        }
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json
new file mode 100644
index 0000000000000..4a77651c93066
--- /dev/null
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json
@@ -0,0 +1,50 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "dummy-test-stateful-ingestion",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "dummy-test-stateful-ingestion",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "dummy-test-stateful-ingestion",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json
new file mode 100644
index 0000000000000..9d6f755374462
--- /dev/null
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json
@@ -0,0 +1,50 @@
+[
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "dummy-test-stateful-ingestion",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "dummy-test-stateful-ingestion",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": true
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "dummy-test-stateful-ingestion",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py
new file mode 100644
index 0000000000000..2b811d5e5e3a3
--- /dev/null
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py
@@ -0,0 +1,227 @@
+from dataclasses import dataclass, field as dataclass_field
+from typing import Any, Dict, Iterable, List, Optional, cast
+
+import pydantic
+from freezegun import freeze_time
+from pydantic import Field
+
+from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.run.pipeline import Pipeline
+from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+    StaleEntityRemovalSourceReport,
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+    StatefulIngestionSourceBase,
+)
+from datahub.metadata.schema_classes import StatusClass
+from datahub.utilities.urns.dataset_urn import DatasetUrn
+from tests.test_helpers import mce_helpers
+from tests.test_helpers.state_helpers import (
+    get_current_checkpoint_from_pipeline,
+    validate_all_providers_have_committed_successfully,
+)
+
+FROZEN_TIME = "2020-04-14 07:00:00"
+
+dummy_datasets: List = ["dummy_dataset1", "dummy_dataset2", "dummy_dataset3"]
+
+
+@dataclass
+class DummySourceReport(StaleEntityRemovalSourceReport):
+    datasets_scanned: int = 0
+    filtered_datasets: List[str] = dataclass_field(default_factory=list)
+
+    def report_datasets_scanned(self, count: int = 1) -> None:
+        self.datasets_scanned += count
+
+    def report_datasets_dropped(self, model: str) -> None:
+        self.filtered_datasets.append(model)
+
+
+class DummySourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
+    dataset_patterns: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for datasets to filter in ingestion.",
+    )
+    # Configuration for stateful ingestion
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field(
+        default=None, description="Dummy source Ingestion Config."
+    )
+
+
+class DummySource(StatefulIngestionSourceBase):
+    """
+    This is dummy source which only extract dummy datasets
+    """
+
+    source_config: DummySourceConfig
+    reporter: DummySourceReport
+
+    def __init__(self, config: DummySourceConfig, ctx: PipelineContext):
+        super(DummySource, self).__init__(config, ctx)
+        self.source_config = config
+        self.reporter = DummySourceReport()
+        # Create and register the stateful ingestion use-case handler.
+        self.stale_entity_removal_handler = StaleEntityRemovalHandler.create(
+            self, self.source_config, self.ctx
+        )
+
+    @classmethod
+    def create(cls, config_dict, ctx):
+        config = DummySourceConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            self.stale_entity_removal_handler.workunit_processor,
+        ]
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        for dataset in dummy_datasets:
+            if not self.source_config.dataset_patterns.allowed(dataset):
+                self.reporter.report_datasets_dropped(dataset)
+                continue
+            else:
+                self.reporter.report_datasets_scanned()
+            dataset_urn = DatasetUrn.create_from_ids(
+                platform_id="postgres",
+                table_name=dataset,
+                env=DEFAULT_ENV,
+            )
+            yield MetadataChangeProposalWrapper(
+                entityUrn=str(dataset_urn),
+                aspect=StatusClass(removed=False),
+            ).as_workunit()
+
+    def get_report(self) -> SourceReport:
+        return self.reporter
+
+
+@freeze_time(FROZEN_TIME)
+def test_stateful_ingestion(pytestconfig, tmp_path, mock_time):
+    # test stateful ingestion using dummy source
+    state_file_name: str = "checkpoint_state_mces.json"
+    golden_state_file_name: str = "golden_test_checkpoint_state.json"
+    golden_state_file_name_after_deleted: str = (
+        "golden_test_checkpoint_state_after_deleted.json"
+    )
+    output_file_name: str = "dummy_mces.json"
+    golden_file_name: str = "golden_test_stateful_ingestion.json"
+    output_file_name_after_deleted: str = "dummy_mces_stateful_after_deleted.json"
+    golden_file_name_after_deleted: str = (
+        "golden_test_stateful_ingestion_after_deleted.json"
+    )
+
+    test_resources_dir = pytestconfig.rootpath / "tests/unit/stateful_ingestion/state"
+
+    base_pipeline_config = {
+        "run_id": "dummy-test-stateful-ingestion",
+        "pipeline_name": "dummy_stateful",
+        "source": {
+            "type": "tests.unit.stateful_ingestion.state.test_stateful_ingestion.DummySource",
+            "config": {
+                "stateful_ingestion": {
+                    "enabled": True,
+                    "remove_stale_metadata": True,
+                    "state_provider": {
+                        "type": "file",
+                        "config": {
+                            "filename": f"{tmp_path}/{state_file_name}",
+                        },
+                    },
+                },
+            },
+        },
+        "sink": {
+            "type": "file",
+            "config": {},
+        },
+    }
+
+    pipeline_run1 = None
+    pipeline_run1_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(  # type: ignore
+        base_pipeline_config  # type: ignore
+    )
+    pipeline_run1_config["sink"]["config"][
+        "filename"
+    ] = f"{tmp_path}/{output_file_name}"
+    pipeline_run1 = Pipeline.create(pipeline_run1_config)
+    pipeline_run1.run()
+    pipeline_run1.raise_from_status()
+    pipeline_run1.pretty_print_summary()
+
+    # validate both dummy source mces and checkpoint state mces files
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / output_file_name,
+        golden_path=f"{test_resources_dir}/{golden_file_name}",
+    )
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / state_file_name,
+        golden_path=f"{test_resources_dir}/{golden_state_file_name}",
+    )
+    checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1)
+    assert checkpoint1
+    assert checkpoint1.state
+
+    pipeline_run2 = None
+    pipeline_run2_config: Dict[str, Dict[str, Dict[str, Any]]] = dict(base_pipeline_config)  # type: ignore
+    pipeline_run2_config["source"]["config"]["dataset_patterns"] = {
+        "allow": ["dummy_dataset1", "dummy_dataset2"],
+    }
+    pipeline_run2_config["sink"]["config"][
+        "filename"
+    ] = f"{tmp_path}/{output_file_name_after_deleted}"
+    pipeline_run2 = Pipeline.create(pipeline_run2_config)
+    pipeline_run2.run()
+    pipeline_run2.raise_from_status()
+    pipeline_run2.pretty_print_summary()
+
+    # validate both updated dummy source mces and checkpoint state mces files after deleting dataset
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / output_file_name_after_deleted,
+        golden_path=f"{test_resources_dir}/{golden_file_name_after_deleted}",
+    )
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / state_file_name,
+        golden_path=f"{test_resources_dir}/{golden_state_file_name_after_deleted}",
+    )
+    checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2)
+    assert checkpoint2
+    assert checkpoint2.state
+
+    # Validate that all providers have committed successfully.
+    validate_all_providers_have_committed_successfully(
+        pipeline=pipeline_run1, expected_providers=1
+    )
+    validate_all_providers_have_committed_successfully(
+        pipeline=pipeline_run2, expected_providers=1
+    )
+
+    # Perform all assertions on the states. The deleted table should not be
+    # part of the second state
+    state1 = cast(GenericCheckpointState, checkpoint1.state)
+    state2 = cast(GenericCheckpointState, checkpoint2.state)
+
+    difference_dataset_urns = list(
+        state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2)
+    )
+    # the difference in dataset urns is the dataset which is not allowed to ingest
+    assert len(difference_dataset_urns) == 1
+    deleted_dataset_urns: List[str] = [
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)",
+    ]
+    assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns)
diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py
index 9edfe8c4a957b..0e6d60e3440b2 100644
--- a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py
+++ b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py
@@ -3,9 +3,10 @@
 import pytest
 from pydantic import ValidationError
 
-from datahub.configuration.common import ConfigModel, DynamicTypedConfig
+from datahub.configuration.common import ConfigModel
 from datahub.ingestion.graph.client import DatahubClientConfig
 from datahub.ingestion.source.state.stateful_ingestion_base import (
+    DynamicTypedStateProviderConfig,
     StatefulIngestionConfig,
 )
 from datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider import (
@@ -23,7 +24,6 @@
     },
     "simple": {},
     "default": {},
-    "none": None,
 }
 
 
@@ -81,13 +81,6 @@
         ),
         False,
     ),
-    # None
-    "checkpointing_bad_config": (
-        DatahubIngestionStateProviderConfig,
-        datahub_client_configs["none"],
-        None,
-        True,
-    ),
 }
 
 
@@ -119,7 +112,7 @@
             max_checkpoint_state_size=1024,
             ignore_old_state=True,
             ignore_new_state=True,
-            state_provider=DynamicTypedConfig(
+            state_provider=DynamicTypedStateProviderConfig(
                 type="datahub",
                 config=datahub_client_configs["full"],
             ),
@@ -148,7 +141,7 @@
             max_checkpoint_state_size=2**24,
             ignore_old_state=False,
             ignore_new_state=False,
-            state_provider=DynamicTypedConfig(type="datahub", config=None),
+            state_provider=DynamicTypedStateProviderConfig(type="datahub"),
         ),
         False,
     ),

From bb7300251f6d65024b3440379d8eef3e2413a2b0 Mon Sep 17 00:00:00 2001
From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com>
Date: Sat, 11 Nov 2023 05:29:18 +0530
Subject: [PATCH 195/259] feat(ingestion/airflow): support datajobs as task
 inlets (#9211)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../datahub_listener.py                       | 14 ++--
 .../src/datahub_airflow_plugin/entities.py    | 28 ++++++--
 .../example_dags/lineage_backend_demo.py      |  1 +
 .../lineage_backend_taskflow_demo.py          |  1 +
 .../lineage/_lineage_core.py                  | 18 ++---
 .../tests/integration/dags/basic_iolets.py    |  1 +
 .../tests/integration/dags/simple_dag.py      |  1 +
 .../integration/goldens/v1_basic_iolets.json  | 56 ++++++++++++---
 .../integration/goldens/v1_simple_dag.json    | 70 ++++++++++++++-----
 .../integration/goldens/v2_basic_iolets.json  | 26 ++++---
 .../v2_basic_iolets_no_dag_listener.json      | 24 ++++---
 .../integration/goldens/v2_simple_dag.json    | 38 +++++-----
 .../v2_simple_dag_no_dag_listener.json        | 32 +++++----
 .../goldens/v2_snowflake_operator.json        |  2 +-
 .../goldens/v2_sqlite_operator.json           |  2 +-
 .../v2_sqlite_operator_no_dag_listener.json   | 40 +++++------
 .../airflow-plugin/tests/unit/test_airflow.py | 31 ++++++--
 17 files changed, 261 insertions(+), 124 deletions(-)

diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
index a3f5cb489e29f..d00b10bbe1756 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
@@ -17,7 +17,6 @@
 )
 from datahub.telemetry import telemetry
 from datahub.utilities.sqlglot_lineage import SqlParsingResult
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 from openlineage.airflow.listener import TaskHolder
 from openlineage.airflow.utils import redact_with_exclusions
 from openlineage.client.serde import Serde
@@ -32,7 +31,11 @@
 from datahub_airflow_plugin._datahub_ol_adapter import translate_ol_to_datahub_urn
 from datahub_airflow_plugin._extractors import SQL_PARSING_RESULT_KEY, ExtractorManager
 from datahub_airflow_plugin.client.airflow_generator import AirflowGenerator
-from datahub_airflow_plugin.entities import _Entity
+from datahub_airflow_plugin.entities import (
+    _Entity,
+    entities_to_datajob_urn_list,
+    entities_to_dataset_urn_list,
+)
 
 _F = TypeVar("_F", bound=Callable[..., None])
 if TYPE_CHECKING:
@@ -272,10 +275,9 @@ def _extract_lineage(
         )
 
         # Write the lineage to the datajob object.
-        datajob.inlets.extend(DatasetUrn.create_from_string(urn) for urn in input_urns)
-        datajob.outlets.extend(
-            DatasetUrn.create_from_string(urn) for urn in output_urns
-        )
+        datajob.inlets.extend(entities_to_dataset_urn_list(input_urns))
+        datajob.outlets.extend(entities_to_dataset_urn_list(output_urns))
+        datajob.upstream_urns.extend(entities_to_datajob_urn_list(input_urns))
         datajob.fine_grained_lineages.extend(fine_grained_lineages)
 
         # Merge in extra stuff that was present in the DataJob we constructed
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py
index 69f667cad3241..5a4bcb0097a8c 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/entities.py
@@ -1,8 +1,10 @@
 from abc import abstractmethod
-from typing import Optional
+from typing import List, Optional
 
 import attr
 import datahub.emitter.mce_builder as builder
+from datahub.utilities.urns.data_job_urn import DataJobUrn
+from datahub.utilities.urns.dataset_urn import DatasetUrn
 from datahub.utilities.urns.urn import guess_entity_type
 
 
@@ -38,10 +40,28 @@ class Urn(_Entity):
     def _validate_urn(self, attribute, value):
         if not value.startswith("urn:"):
             raise ValueError("invalid urn provided: urns must start with 'urn:'")
-        if guess_entity_type(value) != "dataset":
-            # This is because DataJobs only support Dataset lineage.
-            raise ValueError("Airflow lineage currently only supports datasets")
+        if guess_entity_type(value) not in ["dataset", "dataJob"]:
+            # This is because DataJobs only support Dataset and upstream Datajob lineage.
+            raise ValueError(
+                "Airflow lineage currently only supports datasets and upstream datajobs"
+            )
 
     @property
     def urn(self):
         return self._urn
+
+
+def entities_to_dataset_urn_list(iolets: List[str]) -> List[DatasetUrn]:
+    dataset_urn_list: List[DatasetUrn] = []
+    for let in iolets:
+        if guess_entity_type(let) == "dataset":
+            dataset_urn_list.append(DatasetUrn.create_from_string(let))
+    return dataset_urn_list
+
+
+def entities_to_datajob_urn_list(inlets: List[str]) -> List[DataJobUrn]:
+    datajob_urn_list: List[DataJobUrn] = []
+    for let in inlets:
+        if guess_entity_type(let) == "dataJob":
+            datajob_urn_list.append(DataJobUrn.create_from_string(let))
+    return datajob_urn_list
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_demo.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_demo.py
index 3caea093b932d..ce161d6a415e9 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_demo.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_demo.py
@@ -46,6 +46,7 @@
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
+            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,dag1,prod),task1)"),
         ],
         outlets=[Dataset("snowflake", "mydb.schema.tableD")],
     )
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py
index ceb0f452b540a..80df7053a49f9 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py
@@ -37,6 +37,7 @@ def datahub_lineage_backend_taskflow_demo():
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
+            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,dag1,prod),task1)"),
         ],
         outlets=[Dataset("snowflake", "mydb.schema.tableD")],
     )
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py
index f5f519fa23b11..75fc79443e49e 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py
@@ -2,11 +2,14 @@
 from typing import TYPE_CHECKING, Dict, List
 
 from datahub.api.entities.dataprocess.dataprocess_instance import InstanceRunResult
-from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 from datahub_airflow_plugin._config import DatahubLineageConfig
 from datahub_airflow_plugin.client.airflow_generator import AirflowGenerator
-from datahub_airflow_plugin.entities import _Entity
+from datahub_airflow_plugin.entities import (
+    _Entity,
+    entities_to_datajob_urn_list,
+    entities_to_dataset_urn_list,
+)
 
 if TYPE_CHECKING:
     from airflow import DAG
@@ -16,10 +19,6 @@
     from datahub_airflow_plugin._airflow_shims import Operator
 
 
-def _entities_to_urn_list(iolets: List[_Entity]) -> List[DatasetUrn]:
-    return [DatasetUrn.create_from_string(let.urn) for let in iolets]
-
-
 def send_lineage_to_datahub(
     config: DatahubLineageConfig,
     operator: "Operator",
@@ -53,8 +52,11 @@ def send_lineage_to_datahub(
         capture_tags=config.capture_tags_info,
         capture_owner=config.capture_ownership_info,
     )
-    datajob.inlets.extend(_entities_to_urn_list(inlets))
-    datajob.outlets.extend(_entities_to_urn_list(outlets))
+    datajob.inlets.extend(entities_to_dataset_urn_list([let.urn for let in inlets]))
+    datajob.outlets.extend(entities_to_dataset_urn_list([let.urn for let in outlets]))
+    datajob.upstream_urns.extend(
+        entities_to_datajob_urn_list([let.urn for let in inlets])
+    )
 
     datajob.emit(emitter)
     operator.log.info(f"Emitted from Lineage: {datajob}")
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py
index 8b0803ab98422..11b3731c52bca 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py
@@ -26,6 +26,7 @@
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
+            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"),
         ],
         outlets=[
             Dataset("snowflake", "mydb.schema.tableD"),
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py
index 1dd047f0a6dcc..71b462159ac60 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py
@@ -21,6 +21,7 @@
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
+            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"),
         ],
         outlets=[Dataset("snowflake", "mydb.schema.tableD")],
     )
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
index a4c17c73e9c7e..6b460e99b1f28 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json
@@ -11,7 +11,7 @@
                 "catchup": "False",
                 "description": "None",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
+                "fileloc": "'/home/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "None",
@@ -95,7 +95,8 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)",
-                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)",
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
             ],
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -150,6 +151,17 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -245,7 +257,8 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)",
-                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)",
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
             ],
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -300,6 +313,17 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -365,9 +389,9 @@
         "json": {
             "customProperties": {
                 "run_id": "manual_run_test",
-                "duration": "0.176536",
-                "start_date": "2023-09-30 00:49:56.670239+00:00",
-                "end_date": "2023-09-30 00:49:56.846775+00:00",
+                "duration": "0.143271",
+                "start_date": "2023-11-08 09:55:05.801617+00:00",
+                "end_date": "2023-11-08 09:55:05.944888+00:00",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "1",
                 "max_tries": "0",
@@ -384,7 +408,7 @@
             "name": "basic_iolets_run_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696034996670,
+                "time": 1699437305801,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -413,7 +437,8 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)",
-                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)",
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
             ]
         }
     }
@@ -476,6 +501,17 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -505,7 +541,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696034996670,
+            "timestampMillis": 1699437305801,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -522,7 +558,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696034996846,
+            "timestampMillis": 1699437305944,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
index a0a95716a0993..7ec172e3678dc 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json
@@ -11,7 +11,7 @@
                 "catchup": "False",
                 "description": "'A simple DAG that runs a few fake data tasks.'",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
+                "fileloc": "'/home/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "None",
@@ -94,7 +94,8 @@
         "json": {
             "inputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)",
-                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)",
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
             ],
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)"
@@ -126,6 +127,17 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -208,7 +220,8 @@
         "json": {
             "inputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)",
-                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)",
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
             ],
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)"
@@ -240,6 +253,17 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -294,9 +318,9 @@
         "json": {
             "customProperties": {
                 "run_id": "manual_run_test",
-                "duration": "0.175983",
-                "start_date": "2023-09-30 00:48:58.943850+00:00",
-                "end_date": "2023-09-30 00:48:59.119833+00:00",
+                "duration": "0.120524",
+                "start_date": "2023-11-08 09:54:06.065112+00:00",
+                "end_date": "2023-11-08 09:54:06.185636+00:00",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "1",
                 "max_tries": "0",
@@ -313,7 +337,7 @@
             "name": "simple_dag_task_1_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696034938943,
+                "time": 1699437246065,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -340,7 +364,8 @@
         "json": {
             "inputs": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)",
-                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
+                "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)",
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
             ]
         }
     }
@@ -380,6 +405,17 @@
         }
     }
 },
+{
+    "entityType": "dataJob",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    }
+},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
@@ -398,7 +434,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696034938943,
+            "timestampMillis": 1699437246065,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -415,7 +451,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696034939119,
+            "timestampMillis": 1699437246185,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -440,7 +476,7 @@
                 "catchup": "False",
                 "description": "'A simple DAG that runs a few fake data tasks.'",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
+                "fileloc": "'/home/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "None",
@@ -651,9 +687,9 @@
         "json": {
             "customProperties": {
                 "run_id": "manual_run_test",
-                "duration": "0.129888",
-                "start_date": "2023-09-30 00:49:02.158752+00:00",
-                "end_date": "2023-09-30 00:49:02.288640+00:00",
+                "duration": "0.099975",
+                "start_date": "2023-11-08 09:54:09.744583+00:00",
+                "end_date": "2023-11-08 09:54:09.844558+00:00",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "1",
                 "max_tries": "0",
@@ -670,7 +706,7 @@
             "name": "simple_dag_run_another_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696034942158,
+                "time": 1699437249744,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -695,7 +731,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696034942158,
+            "timestampMillis": 1699437249744,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -712,7 +748,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696034942288,
+            "timestampMillis": 1699437249844,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
index 1974f1f085df0..6767a368f366a 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json
@@ -11,7 +11,7 @@
                 "catchup": "False",
                 "description": "None",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
+                "fileloc": "'/Users/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "[]",
@@ -73,9 +73,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "[]",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_is_setup\": false, \"_is_teardown\": false, \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": {}, \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
             "name": "run_data_task",
@@ -102,7 +102,9 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -217,7 +219,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 01:13:14.266272+00:00",
+                "start_date": "2023-10-30 13:07:55.311482+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -235,7 +237,7 @@
             "name": "basic_iolets_run_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696036394266,
+                "time": 1698671275311,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -356,7 +358,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696036394266,
+            "timestampMillis": 1698671275311,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -383,9 +385,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "[]",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_is_setup\": false, \"_is_teardown\": false, \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": {}, \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
             "name": "run_data_task",
@@ -412,7 +414,9 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -524,7 +528,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696036394833,
+            "timestampMillis": 1698671276777,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
index d02951bc9e82d..63b0a05935554 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json
@@ -73,9 +73,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "[]",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
             "name": "run_data_task",
@@ -102,7 +102,9 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -217,7 +219,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:59:52.401211+00:00",
+                "start_date": "2023-11-10 19:11:17.444435+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -235,7 +237,7 @@
             "name": "basic_iolets_run_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057192401,
+                "time": 1699643477444,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -356,7 +358,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057192401,
+            "timestampMillis": 1699643477444,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -383,9 +385,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "[]",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
             "name": "run_data_task",
@@ -412,7 +414,9 @@
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)",
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -524,7 +528,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057192982,
+            "timestampMillis": 1699643478123,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
index 9acc47ec1321e..c558f79c32e15 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json
@@ -11,7 +11,7 @@
                 "catchup": "False",
                 "description": "'A simple DAG that runs a few fake data tasks.'",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
+                "fileloc": "'/Users/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "[]",
@@ -74,9 +74,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "['run_another_data_task']",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_is_setup\": false, \"_is_teardown\": false, \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": {}, \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1",
             "name": "task_1",
@@ -100,7 +100,9 @@
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -182,7 +184,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:53:58.219003+00:00",
+                "start_date": "2023-10-30 13:06:07.193282+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -200,7 +202,7 @@
             "name": "simple_dag_task_1_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056838219,
+                "time": 1698671167193,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -285,7 +287,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056838219,
+            "timestampMillis": 1698671167193,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -312,9 +314,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "['run_another_data_task']",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_is_setup\": false, \"_is_teardown\": false, \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": {}, \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1",
             "name": "task_1",
@@ -338,7 +340,9 @@
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -417,7 +421,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056838648,
+            "timestampMillis": 1698671168726,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -449,7 +453,7 @@
                 "downstream_task_ids": "[]",
                 "inlets": "[]",
                 "outlets": "[]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<<non-serializable: DAG>>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<<non-serializable: DAG>>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_is_setup\": false, \"_is_teardown\": false, \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": {}, \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task",
             "name": "run_another_data_task",
@@ -519,7 +523,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:54:02.407515+00:00",
+                "start_date": "2023-10-30 13:06:19.970466+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -537,7 +541,7 @@
             "name": "simple_dag_run_another_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696056842407,
+                "time": 1698671179970,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -562,7 +566,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056842407,
+            "timestampMillis": 1698671179970,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -591,7 +595,7 @@
                 "downstream_task_ids": "[]",
                 "inlets": "[]",
                 "outlets": "[]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<<non-serializable: DAG>>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<<non-serializable: DAG>>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_is_setup\": false, \"_is_teardown\": false, \"_lock_for_execution\": true, \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": {}, \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task",
             "name": "run_another_data_task",
@@ -658,7 +662,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696056842831,
+            "timestampMillis": 1698671180730,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
index 03299c483f57f..ec0f3cab1e81f 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json
@@ -74,9 +74,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "['run_another_data_task']",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1",
             "name": "task_1",
@@ -100,7 +100,9 @@
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -182,7 +184,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:58:56.105026+00:00",
+                "start_date": "2023-11-10 19:10:10.856995+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -200,7 +202,7 @@
             "name": "simple_dag_task_1_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057136105,
+                "time": 1699643410856,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -285,7 +287,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057136105,
+            "timestampMillis": 1699643410856,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -312,9 +314,9 @@
                 "trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
                 "wait_for_downstream": "False",
                 "downstream_task_ids": "['run_another_data_task']",
-                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)')]",
+                "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
                 "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]",
-                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
+                "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<<non-serializable: DAG>>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<<non-serializable: DAG>>\", \"_log\": \"<<non-serializable: Logger>>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<<non-serializable: ParamsDict>>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<<non-serializable: timedelta>>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<<non-serializable: DateTime>>\", \"task_group\": \"<<non-serializable: TaskGroup>>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1",
             "name": "task_1",
@@ -338,7 +340,9 @@
             "outputDatasets": [
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)"
             ],
-            "inputDatajobs": [],
+            "inputDatajobs": [
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"
+            ],
             "fineGrainedLineages": []
         }
     }
@@ -417,7 +421,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057136612,
+            "timestampMillis": 1699643411390,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -577,7 +581,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-09-30 06:58:59.567004+00:00",
+                "start_date": "2023-11-10 19:10:15.128009+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -595,7 +599,7 @@
             "name": "simple_dag_run_another_data_task_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1696057139567,
+                "time": 1699643415128,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -620,7 +624,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057139567,
+            "timestampMillis": 1699643415128,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -716,7 +720,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1696057140164,
+            "timestampMillis": 1699643415856,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
index 11a0b17b45b95..0a704ed10c911 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json
@@ -11,7 +11,7 @@
                 "catchup": "False",
                 "description": "None",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py'",
+                "fileloc": "'/Users/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "[]",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
index 19e4aac9fb95e..3b4b60174f99f 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json
@@ -11,7 +11,7 @@
                 "catchup": "False",
                 "description": "None",
                 "doc_md": "None",
-                "fileloc": "'/Users/hsheth/projects/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
+                "fileloc": "'/Users/shubham/airflow1/datahub/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/sqlite_operator.py'",
                 "is_paused_upon_creation": "None",
                 "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))",
                 "tags": "[]",
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
index b67464b385335..99a8aadb7fd9c 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
+++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json
@@ -194,7 +194,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-10-15 20:27:26.883178+00:00",
+                "start_date": "2023-11-10 19:12:17.805860+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -212,7 +212,7 @@
             "name": "sqlite_operator_create_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1697401646883,
+                "time": 1699643537805,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -261,7 +261,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401646883,
+            "timestampMillis": 1699643537805,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -442,7 +442,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401647826,
+            "timestampMillis": 1699643538759,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -615,7 +615,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-10-15 20:27:31.398799+00:00",
+                "start_date": "2023-11-10 19:12:22.560376+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -633,7 +633,7 @@
             "name": "sqlite_operator_populate_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1697401651398,
+                "time": 1699643542560,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -682,7 +682,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401651398,
+            "timestampMillis": 1699643542560,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -792,7 +792,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401652651,
+            "timestampMillis": 1699643543925,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1035,7 +1035,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-10-15 20:27:37.697995+00:00",
+                "start_date": "2023-11-10 19:12:29.429032+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1053,7 +1053,7 @@
             "name": "sqlite_operator_transform_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1697401657697,
+                "time": 1699643549429,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1126,7 +1126,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401657697,
+            "timestampMillis": 1699643549429,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1362,7 +1362,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401659496,
+            "timestampMillis": 1699643551423,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1537,7 +1537,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-10-15 20:27:45.670215+00:00",
+                "start_date": "2023-11-10 19:12:37.423556+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1555,7 +1555,7 @@
             "name": "sqlite_operator_cleanup_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1697401665670,
+                "time": 1699643557423,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1604,7 +1604,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401665670,
+            "timestampMillis": 1699643557423,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1716,7 +1716,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401667670,
+            "timestampMillis": 1699643559607,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -1891,7 +1891,7 @@
             "customProperties": {
                 "run_id": "manual_run_test",
                 "duration": "None",
-                "start_date": "2023-10-15 20:27:51.559194+00:00",
+                "start_date": "2023-11-10 19:12:43.792375+00:00",
                 "end_date": "None",
                 "execution_date": "2023-09-27 21:34:38+00:00",
                 "try_number": "0",
@@ -1909,7 +1909,7 @@
             "name": "sqlite_operator_cleanup_processed_costs_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1697401671559,
+                "time": 1699643563792,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -1958,7 +1958,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401671559,
+            "timestampMillis": 1699643563792,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
@@ -2070,7 +2070,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1697401673788,
+            "timestampMillis": 1699643566350,
             "partitionSpec": {
                 "type": "FULL_TABLE",
                 "partition": "FULL_TABLE_SNAPSHOT"
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
index d8620e74d7e30..7fbf707995994 100644
--- a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
+++ b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
@@ -188,10 +188,17 @@ def test_entities():
         == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableConsumed,PROD)"
     )
 
+    assert (
+        Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,testDag,PROD),testTask)").urn
+        == "urn:li:dataJob:(urn:li:dataFlow:(airflow,testDag,PROD),testTask)"
+    )
+
     with pytest.raises(ValueError, match="invalid"):
         Urn("not a URN")
 
-    with pytest.raises(ValueError, match="only supports datasets"):
+    with pytest.raises(
+        ValueError, match="only supports datasets and upstream datajobs"
+    ):
         Urn("urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)")
 
 
@@ -199,13 +206,19 @@ def test_entities():
     ["inlets", "outlets", "capture_executions"],
     [
         pytest.param(
-            [Dataset("snowflake", "mydb.schema.tableConsumed")],
+            [
+                Dataset("snowflake", "mydb.schema.tableConsumed"),
+                Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,testDag,PROD),testTask)"),
+            ],
             [Dataset("snowflake", "mydb.schema.tableProduced")],
             False,
             id="airflow-lineage-no-executions",
         ),
         pytest.param(
-            [Dataset("snowflake", "mydb.schema.tableConsumed")],
+            [
+                Dataset("snowflake", "mydb.schema.tableConsumed"),
+                Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,testDag,PROD),testTask)"),
+            ],
             [Dataset("snowflake", "mydb.schema.tableProduced")],
             True,
             id="airflow-lineage-capture-executions",
@@ -293,9 +306,13 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions):
 
         # Verify that the inlets and outlets are registered and recognized by Airflow correctly,
         # or that our lineage backend forces it to.
-        assert len(op2.inlets) == 1
+        assert len(op2.inlets) == 2
         assert len(op2.outlets) == 1
-        assert all(map(lambda let: isinstance(let, Dataset), op2.inlets))
+        assert all(
+            map(
+                lambda let: isinstance(let, Dataset) or isinstance(let, Urn), op2.inlets
+            )
+        )
         assert all(map(lambda let: isinstance(let, Dataset), op2.outlets))
 
         # Check that the right things were emitted.
@@ -338,6 +355,10 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions):
                 mock_emitter.method_calls[4].args[0].aspect.inputDatajobs[0]
                 == "urn:li:dataJob:(urn:li:dataFlow:(airflow,test_lineage_is_sent_to_backend,prod),task1_upstream)"
             )
+            assert (
+                mock_emitter.method_calls[4].args[0].aspect.inputDatajobs[1]
+                == "urn:li:dataJob:(urn:li:dataFlow:(airflow,testDag,PROD),testTask)"
+            )
             assert (
                 mock_emitter.method_calls[4].args[0].aspect.inputDatasets[0]
                 == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableConsumed,PROD)"

From 0e3efabd2c19e24bcfb81602f897802be1cb1d06 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Fri, 10 Nov 2023 16:00:20 -0800
Subject: [PATCH 196/259] fix(build): set `@cliMajorVersion@` correctly (#9228)

---
 .github/workflows/build-and-test.yml          |  6 +--
 .github/workflows/check-datahub-jars.yml      |  4 +-
 .github/workflows/code-checks.yml             |  4 +-
 .github/workflows/docker-ingestion-smoke.yml  |  4 +-
 .github/workflows/docker-postgres-setup.yml   |  4 +-
 .github/workflows/docker-unified.yml          | 52 +++++--------------
 .github/workflows/publish-datahub-jars.yml    |  4 +-
 .github/workflows/spark-smoke-test.yml        |  5 +-
 metadata-service/configuration/build.gradle   |  7 ++-
 .../src/main/resources/application.yml        |  9 ++--
 metadata-service/factories/build.gradle       |  5 --
 .../tests/read_only/test_services_up.py       |  8 +++
 12 files changed, 39 insertions(+), 73 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 25f3957e8f086..10c137a206531 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -27,7 +27,7 @@ jobs:
         command: [
             # metadata-ingestion and airflow-plugin each have dedicated build jobs
             "except_metadata_ingestion",
-            "frontend"
+            "frontend",
           ]
         timezone: ["UTC", "America/New_York"]
     runs-on: ubuntu-latest
@@ -36,9 +36,7 @@ jobs:
       - uses: szenius/set-timezone@v1.0
         with:
           timezoneLinux: ${{ matrix.timezone }}
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
index 41f9ea91a94e2..8e507ea40fd96 100644
--- a/.github/workflows/check-datahub-jars.yml
+++ b/.github/workflows/check-datahub-jars.yml
@@ -27,9 +27,7 @@ jobs:
         command: ["datahub-client", "datahub-protobuf", "spark-lineage"]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index e12971b8a6208..38f0946678034 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -31,9 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
index 82b57d23609a5..803ddc6fcec75 100644
--- a/.github/workflows/docker-ingestion-smoke.yml
+++ b/.github/workflows/docker-ingestion-smoke.yml
@@ -50,9 +50,7 @@ jobs:
     if: ${{ needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
index fda4349f90bf7..e4d6cfc106f81 100644
--- a/.github/workflows/docker-postgres-setup.yml
+++ b/.github/workflows/docker-postgres-setup.yml
@@ -46,9 +46,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index 5f5a62de6288c..18cb946b951dd 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -74,9 +74,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-service:war:build -x test --parallel
@@ -132,9 +130,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel
@@ -190,9 +186,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
@@ -248,9 +242,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-upgrade:build -x test --parallel
@@ -306,9 +298,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
@@ -366,9 +356,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -388,9 +376,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -410,9 +396,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -434,9 +418,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -468,9 +450,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -510,9 +490,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -554,9 +532,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_slim_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -637,9 +613,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_full_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
index 7cd07b130dd80..ec7985ef3b3d0 100644
--- a/.github/workflows/publish-datahub-jars.yml
+++ b/.github/workflows/publish-datahub-jars.yml
@@ -48,9 +48,7 @@ jobs:
     needs: ["check-secret", "setup"]
     if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
index 541b2019b93ef..70b66d6452b26 100644
--- a/.github/workflows/spark-smoke-test.yml
+++ b/.github/workflows/spark-smoke-test.yml
@@ -29,10 +29,7 @@ jobs:
   spark-smoke-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
-          fetch-tags: true
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle
index bf79469633b0f..80cf6541261c2 100644
--- a/metadata-service/configuration/build.gradle
+++ b/metadata-service/configuration/build.gradle
@@ -1,6 +1,7 @@
 plugins {
     id 'java'
 }
+apply from: "../../gradle/versioning/versioning.gradle"
 
 dependencies {
     implementation externalDependency.jacksonDataBind
@@ -12,4 +13,8 @@ dependencies {
     compileOnly externalDependency.lombok
 
     annotationProcessor externalDependency.lombok
-}
\ No newline at end of file
+}
+
+processResources.configure {
+  finalizedBy printVersionDetails // always print version details
+}
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 46aa02d98572e..40674e13e647f 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -44,6 +44,7 @@ authorization:
   restApiAuthorization: ${REST_API_AUTHORIZATION_ENABLED:false}
 
 ingestion:
+  # The value of cliMajorVersion is substituted in by the processResources Gradle task.
   enabled: ${UI_INGESTION_ENABLED:true}
   defaultCliVersion: "${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}"
   maxSerializedStringLength: "${INGESTION_MAX_SERIALIZED_STRING_LENGTH:16000000}" # Indicates the maximum allowed JSON String length Jackson will handle, impacts the maximum size of ingested aspects
@@ -74,7 +75,7 @@ datahub:
   plugin:
     pluginSecurityMode: ${PLUGIN_SECURITY_MODE:RESTRICTED} # Possible value RESTRICTED or LENIENT, default to RESTRICTED
     entityRegistry:
-        path: ${ENTITY_REGISTRY_PLUGIN_PATH:/etc/datahub/plugins/models}
+      path: ${ENTITY_REGISTRY_PLUGIN_PATH:/etc/datahub/plugins/models}
     retention:
       path: ${RETENTION_PLUGIN_PATH:/etc/datahub/plugins/retention}
     auth:
@@ -280,14 +281,13 @@ updateIndices:
 ingestionScheduler:
   enabled: ${ENABLE_INGESTION_SCHEDULER_HOOK:true} # enable to execute ingestion scheduling
 
-
 bootstrap:
   upgradeDefaultBrowsePaths:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
   backfillBrowsePathsV2:
     enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
   reprocessDefaultBrowsePathsV2:
-    enabled:  ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]}
+    enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]}
   policies:
     file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
     # eg for local file
@@ -295,7 +295,6 @@ bootstrap:
   servlets:
     waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize
 
-
 systemUpdate:
   initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000}
   maxBackOffs: ${BOOTSTRAP_SYSTEM_UPDATE_MAX_BACK_OFFS:50}
@@ -371,4 +370,4 @@ cache:
           corpUserCredentials: 20
           corpUserSettings: 20
 
-springdoc.api-docs.groups.enabled: true
\ No newline at end of file
+springdoc.api-docs.groups.enabled: true
diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle
index f848a5e339781..2e99def17c3c5 100644
--- a/metadata-service/factories/build.gradle
+++ b/metadata-service/factories/build.gradle
@@ -1,5 +1,4 @@
 apply plugin: 'java-library'
-apply from: "../../gradle/versioning/versioning.gradle"
 
 dependencies {
   api project(':metadata-io')
@@ -65,7 +64,3 @@ configurations.all{
   exclude group: "commons-io", module:"commons-io"
   exclude group: "jline", module:"jline"
 }
-
-processResources.configure {
-  finalizedBy printVersionDetails // always print version details
-}
diff --git a/smoke-test/tests/read_only/test_services_up.py b/smoke-test/tests/read_only/test_services_up.py
index e48df52bb9864..cbe92625f4689 100644
--- a/smoke-test/tests/read_only/test_services_up.py
+++ b/smoke-test/tests/read_only/test_services_up.py
@@ -23,3 +23,11 @@ def test_gms_config_accessible():
         assert gms_config["versions"]["linkedin/datahub"]["version"] == DATAHUB_VERSION
     else:
         print("[WARN] TEST_DATAHUB_VERSION is not set")
+
+    # Make sure that the default CLI version gets generated properly.
+    # While we don't want to hardcode the actual value, we can make
+    # sure it mostly looks like a version string.
+    default_cli_version: str = gms_config["managedIngestion"]["defaultCliVersion"]
+    print(f"Default CLI version: {default_cli_version}")
+    assert not default_cli_version.startswith("@")
+    assert "." in default_cli_version

From ebd2e2312bdd23a92bbf403a26f64194807f70e6 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 10 Nov 2023 22:10:00 -0600
Subject: [PATCH 197/259] fix(datahub-ingestion): remove old jars, sync pyspark
 version (#9217)

---
 docker/datahub-ingestion-base/build.gradle |  7 ++++---
 docker/datahub-ingestion/Dockerfile        | 16 ++++++++++++++--
 docker/datahub-ingestion/pyspark_jars.sh   | 22 ++++++++++++++++++++++
 metadata-ingestion/setup.py                |  4 ++--
 4 files changed, 42 insertions(+), 7 deletions(-)
 create mode 100755 docker/datahub-ingestion/pyspark_jars.sh

diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle
index bbd8242553cc5..64635671343ef 100644
--- a/docker/datahub-ingestion-base/build.gradle
+++ b/docker/datahub-ingestion-base/build.gradle
@@ -9,20 +9,21 @@ ext {
     docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
     docker_repo = 'datahub-ingestion-base'
     docker_dir = 'datahub-ingestion-base'
+    docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
 
     revision = 2 // increment to trigger rebuild
 }
 
 docker {
-    name "${docker_registry}/${docker_repo}:v${version}-slim"
-    version "v${version}-slim"
+    name "${docker_registry}/${docker_repo}:v${version}-${docker_target}"
+    version "v${version}-${docker_target}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
         include "docker/${docker_dir}/*"
     }.exclude {
         i -> i.file.isHidden() || i.file == buildDir
     }
-    buildArgs([APP_ENV: 'slim'])
+    buildArgs([APP_ENV: docker_target])
 }
 tasks.getByName('docker').dependsOn('build')
 
diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile
index 0132ceaa9b1a9..2abd4e2f33bef 100644
--- a/docker/datahub-ingestion/Dockerfile
+++ b/docker/datahub-ingestion/Dockerfile
@@ -22,10 +22,22 @@ ENV PATH="/datahub-ingestion/.local/bin:$PATH"
 FROM base as slim-install
 RUN pip install --no-cache --user ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
 
-FROM base as full-install
+FROM base as full-install-build
+
+USER 0
+RUN apt-get update && apt-get install -y -qq maven
+
+USER datahub
+COPY ./docker/datahub-ingestion/pyspark_jars.sh .
+
 RUN pip install --no-cache --user ".[base]" && \
     pip install --no-cache --user "./airflow-plugin[acryl-datahub-airflow-plugin]" && \
-    pip install --no-cache --user ".[all]"
+    pip install --no-cache --user ".[all]" && \
+    ./pyspark_jars.sh
+
+FROM base as full-install
+
+COPY --from=full-install-build /datahub-ingestion/.local /datahub-ingestion/.local
 
 FROM base as dev-install
 # Dummy stage for development. Assumes code is built on your machine and mounted to this image.
diff --git a/docker/datahub-ingestion/pyspark_jars.sh b/docker/datahub-ingestion/pyspark_jars.sh
new file mode 100755
index 0000000000000..ecd24e78c4105
--- /dev/null
+++ b/docker/datahub-ingestion/pyspark_jars.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -ex
+
+HADOOP_CLIENT_DEPENDENCY="${HADOOP_CLIENT_DEPENDENCY:-org.apache.hadoop:hadoop-client:3.3.6}"
+ZOOKEEPER_DEPENDENCY="${ZOOKEEPER_DEPENDENCY:-org.apache.zookeeper:zookeeper:3.7.2}"
+PYSPARK_JARS="$(python -m site --user-site)/pyspark/jars"
+
+# Remove conflicting versions
+echo "Removing version conflicts from $PYSPARK_JARS"
+CONFLICTS="zookeeper hadoop- slf4j-"
+for jar in $CONFLICTS; do
+  rm "$PYSPARK_JARS/$jar"*.jar
+done
+
+# Fetch dependencies
+mvn dependency:get -Dtransitive=true -Dartifact="$HADOOP_CLIENT_DEPENDENCY"
+mvn dependency:get -Dtransitive=true -Dartifact="$ZOOKEEPER_DEPENDENCY"
+
+# Move to pyspark location
+echo "Moving jars to $PYSPARK_JARS"
+find "$HOME/.m2" -type f -name "*.jar" -exec mv {} "$PYSPARK_JARS/" \;
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 5f44f14c3d74c..f3782abe576d3 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -242,7 +242,7 @@
 }
 
 data_lake_profiling = {
-    "pydeequ==1.1.0",
+    "pydeequ~=1.1.0",
     "pyspark~=3.3.0",
 }
 
@@ -256,7 +256,7 @@
 databricks = {
     # 0.1.11 appears to have authentication issues with azure databricks
     "databricks-sdk>=0.9.0",
-    "pyspark",
+    "pyspark~=3.3.0",
     "requests",
 }
 

From 7ba54fdb9820cd79296801c48d05dc177e3739f1 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Mon, 13 Nov 2023 12:33:34 +0900
Subject: [PATCH 198/259] fix: re-add security.md to sidebar (#9229)

---
 SECURITY.md              | 2 +-
 docs-website/sidebars.js | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/SECURITY.md b/SECURITY.md
index 3ca87b08d844d..0e301d3748373 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -1,4 +1,4 @@
-# Reporting security issues
+# Reporting Security Issues
 
 If you think you have found a security vulnerability, please send a report to security@datahubproject.io. This address can be used for all of Acryl Data’s open source and commercial products (including but not limited to DataHub and Acryl Data). We can accept only vulnerability reports at this address.
 
diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index f15f2927379c5..801e0fbd07d36 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -660,6 +660,7 @@ module.exports = {
         "docs/CONTRIBUTING",
         "docs/links",
         "docs/rfc",
+        "SECURITY",
       ],
     },
     {

From 582eebe739ef7f8fc7651a78eee9306143360b68 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Mon, 13 Nov 2023 10:57:49 -0600
Subject: [PATCH 199/259] feat(policies): reduce default access for all users
 (#9067)

Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
---
 .github/scripts/check_policies.py             | 28 ++++--
 .../war/src/main/resources/boot/policies.json | 89 ++++++++++---------
 .../cypress/e2e/settings/managing_groups.js   |  2 +-
 3 files changed, 70 insertions(+), 49 deletions(-)

diff --git a/.github/scripts/check_policies.py b/.github/scripts/check_policies.py
index 2ad5f7fff015b..cc3576e05413c 100644
--- a/.github/scripts/check_policies.py
+++ b/.github/scripts/check_policies.py
@@ -20,7 +20,7 @@
     elif urn == "urn:li:dataHubPolicy:editor-platform-policy":
         editor_platform_policy_privileges = policy["info"]["privileges"]
     elif urn == "urn:li:dataHubPolicy:7":
-        all_user_platform_policy_privilges = policy["info"]["privileges"]
+        all_user_platform_policy_privileges = policy["info"]["privileges"]
     try:
         doc_type = policy["info"]["type"]
         privileges = policy["info"]["privileges"]
@@ -54,10 +54,22 @@
 )
 assert len(diff_policies) == 0, f"Missing privileges for root user are {diff_policies}"
 
-diff_policies = set(editor_platform_policy_privileges).difference(
-    set(all_user_platform_policy_privilges)
-)
-assert "MANAGE_POLICIES" not in all_user_platform_policy_privilges
-assert (
-    len(diff_policies) == 0
-), f"Missing privileges for all user policies are {diff_policies}"
+# All users privileges checks
+assert "MANAGE_POLICIES" not in all_user_platform_policy_privileges
+assert "MANAGE_USERS_AND_GROUPS" not in all_user_platform_policy_privileges
+assert "MANAGE_SECRETS" not in all_user_platform_policy_privileges
+assert "MANAGE_USER_CREDENTIALS" not in all_user_platform_policy_privileges
+assert "MANAGE_ACCESS_TOKENS" not in all_user_platform_policy_privileges
+assert "EDIT_ENTITY" not in all_user_platform_policy_privileges
+assert "DELETE_ENTITY" not in all_user_platform_policy_privileges
+
+# Editor checks
+assert "MANAGE_POLICIES" not in editor_platform_policy_privileges
+assert "MANAGE_USERS_AND_GROUPS" not in editor_platform_policy_privileges
+assert "MANAGE_SECRETS" not in editor_platform_policy_privileges
+assert "MANAGE_USER_CREDENTIALS" not in editor_platform_policy_privileges
+assert "MANAGE_ACCESS_TOKENS" not in editor_platform_policy_privileges
+# These don't prevent a user from modifying entities they are an asset owner of, i.e. their own profile info
+assert "EDIT_CONTACT_INFO" not in editor_platform_policy_privileges
+assert "EDIT_USER_PROFILE" not in editor_platform_policy_privileges
+assert "EDIT_ENTITY_OWNERS" not in editor_platform_policy_privileges
diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json
index b7ffc11c08f05..32e68e7b13343 100644
--- a/metadata-service/war/src/main/resources/boot/policies.json
+++ b/metadata-service/war/src/main/resources/boot/policies.json
@@ -74,21 +74,6 @@
       "editable":false
     }
   },
-  {
-    "urn": "urn:li:dataHubPolicy:2"
-  },
-  {
-    "urn": "urn:li:dataHubPolicy:3"
-  },
-  {
-    "urn": "urn:li:dataHubPolicy:4"
-  },
-  {
-    "urn": "urn:li:dataHubPolicy:5"
-  },
-  {
-    "urn": "urn:li:dataHubPolicy:6"
-  },
   {
     "urn": "urn:li:dataHubPolicy:7",
     "info": {
@@ -99,18 +84,8 @@
         "users":[]
       },
       "privileges":[
-        "MANAGE_INGESTION",
-        "MANAGE_SECRETS",
-        "MANAGE_USERS_AND_GROUPS",
         "VIEW_ANALYTICS",
-        "GENERATE_PERSONAL_ACCESS_TOKENS",
-        "MANAGE_DOMAINS",
-        "MANAGE_GLOBAL_ANNOUNCEMENTS",
-        "MANAGE_TESTS",
-        "MANAGE_GLOSSARIES",
-        "MANAGE_TAGS",
-        "MANAGE_GLOBAL_VIEWS",
-        "MANAGE_GLOBAL_OWNERSHIP_TYPES"
+        "GENERATE_PERSONAL_ACCESS_TOKENS"
       ],
       "displayName":"All Users - Base Platform Privileges",
       "description":"Grants base platform privileges to ALL users of DataHub. Change this policy to alter that behavior.",
@@ -119,15 +94,6 @@
       "editable":true
     }
   },
-  {
-    "urn": "urn:li:dataHubPolicy:8"
-  },
-  {
-    "urn": "urn:li:dataHubPolicy:9"
-  },
-  {
-    "urn": "urn:li:dataHubPolicy:10"
-  },
   {
     "urn": "urn:li:dataHubPolicy:view-entity-page-all",
     "info": {
@@ -313,7 +279,6 @@
         "VIEW_ENTITY_PAGE",
         "EDIT_ENTITY_TAGS",
         "EDIT_ENTITY_GLOSSARY_TERMS",
-        "EDIT_ENTITY_OWNERS",
         "EDIT_ENTITY_DOCS",
         "EDIT_ENTITY_DOC_LINKS",
         "EDIT_ENTITY_STATUS",
@@ -321,16 +286,12 @@
         "EDIT_ENTITY_DATA_PRODUCTS",
         "EDIT_DEPRECATION_PRIVILEGE",
         "EDIT_ENTITY_ASSERTIONS",
-        "EDIT_ENTITY",
         "EDIT_DATASET_COL_TAGS",
         "EDIT_DATASET_COL_GLOSSARY_TERMS",
         "EDIT_DATASET_COL_DESCRIPTION",
         "VIEW_DATASET_USAGE",
         "VIEW_DATASET_PROFILE",
         "EDIT_TAG_COLOR",
-        "EDIT_GROUP_MEMBERS",
-        "EDIT_USER_PROFILE",
-        "EDIT_CONTACT_INFO",
         "EDIT_LINEAGE",
         "EDIT_ENTITY_QUERIES",
         "SEARCH_PRIVILEGE",
@@ -348,6 +309,54 @@
       "editable":false
     }
   },
+  {
+    "urn": "urn:li:dataHubPolicy:editor-metadata-policy-entities",
+    "info": {
+      "actors":{
+        "resourceOwners":false,
+        "allUsers":false,
+        "allGroups":false,
+        "users":[],
+        "groups":[],
+        "roles":[
+          "urn:li:dataHubRole:Editor"
+        ]
+      },
+      "resources": {
+        "allResources": false,
+        "filter": {
+          "criteria": [
+            {
+              "field": "RESOURCE_TYPE",
+              "values": [
+                "dataset",
+                "chart",
+                "dashboard",
+                "dataFlow",
+                "dataJob",
+                "tag",
+                "container",
+                "domain",
+                "glossaryTerm",
+                "glossaryNode",
+                "notebook",
+                "dataProduct"
+              ],
+              "condition": "EQUALS"
+            }
+          ]
+        }
+      },
+      "privileges":[
+        "EDIT_ENTITY"
+      ],
+      "displayName":"Editors - Edit Metadata Entities",
+      "description":"Editors can edit primary metadata entities.",
+      "state":"ACTIVE",
+      "type":"METADATA",
+      "editable":true
+    }
+  },
   {
     "urn": "urn:li:dataHubPolicy:reader-platform-policy",
     "info": {
diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js
index 8d689c7e2303c..70219a550cd8b 100644
--- a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js
+++ b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js
@@ -96,7 +96,7 @@ describe("create and manage group", () => {
     });
 
     it("test user verify group participation", () => {
-        cy.loginWithCredentials(email,password);
+        cy.loginWithCredentials();
         cy.visit("/settings/identities/groups");
         cy.hideOnboardingTour();
         cy.clickOptionWithText(`Test group EDITED ${test_id}`);

From 4461b60583235c27bfe6244e6e0f12d08638aee5 Mon Sep 17 00:00:00 2001
From: Yuriy Gavrilov <44679014+YuriyGavrilov@users.noreply.github.com>
Date: Mon, 13 Nov 2023 21:15:13 +0300
Subject: [PATCH 200/259] Update add new company s7 airlines (#9019)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 79f85433fbc18..6b8fa520e432e 100644
--- a/README.md
+++ b/README.md
@@ -142,6 +142,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [SpotHero](https://spothero.com)
 - [Stash](https://www.stash.com)
 - [Shanghai HuaRui Bank](https://www.shrbank.com)
+- [s7 Airlines](https://www.s7.ru/)
 - [ThoughtWorks](https://www.thoughtworks.com)
 - [TypeForm](http://typeform.com)
 - [Udemy](https://www.udemy.com/)

From 3844b78fa220a92cb2ec9dd8599d9109106f8a24 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Mon, 13 Nov 2023 16:19:41 -0600
Subject: [PATCH 201/259] docs(debug): add debug information for cli (#9208)

---
 docs/actions/README.md | 3 ++-
 docs/cli.md            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/actions/README.md b/docs/actions/README.md
index 23596ec67514e..4fa44eec588be 100644
--- a/docs/actions/README.md
+++ b/docs/actions/README.md
@@ -162,7 +162,8 @@ datahub actions -c <config-1.yaml> -c <config-2.yaml>
 
 ### Running in debug mode
 
-Simply append the `--debug` flag to the CLI to run your action in debug mode.
+Simply append the `--debug` flag to the CLI to run your action in debug mode. NOTE: This will reveal sensitive information in the logs, do not share the logs with outside resources and ensure untrusted
+users will not have access to logs through UI ingestions before enabling on instances.
 
 ```
 datahub actions -c <config.yaml> --debug
diff --git a/docs/cli.md b/docs/cli.md
index 267f289d9f54a..7dfac1e9b2bff 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -134,7 +134,7 @@ The environment variables listed below take precedence over the DataHub CLI conf
 - `DATAHUB_GMS_TOKEN` (default `None`) - Used for communicating with DataHub Cloud.
 - `DATAHUB_TELEMETRY_ENABLED` (default `true`) - Set to `false` to disable telemetry. If CLI is being run in an environment with no access to public internet then this should be disabled.
 - `DATAHUB_TELEMETRY_TIMEOUT` (default `10`) - Set to a custom integer value to specify timeout in secs when sending telemetry.
-- `DATAHUB_DEBUG` (default `false`) - Set to `true` to enable debug logging for CLI. Can also be achieved through `--debug` option of the CLI.
+- `DATAHUB_DEBUG` (default `false`) - Set to `true` to enable debug logging for CLI. Can also be achieved through `--debug` option of the CLI. This exposes sensitive information in logs, enabling on production instances should be avoided especially if UI ingestion is in use as logs can be made available for runs through the UI.
 - `DATAHUB_VERSION` (default `head`) - Set to a specific version to run quickstart with the particular version of docker images.
 - `ACTIONS_VERSION` (default `head`) - Set to a specific version to run quickstart with that image tag of `datahub-actions` container.
 - `DATAHUB_ACTIONS_IMAGE` (default `acryldata/datahub-actions`) - Set to `-slim` to run a slimmer actions container without pyspark/deequ features.

From ff90fb633da78a25f19f33dca0dae58df2b5ff82 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Mon, 13 Nov 2023 16:26:53 -0600
Subject: [PATCH 202/259] =?UTF-8?q?fix(datahub-ingestion):=20prevent=20tra?=
 =?UTF-8?q?nsitive=20deps,=20bump=20addtional=20pyspa=E2=80=A6=20(#9233)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .dockerignore                               |  5 +--
 datahub-frontend/build.gradle               |  3 +-
 datahub-upgrade/build.gradle                |  3 +-
 docker/datahub-ingestion-base/build.gradle  |  8 +++--
 docker/datahub-ingestion/Dockerfile         |  4 +--
 docker/datahub-ingestion/README.md          |  7 ++++
 docker/datahub-ingestion/build.gradle       | 16 ++++-----
 docker/datahub-ingestion/pyspark_jars.sh    | 40 +++++++++++++--------
 docker/elasticsearch-setup/build.gradle     |  5 +--
 docker/kafka-setup/build.gradle             |  3 +-
 docker/mysql-setup/build.gradle             |  3 +-
 docker/postgres-setup/build.gradle          |  3 +-
 metadata-jobs/mae-consumer-job/build.gradle |  3 +-
 metadata-jobs/mce-consumer-job/build.gradle |  3 +-
 metadata-service/war/build.gradle           |  3 +-
 15 files changed, 70 insertions(+), 39 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 29c6c45bb0653..701263f5fedde 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,6 +1,7 @@
 **/node_modules/
-datahub-frontend/build/
-metadata-ingestion/venv/
+*/build/
+*/*/build/
+*/venv/
 out
 **/*.class
 # Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars
diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle
index fdf13bac0accc..eb81b31745536 100644
--- a/datahub-frontend/build.gradle
+++ b/datahub-frontend/build.gradle
@@ -77,10 +77,11 @@ docker {
   version "v${version}"
   dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
   files fileTree(rootProject.projectDir) {
+    include '.dockerignore'
     include 'docker/monitoring/*'
     include "docker/${docker_dir}/*"
   }.exclude {
-    i -> i.file.isHidden() || i.file == buildDir
+    i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
   tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
index 5d0edf3ee8427..81e6e79c2a5e5 100644
--- a/datahub-upgrade/build.gradle
+++ b/datahub-upgrade/build.gradle
@@ -88,10 +88,11 @@ docker {
   dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
   files bootJar.outputs.files
   files fileTree(rootProject.projectDir) {
+    include '.dockerignore'
     include "docker/${docker_repo}/*"
     include 'metadata-models/src/main/resources/*'
   }.exclude {
-    i -> i.file.isHidden() || i.file == buildDir
+    i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
   tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle
index 64635671343ef..c4d8a962dcd32 100644
--- a/docker/datahub-ingestion-base/build.gradle
+++ b/docker/datahub-ingestion-base/build.gradle
@@ -10,18 +10,20 @@ ext {
     docker_repo = 'datahub-ingestion-base'
     docker_dir = 'datahub-ingestion-base'
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
+    docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
     revision = 2 // increment to trigger rebuild
 }
 
 docker {
-    name "${docker_registry}/${docker_repo}:v${version}-${docker_target}"
-    version "v${version}-${docker_target}"
+    name "${docker_registry}/${docker_repo}:v${docker_version}"
+    version "v${docker_version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include "docker/${docker_dir}/*"
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     buildArgs([APP_ENV: docker_target])
 }
diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile
index 2abd4e2f33bef..1aee79a428a98 100644
--- a/docker/datahub-ingestion/Dockerfile
+++ b/docker/datahub-ingestion/Dockerfile
@@ -32,8 +32,8 @@ COPY ./docker/datahub-ingestion/pyspark_jars.sh .
 
 RUN pip install --no-cache --user ".[base]" && \
     pip install --no-cache --user "./airflow-plugin[acryl-datahub-airflow-plugin]" && \
-    pip install --no-cache --user ".[all]" && \
-    ./pyspark_jars.sh
+    pip install --no-cache --user ".[all]"
+RUN ./pyspark_jars.sh
 
 FROM base as full-install
 
diff --git a/docker/datahub-ingestion/README.md b/docker/datahub-ingestion/README.md
index 6580199bcce21..ed856314c5cc0 100644
--- a/docker/datahub-ingestion/README.md
+++ b/docker/datahub-ingestion/README.md
@@ -2,3 +2,10 @@
 [![datahub-ingestion docker](https://github.com/datahub-project/datahub/actions/workflows/docker-ingestion.yml/badge.svg)](https://github.com/datahub-project/datahub/actions/workflows/docker-ingestion.yml)
 
 Refer to the [metadata ingestion framework](../../metadata-ingestion) to understand the architecture and responsibilities of this service.
+
+## Slim vs Full Image Build
+
+There are two versions of this image. One includes pyspark and Oracle dependencies and is larger due to the java dependencies.
+
+Running the standard build results in the `slim` image without pyspark being generated by default. In order to build the full
+image with pyspark use the following project property `-PdockerTarget=full`.
diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle
index fed33752a4b81..247b896d6955c 100644
--- a/docker/datahub-ingestion/build.gradle
+++ b/docker/datahub-ingestion/build.gradle
@@ -9,6 +9,8 @@ ext {
     docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
     docker_repo = 'datahub-ingestion'
     docker_dir = 'datahub-ingestion'
+    docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
+    docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
     revision = 2 // increment to trigger rebuild
 }
@@ -19,21 +21,19 @@ dependencies {
 }
 
 docker {
-    name "${docker_registry}/${docker_repo}:v${version}-slim"
-    version "v${version}-slim"
-    dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile-slim-only")
+    name "${docker_registry}/${docker_repo}:v${docker_version}"
+    version "v${docker_version}"
+    dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile${docker_target == "slim" ? "-slim-only" : ""}")
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include "docker/${docker_dir}/*"
         include "metadata-ingestion/**"
         include "metadata-ingestion-modules/**"
     }.exclude {
-        i -> i.file.isHidden() ||
-                i.file == buildDir ||
-                i.file == project(':metadata-ingestion').buildDir ||
-                i.file == project(':metadata-ingestion-modules').buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     buildArgs([DOCKER_VERSION: version,
-               RELEASE_VERSION: version.replace('-SNAPSHOT', '').replace('v', '').replace('-slim', '')])
+               RELEASE_VERSION: version.replace('-SNAPSHOT', '').replace('v', '').replace("-slim", '')])
 }
 tasks.getByName('docker').dependsOn(['build',
                                      ':docker:datahub-ingestion-base:docker',
diff --git a/docker/datahub-ingestion/pyspark_jars.sh b/docker/datahub-ingestion/pyspark_jars.sh
index ecd24e78c4105..ab4b223f0358a 100755
--- a/docker/datahub-ingestion/pyspark_jars.sh
+++ b/docker/datahub-ingestion/pyspark_jars.sh
@@ -2,21 +2,33 @@
 
 set -ex
 
-HADOOP_CLIENT_DEPENDENCY="${HADOOP_CLIENT_DEPENDENCY:-org.apache.hadoop:hadoop-client:3.3.6}"
-ZOOKEEPER_DEPENDENCY="${ZOOKEEPER_DEPENDENCY:-org.apache.zookeeper:zookeeper:3.7.2}"
 PYSPARK_JARS="$(python -m site --user-site)/pyspark/jars"
 
-# Remove conflicting versions
-echo "Removing version conflicts from $PYSPARK_JARS"
-CONFLICTS="zookeeper hadoop- slf4j-"
-for jar in $CONFLICTS; do
-  rm "$PYSPARK_JARS/$jar"*.jar
-done
+function replace_jar {
+  JAR_PREFIX=$1
+  TRANSITIVE=$2
+  DEPENDENCY=$3
 
-# Fetch dependencies
-mvn dependency:get -Dtransitive=true -Dartifact="$HADOOP_CLIENT_DEPENDENCY"
-mvn dependency:get -Dtransitive=true -Dartifact="$ZOOKEEPER_DEPENDENCY"
+  echo "Removing version conflicts for $PYSPARK_JARS/$JAR_PREFIX*.jar"
+  ls "$PYSPARK_JARS/$JAR_PREFIX"*.jar || true
+  rm "$PYSPARK_JARS/$JAR_PREFIX"*.jar || true
+  rm -r "$HOME/.m2" || true
 
-# Move to pyspark location
-echo "Moving jars to $PYSPARK_JARS"
-find "$HOME/.m2" -type f -name "*.jar" -exec mv {} "$PYSPARK_JARS/" \;
+  if [ ! -z "$DEPENDENCY" ]; then
+    echo "Resolving $DEPENDENCY"
+    mvn dependency:get -Dtransitive=$TRANSITIVE -Dartifact="$DEPENDENCY" >/dev/null
+
+    echo "Moving jars to $PYSPARK_JARS"
+    find "$HOME/.m2" -type f -name "$JAR_PREFIX*.jar" -exec echo "{}" \;
+    find "$HOME/.m2" -type f -name "$JAR_PREFIX*.jar" -exec cp {} "$PYSPARK_JARS/" \;
+  fi
+}
+
+replace_jar "zookeeper-" "false" "${ZOOKEEPER_DEPENDENCY:-org.apache.zookeeper:zookeeper:3.7.2}"
+replace_jar "hadoop-client-" "true" "${HADOOP_CLIENT_API_DEPENDENCY:-org.apache.hadoop:hadoop-client-api:3.3.6}"
+replace_jar "hadoop-client-" "true" "${HADOOP_CLIENT_RUNTIME_DEPENDENCY:-org.apache.hadoop:hadoop-client-runtime:3.3.6}"
+replace_jar "hadoop-yarn-" "true" "${HADOOP_YARN_DEPENDENCY:-org.apache.hadoop:hadoop-yarn-server-web-proxy:3.3.6}"
+replace_jar "snappy-java-" "false" "${SNAPPY_JAVA_DEPENDENCY:-org.xerial.snappy:snappy-java:1.1.10.5}"
+replace_jar "libthrift-" "false" "${LIBTHRIFT_DEPENDENCY:-org.apache.thrift:libthrift:0.19.0}"
+replace_jar "ivy-" "false" "${IVY_DEPENDENCY:-org.apache.ivy:ivy:2.5.2}"
+replace_jar "parquet-jackson-" "false" "${PARQUET_JACKSON_DEPENDENCY:-org.apache.parquet:parquet-jackson:1.13.1}"
diff --git a/docker/elasticsearch-setup/build.gradle b/docker/elasticsearch-setup/build.gradle
index ffee3b9c65cf4..ac935ca42fd12 100644
--- a/docker/elasticsearch-setup/build.gradle
+++ b/docker/elasticsearch-setup/build.gradle
@@ -15,10 +15,11 @@ docker {
     version "v${version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include "docker/${docker_dir}/*"
-        include "metadata-service/restli-servlet-impl/src/main/resources/index/**"
+        include 'metadata-service/restli-servlet-impl/src/main/resources/index/**'
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/docker/kafka-setup/build.gradle b/docker/kafka-setup/build.gradle
index 573ef21c88bf9..25f9847190de3 100644
--- a/docker/kafka-setup/build.gradle
+++ b/docker/kafka-setup/build.gradle
@@ -15,9 +15,10 @@ docker {
     version "v${version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include "docker/${docker_dir}/*"
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/docker/mysql-setup/build.gradle b/docker/mysql-setup/build.gradle
index 0d8941cce4833..1598866914c0e 100644
--- a/docker/mysql-setup/build.gradle
+++ b/docker/mysql-setup/build.gradle
@@ -16,9 +16,10 @@ docker {
     version "v${version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include "docker/${docker_dir}/*"
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/docker/postgres-setup/build.gradle b/docker/postgres-setup/build.gradle
index 8a026be09d2b4..e24e206c99145 100644
--- a/docker/postgres-setup/build.gradle
+++ b/docker/postgres-setup/build.gradle
@@ -16,9 +16,10 @@ docker {
     version "v${version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include "docker/${docker_dir}/*"
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle
index 51c758f434328..5e735e118493c 100644
--- a/metadata-jobs/mae-consumer-job/build.gradle
+++ b/metadata-jobs/mae-consumer-job/build.gradle
@@ -45,11 +45,12 @@ docker {
     dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
     files bootJar.outputs.files
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include 'docker/monitoring/*'
         include "docker/${docker_repo}/*"
         include 'metadata-models/src/main/resources/*'
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/metadata-jobs/mce-consumer-job/build.gradle b/metadata-jobs/mce-consumer-job/build.gradle
index daf41a1e0303e..ef042188bc3d8 100644
--- a/metadata-jobs/mce-consumer-job/build.gradle
+++ b/metadata-jobs/mce-consumer-job/build.gradle
@@ -56,11 +56,12 @@ docker {
     dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
     files bootJar.outputs.files
     files fileTree(rootProject.projectDir) {
+        include '.dockerignore'
         include 'docker/monitoring/*'
         include "docker/${docker_repo}/*"
         include 'metadata-models/src/main/resources/*'
     }.exclude {
-        i -> i.file.isHidden() || i.file == buildDir
+        i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
     }
     tag("Debug", "${docker_registry}/${docker_repo}:debug")
 
diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle
index 54e95fdcfe579..35730ad6dfa9f 100644
--- a/metadata-service/war/build.gradle
+++ b/metadata-service/war/build.gradle
@@ -70,11 +70,12 @@ docker {
   dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
   files war.outputs.files
   files fileTree(rootProject.projectDir) {
+    include '.dockerignore'
     include 'docker/monitoring/*'
     include "docker/${docker_repo}/*"
     include 'metadata-models/src/main/resources/*'
   }.exclude {
-    i -> i.file.isHidden() || i.file == buildDir
+    i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
   tag("Debug", "${docker_registry}/${docker_repo}:debug")
 

From 19aa21506886692ff221f1b859e0633df995fb43 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 13 Nov 2023 19:00:30 -0500
Subject: [PATCH 203/259] feat(ingest/dbt): dbt column-level lineage (#8991)

---
 .../airflow-plugin/setup.py                   |   2 +-
 metadata-ingestion/setup.py                   |   4 +-
 .../api/incremental_lineage_helper.py         |  16 +-
 .../ingestion/source/dbt/dbt_common.py        | 499 ++++++++++--
 .../datahub/ingestion/source/dbt/dbt_core.py  |   3 +-
 .../src/datahub/utilities/sqlglot_lineage.py  |  81 +-
 .../src/datahub/utilities/topological_sort.py |  49 ++
 .../dbt_enabled_with_schemas_mces_golden.json | 255 ++++--
 .../dbt_test_column_meta_mapping_golden.json  | 283 +++++--
 .../dbt/dbt_test_events_golden.json           | 731 ++++++++++++------
 ...th_complex_owner_patterns_mces_golden.json | 240 +++++-
 ...th_data_platform_instance_mces_golden.json | 246 ++++--
 ...h_non_incremental_lineage_mces_golden.json | 198 ++++-
 ..._target_platform_instance_mces_golden.json | 246 ++++--
 .../tests/integration/dbt/test_dbt.py         |   8 +-
 .../test_snowflake_cte_name_collision.json    |  47 ++
 ...owflake_full_table_name_col_reference.json |  55 ++
 .../goldens/test_snowflake_unused_cte.json    |  39 +
 .../unit/sql_parsing/test_sqlglot_lineage.py  | 128 ++-
 .../tests/unit/test_topological_sort.py       |  33 +
 20 files changed, 2550 insertions(+), 613 deletions(-)
 create mode 100644 metadata-ingestion/src/datahub/utilities/topological_sort.py
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_cte_name_collision.json
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_full_table_name_col_reference.json
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_unused_cte.json
 create mode 100644 metadata-ingestion/tests/unit/test_topological_sort.py

diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py
index e88fc870cb333..838322f83833b 100644
--- a/metadata-ingestion-modules/airflow-plugin/setup.py
+++ b/metadata-ingestion-modules/airflow-plugin/setup.py
@@ -14,7 +14,7 @@ def get_long_description():
     return pathlib.Path(os.path.join(root, "README.md")).read_text()
 
 
-_version = package_metadata["__version__"]
+_version: str = package_metadata["__version__"]
 _self_pin = f"=={_version}" if not _version.endswith("dev0") else ""
 
 
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index f3782abe576d3..ebe180703051f 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -305,8 +305,8 @@
     "datahub-lineage-file": set(),
     "datahub-business-glossary": set(),
     "delta-lake": {*data_lake_profiling, *delta_lake},
-    "dbt": {"requests"} | aws_common,
-    "dbt-cloud": {"requests"},
+    "dbt": {"requests"} | sqlglot_lib | aws_common,
+    "dbt-cloud": {"requests"} | sqlglot_lib,
     "druid": sql_common | {"pydruid>=0.6.2"},
     "dynamodb": aws_common,
     # Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws
diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
index 945b201ca5758..479486ce22899 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py
@@ -15,7 +15,7 @@
 from datahub.specific.dataset import DatasetPatchBuilder
 
 
-def _convert_upstream_lineage_to_patch(
+def convert_upstream_lineage_to_patch(
     urn: str,
     aspect: UpstreamLineageClass,
     system_metadata: Optional[SystemMetadataClass],
@@ -86,16 +86,11 @@ def _merge_upstream_lineage(
 
 
 def _lineage_wu_via_read_modify_write(
-    graph: Optional[DataHubGraph],
+    graph: DataHubGraph,
     urn: str,
     aspect: UpstreamLineageClass,
     system_metadata: Optional[SystemMetadataClass],
 ) -> MetadataWorkUnit:
-    if graph is None:
-        raise ValueError(
-            "Failed to handle incremental lineage, DataHubGraph is missing. "
-            "Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
-        )
     gms_aspect = graph.get_aspect(urn, UpstreamLineageClass)
     if gms_aspect:
         new_aspect = _merge_upstream_lineage(aspect, gms_aspect)
@@ -131,11 +126,16 @@ def auto_incremental_lineage(
                     yield wu
 
             if lineage_aspect.fineGrainedLineages:
+                if graph is None:
+                    raise ValueError(
+                        "Failed to handle incremental lineage, DataHubGraph is missing. "
+                        "Use `datahub-rest` sink OR provide `datahub-api` config in recipe. "
+                    )
                 yield _lineage_wu_via_read_modify_write(
                     graph, urn, lineage_aspect, wu.metadata.systemMetadata
                 )
             elif lineage_aspect.upstreams:
-                yield _convert_upstream_lineage_to_patch(
+                yield convert_upstream_lineage_to_patch(
                     urn, lineage_aspect, wu.metadata.systemMetadata
                 )
         else:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 76cb82aaa5b4b..94df0a4f8a166 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -1,3 +1,4 @@
+import itertools
 import logging
 import re
 from abc import abstractmethod
@@ -30,6 +31,9 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.incremental_lineage_helper import (
+    convert_upstream_lineage_to_patch,
+)
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import DatasetSubTypes
@@ -67,6 +71,9 @@
 )
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageTypeClass,
+    FineGrainedLineage,
+    FineGrainedLineageDownstreamType,
+    FineGrainedLineageUpstreamType,
     UpstreamClass,
     UpstreamLineage,
 )
@@ -100,9 +107,17 @@
     UpstreamLineageClass,
     ViewPropertiesClass,
 )
-from datahub.specific.dataset import DatasetPatchBuilder
 from datahub.utilities.mapping import Constants, OperationProcessor
+from datahub.utilities.sqlglot_lineage import (
+    SchemaInfo,
+    SchemaResolver,
+    SqlParsingDebugInfo,
+    SqlParsingResult,
+    detach_ctes,
+    sqlglot_lineage,
+)
 from datahub.utilities.time import datetime_to_ts_millis
+from datahub.utilities.topological_sort import topological_sort
 
 logger = logging.getLogger(__name__)
 DBT_PLATFORM = "dbt"
@@ -280,10 +295,19 @@ class DBTCommonConfig(
         default=False,
         description="When enabled, dbt test warnings will be treated as failures.",
     )
-    # override fault value to True.
+    infer_dbt_schemas: bool = Field(
+        default=True,
+        description="When enabled, schemas will be inferred from the dbt node definition.",
+    )
+    include_column_lineage: bool = Field(
+        default=False,
+        description="When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. "
+        "If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.",
+    )
+    # override default value to True.
     incremental_lineage: bool = Field(
         default=True,
-        description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
+        description="When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run.",
     )
 
     @validator("target_platform")
@@ -340,6 +364,17 @@ def meta_mapping_validator(
                         )
         return meta_mapping
 
+    @validator("include_column_lineage")
+    def validate_include_column_lineage(
+        cls, include_column_lineage: bool, values: Dict
+    ) -> bool:
+        if include_column_lineage and not values.get("infer_dbt_schemas"):
+            raise ValueError(
+                "`infer_dbt_schemas` must be enabled to use `include_column_lineage`"
+            )
+
+        return include_column_lineage
+
 
 @dataclass
 class DBTColumn:
@@ -352,6 +387,16 @@ class DBTColumn:
     meta: Dict[str, Any] = field(default_factory=dict)
     tags: List[str] = field(default_factory=list)
 
+    datahub_data_type: Optional[SchemaFieldDataType] = None
+
+
+@dataclass
+class DBTColumnLineageInfo:
+    upstream_dbt_name: str
+
+    upstream_col: str
+    downstream_col: str
+
 
 @dataclass
 class DBTNode:
@@ -383,7 +428,9 @@ class DBTNode:
     owner: Optional[str]
 
     columns: List[DBTColumn] = field(default_factory=list)
-    upstream_nodes: List[str] = field(default_factory=list)
+    upstream_nodes: List[str] = field(default_factory=list)  # list of upstream dbt_name
+    upstream_cll: List[DBTColumnLineageInfo] = field(default_factory=list)
+    cll_debug_info: Optional[SqlParsingDebugInfo] = None
 
     meta: Dict[str, Any] = field(default_factory=dict)
     query_tag: Dict[str, Any] = field(default_factory=dict)
@@ -394,17 +441,23 @@ class DBTNode:
     test_info: Optional["DBTTest"] = None  # only populated if node_type == 'test'
     test_result: Optional["DBTTestResult"] = None
 
+    @staticmethod
+    def _join_parts(parts: List[Optional[str]]) -> str:
+        joined = ".".join([part for part in parts if part])
+        assert joined
+        return joined
+
     def get_db_fqn(self) -> str:
-        if self.database:
-            fqn = f"{self.database}.{self.schema}.{self.name}"
-        else:
-            fqn = f"{self.schema}.{self.name}"
+        # Database might be None, but schema and name should always be present.
+        fqn = self._join_parts([self.database, self.schema, self.name])
         return fqn.replace('"', "")
 
     def get_urn(
         self,
         target_platform: str,
         env: str,
+        # If target_platform = dbt, this is the dbt platform instance.
+        # Otherwise, it's the target platform instance.
         data_platform_instance: Optional[str],
     ) -> str:
         db_fqn = self.get_db_fqn()
@@ -417,6 +470,80 @@ def get_urn(
             env=env,
         )
 
+    def is_ephemeral_model(self) -> bool:
+        return self.materialization == "ephemeral"
+
+    def get_fake_ephemeral_table_name(self) -> str:
+        assert self.is_ephemeral_model()
+
+        # Similar to get_db_fqn.
+        fqn = self._join_parts(
+            [self.database, self.schema, f"__datahub__dbt__ephemeral__{self.name}"]
+        )
+        return fqn.replace('"', "")
+
+    def get_urn_for_upstream_lineage(
+        self,
+        dbt_platform_instance: Optional[str],
+        target_platform: str,
+        target_platform_instance: Optional[str],
+        env: str,
+    ) -> str:
+        """
+        Get the urn to use when referencing this node in a dbt node's upstream lineage.
+
+        If the node is a source or an ephemeral dbt node, we should point at the dbt node.
+        Otherwise, the node is materialized in the target platform, and so lineage should
+        point there.
+        """
+        # TODO: This logic shouldn't live in the DBTNode class. It should be moved to the source.
+
+        platform_value = DBT_PLATFORM
+        platform_instance_value = dbt_platform_instance
+
+        materialized = self.materialization
+        if materialized in {
+            "view",
+            "materialized_view",
+            "table",
+            "incremental",
+            "snapshot",
+        }:
+            # upstream urns point to the target platform
+            platform_value = target_platform
+            platform_instance_value = target_platform_instance
+
+        return self.get_urn(
+            target_platform=platform_value,
+            env=env,
+            data_platform_instance=platform_instance_value,
+        )
+
+    @property
+    def exists_in_target_platform(self):
+        return not (self.is_ephemeral_model() or self.node_type == "test")
+
+    def columns_setdefault(self, schema_fields: List[SchemaField]) -> None:
+        """
+        Update the column list if they are not already set.
+        """
+
+        if self.columns:
+            # If we already have columns, don't overwrite them.
+            return
+
+        self.columns = [
+            DBTColumn(
+                name=schema_field.fieldPath,
+                comment="",
+                description="",
+                index=i,
+                data_type=schema_field.nativeDataType,
+                datahub_data_type=schema_field.type,
+            )
+            for i, schema_field in enumerate(schema_fields)
+        ]
+
 
 def get_custom_properties(node: DBTNode) -> Dict[str, str]:
     # initialize custom properties to node's meta props
@@ -442,6 +569,31 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
     return custom_properties
 
 
+def _get_dbt_cte_names(name: str, target_platform: str) -> List[str]:
+    # Match the dbt CTE naming scheme:
+    # The default is defined here https://github.com/dbt-labs/dbt-core/blob/4122f6c308c88be4a24c1ea490802239a4c1abb8/core/dbt/adapters/base/relation.py#L222
+    # However, since this PR https://github.com/dbt-labs/dbt-core/pull/2712, it's also possible
+    # for adapters to override this default. Only a handful actually do though:
+    # https://github.com/search?type=code&q=add_ephemeral_prefix+path:/%5Edbt%5C/adapters%5C//
+
+    # Regardless, we need to keep the original name to work with older dbt versions.
+    default_cte_name = f"__dbt__cte__{name}"
+
+    adapter_cte_names = {
+        "hive": f"tmp__dbt__cte__{name}",
+        "oracle": f"dbt__cte__{name}__",
+        "netezza": f"dbt__cte__{name}",
+        "exasol": f"dbt__CTE__{name}",
+        "db2": f"DBT_CTE__{name}",  # ibm db2
+    }
+
+    cte_names = [default_cte_name]
+    if target_platform in adapter_cte_names:
+        cte_names.append(adapter_cte_names[target_platform])
+
+    return cte_names
+
+
 def get_upstreams(
     upstreams: List[str],
     all_nodes: Dict[str, DBTNode],
@@ -462,21 +614,12 @@ def get_upstreams(
         upstream_manifest_node = all_nodes[upstream]
 
         # This logic creates lineages among dbt nodes.
-        platform_value = DBT_PLATFORM
-        platform_instance_value = platform_instance
-
-        materialized = upstream_manifest_node.materialization
-
-        if materialized in {"view", "table", "incremental", "snapshot"}:
-            # upstream urns point to the target platform
-            platform_value = target_platform
-            platform_instance_value = target_platform_instance
-
         upstream_urns.append(
-            upstream_manifest_node.get_urn(
-                platform_value,
-                environment,
-                platform_instance_value,
+            upstream_manifest_node.get_urn_for_upstream_lineage(
+                dbt_platform_instance=platform_instance,
+                target_platform=target_platform,
+                target_platform_instance=target_platform_instance,
+                env=environment,
             )
         )
     return upstream_urns
@@ -553,7 +696,7 @@ def get_column_type(
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
-@capability(SourceCapability.USAGE_STATS, "", supported=False)
+@capability(SourceCapability.LINEAGE_FINE, "Enabled using `include_column_lineage`")
 class DBTSourceBase(StatefulIngestionSourceBase):
     def __init__(self, config: DBTCommonConfig, ctx: PipelineContext, platform: str):
         super().__init__(config, ctx)
@@ -614,9 +757,10 @@ def create_test_entity_mcps(
                 target_platform=self.config.target_platform,
                 target_platform_instance=self.config.target_platform_instance,
                 environment=self.config.env,
-                platform_instance=None,
+                platform_instance=self.config.platform_instance,
             )
 
+            # In case a dbt test depends on multiple tables, we create separate assertions for each.
             for upstream_urn in sorted(upstream_urns):
                 if self.config.entities_enabled.can_emit_node_type("test"):
                     yield make_assertion_from_test(
@@ -651,23 +795,24 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        if self.config.write_semantics == "PATCH" and not self.ctx.graph:
-            raise ConfigurationError(
-                "With PATCH semantics, dbt source requires a datahub_api to connect to. "
-                "Consider using the datahub-rest sink or provide a datahub_api: configuration on your ingestion recipe."
-            )
+        if self.config.write_semantics == "PATCH":
+            self.ctx.require_graph("Using dbt with write_semantics=PATCH")
 
         all_nodes, additional_custom_props = self.load_nodes()
 
         all_nodes_map = {node.dbt_name: node for node in all_nodes}
-        nodes = self.filter_nodes(all_nodes)
-
         additional_custom_props_filtered = {
             key: value
             for key, value in additional_custom_props.items()
             if value is not None
         }
 
+        # We need to run this before filtering nodes, because the info generated
+        # for a filtered node may be used by an unfiltered node.
+        # NOTE: This method mutates the DBTNode objects directly.
+        self._infer_schemas_and_update_cll(all_nodes_map)
+
+        nodes = self._filter_nodes(all_nodes)
         non_test_nodes = [
             dataset_node for dataset_node in nodes if dataset_node.node_type != "test"
         ]
@@ -695,7 +840,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             all_nodes_map,
         )
 
-    def filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
+    def _filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
         nodes = []
         for node in all_nodes:
             key = node.dbt_name
@@ -707,6 +852,193 @@ def filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
 
         return nodes
 
+    @staticmethod
+    def _to_schema_info(schema_fields: List[SchemaField]) -> SchemaInfo:
+        return {column.fieldPath: column.nativeDataType for column in schema_fields}
+
+    def _infer_schemas_and_update_cll(self, all_nodes_map: Dict[str, DBTNode]) -> None:
+        """Annotate the DBTNode objects with schema information and column-level lineage.
+
+        Note that this mutates the DBTNode objects directly.
+
+        This method does the following:
+        1. Iterate over the dbt nodes in topological order.
+        2. For each node, either load the schema from the graph or from the dbt catalog info.
+           We also add this schema to the schema resolver.
+        3. Run sql parser to infer the schema + generate column lineage.
+        4. Write the schema and column lineage back to the DBTNode object.
+        5. If we haven't already added the node's schema to the schema resolver, do that.
+        """
+
+        if not self.config.infer_dbt_schemas:
+            if self.config.include_column_lineage:
+                raise ConfigurationError(
+                    "`infer_dbt_schemas` must be enabled to use `include_column_lineage`"
+                )
+            return
+
+        graph = self.ctx.graph
+
+        schema_resolver = SchemaResolver(
+            platform=self.config.target_platform,
+            platform_instance=self.config.target_platform_instance,
+            env=self.config.env,
+        )
+
+        target_platform_urn_to_dbt_name: Dict[str, str] = {}
+
+        # Iterate over the dbt nodes in topological order.
+        # This ensures that we process upstream nodes before downstream nodes.
+        for dbt_name in topological_sort(
+            list(all_nodes_map.keys()),
+            edges=list(
+                (upstream, node.dbt_name)
+                for node in all_nodes_map.values()
+                for upstream in node.upstream_nodes
+            ),
+        ):
+            node = all_nodes_map[dbt_name]
+
+            target_node_urn = None
+            should_fetch_target_node_schema = False
+            if node.exists_in_target_platform:
+                target_node_urn = node.get_urn(
+                    self.config.target_platform,
+                    self.config.env,
+                    self.config.target_platform_instance,
+                )
+                should_fetch_target_node_schema = True
+            elif node.is_ephemeral_model():
+                # For ephemeral nodes, we "pretend" that they exist in the target platform
+                # for schema resolution purposes.
+                target_node_urn = mce_builder.make_dataset_urn_with_platform_instance(
+                    platform=self.config.target_platform,
+                    name=node.get_fake_ephemeral_table_name(),
+                    platform_instance=self.config.target_platform_instance,
+                    env=self.config.env,
+                )
+            if target_node_urn:
+                target_platform_urn_to_dbt_name[target_node_urn] = node.dbt_name
+
+            # Our schema resolver preference is:
+            # 1. graph
+            # 2. dbt catalog
+            # 3. inferred
+            # Exception: if convert_column_urns_to_lowercase is enabled, swap 1 and 2.
+            # Cases 1 and 2 are handled here, and case 3 is handled after schema inference has occurred.
+            schema_fields: Optional[List[SchemaField]] = None
+
+            # Fetch the schema from the graph.
+            if target_node_urn and should_fetch_target_node_schema and graph:
+                schema_metadata = graph.get_aspect(target_node_urn, SchemaMetadata)
+                if schema_metadata:
+                    schema_fields = schema_metadata.fields
+
+            # Otherwise, load the schema from the dbt catalog.
+            # Note that this might get the casing wrong relative to DataHub, but
+            # has a more up-to-date column list.
+            if node.columns and (
+                not schema_fields or self.config.convert_column_urns_to_lowercase
+            ):
+                schema_fields = [
+                    SchemaField(
+                        fieldPath=column.name.lower()
+                        if self.config.convert_column_urns_to_lowercase
+                        else column.name,
+                        type=column.datahub_data_type
+                        or SchemaFieldDataType(type=NullTypeClass()),
+                        nativeDataType=column.data_type,
+                    )
+                    for column in node.columns
+                ]
+
+            # Add the node to the schema resolver, so that we can get column
+            # casing to match the upstream platform.
+            added_to_schema_resolver = False
+            if target_node_urn and schema_fields:
+                schema_resolver.add_raw_schema_info(
+                    target_node_urn, self._to_schema_info(schema_fields)
+                )
+                added_to_schema_resolver = True
+
+            # Run sql parser to infer the schema + generate column lineage.
+            sql_result = None
+            if node.compiled_code:
+                try:
+                    # Add CTE stops based on the upstreams list.
+                    preprocessed_sql = detach_ctes(
+                        node.compiled_code,
+                        platform=schema_resolver.platform,
+                        cte_mapping={
+                            cte_name: upstream_node.get_fake_ephemeral_table_name()
+                            for upstream_node in [
+                                all_nodes_map[upstream_node_name]
+                                for upstream_node_name in node.upstream_nodes
+                                if upstream_node_name in all_nodes_map
+                            ]
+                            if upstream_node.is_ephemeral_model()
+                            for cte_name in _get_dbt_cte_names(
+                                upstream_node.name, schema_resolver.platform
+                            )
+                        },
+                    )
+                except Exception as e:
+                    sql_result = SqlParsingResult.make_from_error(e)
+                else:
+                    sql_result = sqlglot_lineage(
+                        preprocessed_sql, schema_resolver=schema_resolver
+                    )
+
+            # Save the column lineage.
+            if self.config.include_column_lineage and sql_result:
+                # We only save the debug info here. We're report errors based on it later, after
+                # applying the configured node filters.
+                node.cll_debug_info = sql_result.debug_info
+
+                if sql_result.column_lineage:
+                    node.upstream_cll = [
+                        DBTColumnLineageInfo(
+                            upstream_dbt_name=target_platform_urn_to_dbt_name[
+                                upstream_column.table
+                            ],
+                            upstream_col=upstream_column.column,
+                            downstream_col=column_lineage_info.downstream.column,
+                        )
+                        for column_lineage_info in sql_result.column_lineage
+                        for upstream_column in column_lineage_info.upstreams
+                        # Only include the CLL if the table in in the upstream list.
+                        if target_platform_urn_to_dbt_name.get(upstream_column.table)
+                        in node.upstream_nodes
+                    ]
+
+            # If we didn't fetch the schema from the graph, use the inferred schema.
+            inferred_schema_fields = None
+            if sql_result and sql_result.column_lineage:
+                inferred_schema_fields = [
+                    SchemaField(
+                        fieldPath=column_lineage.downstream.column,
+                        type=column_lineage.downstream.column_type
+                        or SchemaFieldDataType(type=NullTypeClass()),
+                        nativeDataType=column_lineage.downstream.native_column_type
+                        or "",
+                    )
+                    for column_lineage in sql_result.column_lineage
+                ]
+
+            # Conditionally add the inferred schema to the schema resolver.
+            if (
+                not added_to_schema_resolver
+                and target_node_urn
+                and inferred_schema_fields
+            ):
+                schema_resolver.add_raw_schema_info(
+                    target_node_urn, self._to_schema_info(inferred_schema_fields)
+                )
+
+            # Save the inferred schema fields into the dbt node.
+            if inferred_schema_fields:
+                node.columns_setdefault(inferred_schema_fields)
+
     def create_platform_mces(
         self,
         dbt_nodes: List[DBTNode],
@@ -762,7 +1094,7 @@ def create_platform_mces(
                 )  # mutates meta_aspects
 
             if mce_platform == DBT_PLATFORM:
-                aspects = self._generate_base_aspects(
+                aspects = self._generate_base_dbt_aspects(
                     node, additional_custom_props_filtered, mce_platform, meta_aspects
                 )
 
@@ -786,7 +1118,7 @@ def create_platform_mces(
             else:
                 # We are creating empty node for platform and only add lineage/keyaspect.
                 aspects = []
-                if node.materialization == "ephemeral" or node.node_type == "test":
+                if not node.exists_in_target_platform:
                     continue
 
                 # This code block is run when we are generating entities of platform type.
@@ -799,19 +1131,15 @@ def create_platform_mces(
                         self.config.platform_instance,
                     )
                     upstreams_lineage_class = get_upstream_lineage([upstream_dbt_urn])
-                    if self.config.incremental_lineage:
-                        patch_builder: DatasetPatchBuilder = DatasetPatchBuilder(
-                            urn=node_datahub_urn
+                    if not is_primary_source and self.config.incremental_lineage:
+                        # We only generate incremental lineage for non-dbt nodes.
+                        wu = convert_upstream_lineage_to_patch(
+                            urn=node_datahub_urn,
+                            aspect=upstreams_lineage_class,
+                            system_metadata=None,
                         )
-                        for upstream in upstreams_lineage_class.upstreams:
-                            patch_builder.add_upstream_lineage(upstream)
-
-                        for mcp in patch_builder.build():
-                            yield MetadataWorkUnit(
-                                id=f"upstreamLineage-for-{node_datahub_urn}",
-                                mcp_raw=mcp,
-                                is_primary_source=is_primary_source,
-                            )
+                        wu.is_primary_source = is_primary_source
+                        yield wu
                     else:
                         aspects.append(upstreams_lineage_class)
 
@@ -918,7 +1246,7 @@ def _create_view_properties_aspect(
         )
         return view_properties
 
-    def _generate_base_aspects(
+    def _generate_base_dbt_aspects(
         self,
         node: DBTNode,
         additional_custom_props_filtered: Dict[str, str],
@@ -926,8 +1254,7 @@ def _generate_base_aspects(
         meta_aspects: Dict[str, Any],
     ) -> List[Any]:
         """
-        There are some common aspects that get generated for both dbt node and platform node depending on whether dbt
-        node creation is enabled or not.
+        Some common aspects that get generated for dbt nodes.
         """
 
         # create an empty list of aspects and keep adding to it. Initializing with Any to avoid a
@@ -987,6 +1314,8 @@ def get_schema_metadata(
             self.config.strip_user_ids_from_email,
         )
 
+        # TODO if infer_dbt_schemas, load from saved schemas too
+
         canonical_schema: List[SchemaField] = []
         for column in node.columns:
             description = None
@@ -1034,7 +1363,8 @@ def get_schema_metadata(
             field = SchemaField(
                 fieldPath=field_name,
                 nativeDataType=column.data_type,
-                type=get_column_type(
+                type=column.datahub_data_type
+                or get_column_type(
                     report, node.dbt_name, column.data_type, node.dbt_adapter
                 ),
                 description=description,
@@ -1140,27 +1470,78 @@ def _create_lineage_aspect_for_dbt_node(
         """
         This method creates lineage amongst dbt nodes. A dbt node can be linked to other dbt nodes or a platform node.
         """
-        upstream_urns = get_upstreams(
-            node.upstream_nodes,
-            all_nodes_map,
-            self.config.target_platform,
-            self.config.target_platform_instance,
-            self.config.env,
-            self.config.platform_instance,
-        )
 
         # if a node is of type source in dbt, its upstream lineage should have the corresponding table/view
         # from the platform. This code block is executed when we are generating entities of type "dbt".
         if node.node_type == "source":
-            upstream_urns.append(
+            upstream_urns = [
                 node.get_urn(
                     self.config.target_platform,
                     self.config.env,
                     self.config.target_platform_instance,
                 )
+            ]
+            cll = None
+        else:
+            upstream_urns = get_upstreams(
+                node.upstream_nodes,
+                all_nodes_map,
+                self.config.target_platform,
+                self.config.target_platform_instance,
+                self.config.env,
+                self.config.platform_instance,
+            )
+
+            node_urn = node.get_urn(
+                target_platform=DBT_PLATFORM,
+                env=self.config.env,
+                data_platform_instance=self.config.platform_instance,
             )
+
+            def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str:
+                return all_nodes_map[dbt_name].get_urn_for_upstream_lineage(
+                    dbt_platform_instance=self.config.platform_instance,
+                    target_platform=self.config.target_platform,
+                    target_platform_instance=self.config.target_platform_instance,
+                    env=self.config.env,
+                )
+
+            if node.cll_debug_info and node.cll_debug_info.error:
+                self.report.report_warning(
+                    node.dbt_name,
+                    f"Error parsing column lineage: {node.cll_debug_info.error}",
+                )
+            cll = [
+                FineGrainedLineage(
+                    upstreamType=FineGrainedLineageUpstreamType.FIELD_SET,
+                    downstreamType=FineGrainedLineageDownstreamType.FIELD_SET,
+                    upstreams=[
+                        mce_builder.make_schema_field_urn(
+                            _translate_dbt_name_to_upstream_urn(
+                                upstream_column.upstream_dbt_name
+                            ),
+                            upstream_column.upstream_col,
+                        )
+                        for upstream_column in upstreams
+                    ],
+                    downstreams=[
+                        mce_builder.make_schema_field_urn(node_urn, downstream)
+                    ],
+                    confidenceScore=node.cll_debug_info.confidence
+                    if node.cll_debug_info
+                    else None,
+                )
+                for downstream, upstreams in itertools.groupby(
+                    node.upstream_cll, lambda x: x.downstream_col
+                )
+            ]
+
         if upstream_urns:
             upstreams_lineage_class = get_upstream_lineage(upstream_urns)
+
+            if self.config.include_column_lineage and cll:
+                upstreams_lineage_class.fineGrainedLineages = cll
+
             return upstreams_lineage_class
         return None
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
index dc3a84847beb2..a7703b203bcee 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
@@ -171,7 +171,8 @@ def extract_dbt_entities(
         catalog_type = None
 
         if catalog_node is None:
-            if materialization != "test":
+            if materialization not in {"test", "ephemeral"}:
+                # Test and ephemeral nodes will never show up in the catalog.
                 report.report_warning(
                     key,
                     f"Entity {key} ({name}) is in manifest but missing from catalog",
diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index cdffb684d958e..d1209f3ec7b75 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -260,6 +260,16 @@ class SqlParsingResult(_ParserBaseModel):
         exclude=True,
     )
 
+    @classmethod
+    def make_from_error(cls, error: Exception) -> "SqlParsingResult":
+        return cls(
+            in_tables=[],
+            out_tables=[],
+            debug_info=SqlParsingDebugInfo(
+                table_error=error,
+            ),
+        )
+
 
 def _parse_statement(sql: sqlglot.exp.ExpOrStr, dialect: str) -> sqlglot.Expression:
     statement: sqlglot.Expression = sqlglot.maybe_parse(
@@ -1154,14 +1164,60 @@ def sqlglot_lineage(
             default_schema=default_schema,
         )
     except Exception as e:
-        return SqlParsingResult(
-            in_tables=[],
-            out_tables=[],
-            column_lineage=None,
-            debug_info=SqlParsingDebugInfo(
-                table_error=e,
-            ),
-        )
+        return SqlParsingResult.make_from_error(e)
+
+
+def detach_ctes(
+    sql: sqlglot.exp.ExpOrStr, platform: str, cte_mapping: Dict[str, str]
+) -> sqlglot.exp.Expression:
+    """Replace CTE references with table references.
+
+    For example, with cte_mapping = {"__cte_0": "_my_cte_table"}, the following SQL
+
+    WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 ON table2.id = __cte_0.id
+
+    is transformed into
+
+    WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN _my_cte_table ON table2.id = _my_cte_table.id
+
+    Note that the original __cte_0 definition remains in the query, but is simply not referenced.
+    The query optimizer should be able to remove it.
+
+    This method makes a major assumption: that no other table/column has the same name as a
+    key in the cte_mapping.
+    """
+
+    dialect = _get_dialect(platform)
+    statement = _parse_statement(sql, dialect=dialect)
+
+    def replace_cte_refs(node: sqlglot.exp.Expression) -> sqlglot.exp.Expression:
+        if (
+            isinstance(node, sqlglot.exp.Identifier)
+            and node.parent
+            and not isinstance(node.parent.parent, sqlglot.exp.CTE)
+            and node.name in cte_mapping
+        ):
+            full_new_name = cte_mapping[node.name]
+            table_expr = sqlglot.maybe_parse(
+                full_new_name, dialect=dialect, into=sqlglot.exp.Table
+            )
+
+            # We expect node.parent to be a Table or Column.
+            # Either way, it should support catalog/db/name.
+            parent = node.parent
+
+            if "catalog" in parent.arg_types:
+                parent.set("catalog", table_expr.catalog)
+            if "db" in parent.arg_types:
+                parent.set("db", table_expr.db)
+
+            new_node = sqlglot.exp.Identifier(this=table_expr.name)
+
+            return new_node
+        else:
+            return node
+
+    return statement.transform(replace_cte_refs, copy=False)
 
 
 def create_lineage_sql_parsed_result(
@@ -1197,14 +1253,7 @@ def create_lineage_sql_parsed_result(
             default_schema=schema,
         )
     except Exception as e:
-        return SqlParsingResult(
-            in_tables=[],
-            out_tables=[],
-            column_lineage=None,
-            debug_info=SqlParsingDebugInfo(
-                table_error=e,
-            ),
-        )
+        return SqlParsingResult.make_from_error(e)
     finally:
         if needs_close:
             schema_resolver.close()
diff --git a/metadata-ingestion/src/datahub/utilities/topological_sort.py b/metadata-ingestion/src/datahub/utilities/topological_sort.py
new file mode 100644
index 0000000000000..f807dfe96063a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/topological_sort.py
@@ -0,0 +1,49 @@
+from collections import deque
+from typing import Dict, Iterable, List, Tuple, TypeVar
+
+_K = TypeVar("_K")
+
+
+def topological_sort(nodes: List[_K], edges: List[Tuple[_K, _K]]) -> Iterable[_K]:
+    """Topological sort of a directed acyclic graph or forest.
+
+    This is an implementation of Kahn's algorithm.
+
+    Args:
+        nodes: List of nodes.
+        edges: List of edges, as tuples of (source, target).
+
+    Returns:
+        List of nodes in topological order.
+    """
+
+    # Build adjacency list.
+    adj_list: Dict[_K, List[_K]] = {node: [] for node in nodes}
+    for source, target in edges:
+        adj_list[source].append(target)
+
+    # Build in-degree map.
+    in_degrees: Dict[_K, int] = {node: 0 for node in nodes}
+    for _source, target in edges:
+        in_degrees[target] += 1
+
+    # Initialize queue with nodes with in-degree 0.
+    queue = deque(node for node in nodes if in_degrees[node] == 0)
+
+    results = 0
+    while queue:
+        node = queue.popleft()
+
+        results += 1
+        yield node
+
+        # Decrement in-degree of each neighbor.
+        for neighbor in adj_list[node]:
+            in_degrees[neighbor] -= 1
+
+            # If in-degree is 0, add to queue.
+            if in_degrees[neighbor] == 0:
+                queue.append(neighbor)
+
+    if results != len(nodes):
+        raise ValueError("Graph contains cycles.")
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
index 16df7b8e51b24..e4f01ef7a6c53 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
@@ -14,7 +14,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -131,7 +132,92 @@
                                 "tableSchema": ""
                             }
                         },
-                        "fields": []
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
                     }
                 },
                 {
@@ -176,7 +262,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -195,7 +282,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -355,7 +443,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -373,7 +462,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +665,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -594,7 +685,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -712,7 +804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -730,7 +823,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -882,7 +976,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -900,7 +995,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1070,7 +1166,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1088,7 +1185,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1198,7 +1296,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1216,7 +1315,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1338,7 +1438,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1356,7 +1457,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1486,7 +1588,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1504,7 +1607,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1698,7 +1802,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1716,7 +1821,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1862,7 +1968,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1880,7 +1987,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2047,7 +2155,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2065,7 +2174,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2211,7 +2321,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2229,7 +2340,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2375,7 +2487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2393,7 +2506,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2539,7 +2653,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2557,7 +2672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2703,7 +2819,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2712,12 +2829,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2726,12 +2856,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2740,12 +2883,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2760,7 +2916,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2775,7 +2932,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2790,7 +2948,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2805,7 +2964,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2820,7 +2980,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-schemas-dbt-enabled"
+        "runId": "dbt-test-with-schemas-dbt-enabled",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
index 4557cb0324829..4d5b008b695f9 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
@@ -14,7 +14,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -65,7 +66,104 @@
                                 "tableSchema": ""
                             }
                         },
-                        "fields": []
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "initial_full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NullType": {}
+                                    }
+                                },
+                                "nativeDataType": "",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
                     }
                 },
                 {
@@ -118,7 +216,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -137,7 +236,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -275,7 +375,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -293,7 +394,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -487,7 +589,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -506,7 +609,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -652,7 +756,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -670,7 +775,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -920,7 +1026,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -938,7 +1045,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1084,7 +1192,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1102,7 +1211,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1272,7 +1382,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1290,7 +1401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1400,7 +1512,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1418,7 +1531,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1540,7 +1654,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1558,7 +1673,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1688,7 +1804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1706,7 +1823,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1900,7 +2018,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1918,7 +2037,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2064,7 +2184,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2082,7 +2203,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2249,7 +2371,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2267,7 +2390,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2413,7 +2537,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2431,7 +2556,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2577,7 +2703,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2595,7 +2722,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2741,7 +2869,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2759,7 +2888,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2905,7 +3035,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2914,12 +3045,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an-aliased-view-for-monthly-billing%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2928,12 +3072,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an_aliased_view_for_payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.an_aliased_view_for_payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2942,12 +3099,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.payments_by_customer_by_month%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2956,12 +3126,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.customer_snapshot%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.public.customer_snapshot%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2976,7 +3159,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2991,7 +3175,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-column-meta-mapping"
+        "runId": "dbt-column-meta-mapping",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
index 086c5a78e92a4..3e8ddf317f387 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
@@ -15,7 +15,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -203,7 +204,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -222,7 +224,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -429,7 +432,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -447,7 +451,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -565,7 +570,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -583,7 +589,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -713,7 +720,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -731,7 +739,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -861,7 +870,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -878,7 +888,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -975,7 +986,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -992,7 +1004,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1101,7 +1114,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1118,7 +1132,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1227,7 +1242,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1236,12 +1252,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.customers%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.customers%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1250,12 +1279,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.orders%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.orders%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1264,12 +1306,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.stg_customers%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.stg_customers%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1278,12 +1333,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.stg_orders%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.stg_orders%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1292,12 +1360,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.stg_payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.stg_payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1306,12 +1387,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.raw_customers%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.raw_customers%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1320,12 +1414,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.raw_orders%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.raw_orders%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1334,12 +1441,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.raw_payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Ccalm-pagoda-323403.jaffle_shop.raw_payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1354,7 +1474,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1397,7 +1518,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1408,23 +1530,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565131058,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1439,7 +1562,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1482,7 +1606,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1493,23 +1618,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565131075,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1524,7 +1650,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1567,7 +1694,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1578,23 +1706,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565131073,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1609,7 +1738,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1640,7 +1770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1651,23 +1782,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565131077,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1682,7 +1814,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1732,7 +1865,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1747,7 +1881,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1791,7 +1926,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1802,12 +1938,7 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565137668,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)",
             "status": "COMPLETE",
             "result": {
@@ -1815,12 +1946,18 @@
                 "nativeResults": {
                     "message": "Database Error in test dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2 (models/schema.yml)\n  No matching signature for operator = for argument types: INT64, STRING. Supported signature: ANY = ANY at [46:25]\n  compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql"
                 }
+            },
+            "assertionUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1835,7 +1972,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1874,7 +2012,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1885,12 +2024,7 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565137668,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
@@ -1898,12 +2032,18 @@
                 "nativeResults": {
                     "message": "Database Error in test dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0 (models/schema.yml)\n  No matching signature for operator = for argument types: FLOAT64, STRING. Supported signature: ANY = ANY at [36:25]\n  compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql"
                 }
+            },
+            "assertionUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1918,7 +2058,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1954,7 +2095,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1965,23 +2107,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565132560,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1996,7 +2139,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2032,7 +2176,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2043,23 +2188,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565133585,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2074,7 +2220,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2110,7 +2257,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2121,23 +2269,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565133591,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2152,7 +2301,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2188,7 +2338,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2199,23 +2350,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565133595,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:ca065a99637630468f688717590beeab",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:ca065a99637630468f688717590beeab",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2230,7 +2382,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2266,7 +2419,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2277,23 +2431,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565134031,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2308,7 +2463,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2344,7 +2500,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2355,23 +2512,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565134482,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2386,7 +2544,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2422,7 +2581,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2433,23 +2593,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565134485,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2464,7 +2625,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2500,7 +2662,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2511,23 +2674,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565134493,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2542,7 +2706,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2578,7 +2743,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2589,23 +2755,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565134966,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2620,7 +2787,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2656,7 +2824,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2667,23 +2836,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565135368,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2698,7 +2868,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2734,7 +2905,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2745,23 +2917,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565135377,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2776,7 +2949,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2821,7 +2995,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2832,23 +3007,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565135510,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2893,7 +3069,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2904,23 +3081,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565135510,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2935,7 +3113,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2977,7 +3156,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2988,23 +3168,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565135836,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3019,7 +3200,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3061,7 +3243,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3072,23 +3255,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565136269,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3103,7 +3287,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3145,7 +3330,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3156,23 +3342,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565136230,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3187,7 +3374,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3229,7 +3417,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3240,23 +3429,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565136395,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3271,7 +3461,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3313,7 +3504,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3324,23 +3516,24 @@
     "aspect": {
         "json": {
             "timestampMillis": 1655565136719,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
             "runId": "c7a6b778-0e0f-4789-b567-ca7e124a6840",
-            "assertionUrn": "urn:li:assertion:fac27f352406b941125292413afa8096",
             "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)",
             "status": "COMPLETE",
             "result": {
                 "type": "SUCCESS",
                 "nativeResults": {}
+            },
+            "assertionUrn": "urn:li:assertion:fac27f352406b941125292413afa8096",
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
             }
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3355,7 +3548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3370,7 +3564,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3385,7 +3580,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3400,7 +3596,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3415,7 +3612,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3430,7 +3628,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3445,7 +3644,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3460,7 +3660,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3475,7 +3676,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3490,7 +3692,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3505,7 +3708,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3520,7 +3724,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3535,7 +3740,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3550,7 +3756,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3565,7 +3772,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3580,7 +3788,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3595,7 +3804,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3610,7 +3820,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3625,7 +3836,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3640,7 +3852,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3655,7 +3868,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3670,7 +3884,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3685,7 +3900,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3700,7 +3916,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-2022_02_03-07_00_00"
+        "runId": "dbt-2022_02_03-07_00_00",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
index 19bfb60e62a08..0bdd5e3c895c2 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
@@ -14,7 +14,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -95,7 +96,92 @@
                                 "tableSchema": ""
                             }
                         },
-                        "fields": []
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
                     }
                 },
                 {
@@ -140,7 +226,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -159,7 +246,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -302,7 +390,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -320,7 +409,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -522,7 +612,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -541,7 +632,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -659,7 +751,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -677,7 +770,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -826,7 +920,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -844,7 +939,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1014,7 +1110,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1032,7 +1129,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1142,7 +1240,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1160,7 +1259,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1282,7 +1382,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1300,7 +1401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1427,7 +1529,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1445,7 +1548,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1639,7 +1743,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1657,7 +1762,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1803,7 +1909,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1821,7 +1928,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1985,7 +2093,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2003,7 +2112,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2149,7 +2259,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2167,7 +2278,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2313,7 +2425,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2331,7 +2444,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2477,7 +2591,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2486,12 +2601,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2500,12 +2628,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2514,12 +2655,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2534,7 +2688,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2549,7 +2704,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-complex-owner-patterns"
+        "runId": "dbt-test-with-complex-owner-patterns",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
index 242c83003b181..5ab0b11e37771 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
@@ -14,7 +14,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -96,7 +97,92 @@
                                 "tableSchema": ""
                             }
                         },
-                        "fields": []
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
                     }
                 },
                 {
@@ -141,7 +227,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -160,7 +247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -303,7 +391,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -321,7 +410,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -523,7 +613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -542,7 +633,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -660,7 +752,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +771,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -827,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -845,7 +940,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1015,7 +1111,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1033,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1143,7 +1241,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1161,7 +1260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1283,7 +1383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1301,7 +1402,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1428,7 +1530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1446,7 +1549,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1640,7 +1744,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1658,7 +1763,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1804,7 +1910,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1822,7 +1929,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1986,7 +2094,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2004,7 +2113,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2150,7 +2260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2168,7 +2279,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2314,7 +2426,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2332,7 +2445,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2478,7 +2592,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2496,7 +2611,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2642,7 +2758,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2651,12 +2768,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2665,12 +2795,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2679,12 +2822,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cdbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2699,7 +2855,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2714,7 +2871,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-data-platform-instance"
+        "runId": "dbt-test-with-data-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
index d98b63b9da62f..3725e590fee9e 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
@@ -14,7 +14,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -96,7 +97,92 @@
                                 "tableSchema": ""
                             }
                         },
-                        "fields": []
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
                     }
                 },
                 {
@@ -141,7 +227,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -160,7 +247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -303,7 +391,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -321,7 +410,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -523,7 +613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -542,7 +633,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -660,7 +752,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +771,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -827,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -845,7 +940,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1015,7 +1111,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1033,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1143,7 +1241,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1161,7 +1260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1283,7 +1383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1301,7 +1402,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1428,7 +1530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1446,7 +1549,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1640,7 +1744,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1658,7 +1763,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1804,7 +1910,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1822,7 +1929,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1986,7 +2094,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2004,7 +2113,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2150,7 +2260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2168,7 +2279,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2314,7 +2426,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2332,7 +2445,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2478,7 +2592,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2496,7 +2611,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2642,7 +2758,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2669,7 +2786,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2696,7 +2814,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2723,7 +2842,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2738,7 +2858,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2753,7 +2874,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-non-incremental-lineage"
+        "runId": "dbt-test-with-non-incremental-lineage",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
index 7c024f93641b9..a47abab6b40f7 100644
--- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
+++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
@@ -14,7 +14,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -96,7 +97,92 @@
                                 "tableSchema": ""
                             }
                         },
-                        "fields": []
+                        "fields": [
+                            {
+                                "fieldPath": "customer_id",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "INT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "full_name",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "VARCHAR",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "email",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "address",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "postal_code",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "phone",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "TEXT",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            }
+                        ]
                     }
                 },
                 {
@@ -141,7 +227,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -160,7 +247,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -303,7 +391,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -321,7 +410,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -523,7 +613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -542,7 +633,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -660,7 +752,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -678,7 +771,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -827,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -845,7 +940,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1015,7 +1111,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1033,7 +1130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1143,7 +1241,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1161,7 +1260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1283,7 +1383,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1301,7 +1402,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1428,7 +1530,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1446,7 +1549,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1640,7 +1744,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1658,7 +1763,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1804,7 +1910,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1822,7 +1929,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1986,7 +2094,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2004,7 +2113,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2150,7 +2260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2168,7 +2279,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2314,7 +2426,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2332,7 +2445,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2478,7 +2592,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2496,7 +2611,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2642,7 +2758,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2651,12 +2768,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-monthly-billing%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2665,12 +2795,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.an-aliased-view-for-payments%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2679,12 +2822,25 @@
     "changeType": "PATCH",
     "aspectName": "upstreamLineage",
     "aspect": {
-        "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]",
-        "contentType": "application/json-patch+json"
+        "json": [
+            {
+                "op": "add",
+                "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adbt%2Cpagila.dbt_postgres.payments_by_customer_by_month%2CPROD%29",
+                "value": {
+                    "auditStamp": {
+                        "time": 1643871600000,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
+                    "type": "TRANSFORMED"
+                }
+            }
+        ]
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2699,7 +2855,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2714,7 +2871,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "dbt-test-with-target-platform-instance"
+        "runId": "dbt-test-with-target-platform-instance",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py
index a970ff6a5de7a..95b5374bbb41d 100644
--- a/metadata-ingestion/tests/integration/dbt/test_dbt.py
+++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py
@@ -361,11 +361,11 @@ def test_dbt_tests_only_assertions(pytestconfig, tmp_path, mock_time, **kwargs):
                     test_results_path=str(
                         (test_resources_dir / "jaffle_shop_test_results.json").resolve()
                     ),
-                    # this is just here to avoid needing to access datahub server
-                    write_semantics="OVERRIDE",
                     entities_enabled=DBTEntitiesEnabled(
                         test_results=EmitDirective.ONLY
                     ),
+                    # this is just here to avoid needing to access datahub server
+                    write_semantics="OVERRIDE",
                 ),
             ),
             sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),
@@ -440,13 +440,13 @@ def test_dbt_only_test_definitions_and_results(
                     test_results_path=str(
                         (test_resources_dir / "jaffle_shop_test_results.json").resolve()
                     ),
-                    # this is just here to avoid needing to access datahub server
-                    write_semantics="OVERRIDE",
                     entities_enabled=DBTEntitiesEnabled(
                         sources=EmitDirective.NO,
                         seeds=EmitDirective.NO,
                         models=EmitDirective.NO,
                     ),
+                    # this is just here to avoid needing to access datahub server
+                    write_semantics="OVERRIDE",
                 ),
             ),
             sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_cte_name_collision.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_cte_name_collision.json
new file mode 100644
index 0000000000000..44f1075c058ad
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_cte_name_collision.json
@@ -0,0 +1,47 @@
+{
+    "query_type": "SELECT",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table3,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "col2",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR(16777216)"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)",
+                    "column": "col2"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "col1",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR(16777216)"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table3,PROD)",
+                    "column": "col1"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_full_table_name_col_reference.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_full_table_name_col_reference.json
new file mode 100644
index 0000000000000..f8301f1e8189e
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_full_table_name_col_reference.json
@@ -0,0 +1,55 @@
+{
+    "query_type": "SELECT",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "id",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                    "column": "id"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "id_gt_100",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                    }
+                },
+                "native_column_type": "INT"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                    "column": "id"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "struct_field1",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_table,PROD)",
+                    "column": "struct_field.field1"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_unused_cte.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_unused_cte.json
new file mode 100644
index 0000000000000..3916c6dc7c5ef
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_unused_cte.json
@@ -0,0 +1,39 @@
+{
+    "query_type": "SELECT",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,table1,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,table2,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:snowflake,table3,PROD)"
+    ],
+    "out_tables": [],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": null,
+                "column": "col1",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,table1,PROD)",
+                    "column": "col1"
+                }
+            ]
+        },
+        {
+            "downstream": {
+                "table": null,
+                "column": "col6",
+                "column_type": null,
+                "native_column_type": null
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,table3,PROD)",
+                    "column": "col6"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
index c420f2b8438ce..7f69e358f8f11 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -3,11 +3,59 @@
 import pytest
 
 from datahub.testing.check_sql_parser_result import assert_sql_result
-from datahub.utilities.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT
+from datahub.utilities.sqlglot_lineage import (
+    _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT,
+    detach_ctes,
+)
 
 RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens"
 
 
+def test_detach_ctes_simple():
+    original = "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 ON table2.id = __cte_0.id"
+    detached_expr = detach_ctes(
+        original,
+        platform="snowflake",
+        cte_mapping={"__cte_0": "_my_cte_table"},
+    )
+    detached = detached_expr.sql(dialect="snowflake")
+
+    assert (
+        detached
+        == "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN _my_cte_table ON table2.id = _my_cte_table.id"
+    )
+
+
+def test_detach_ctes_with_alias():
+    original = "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 AS tablealias ON table2.id = tablealias.id"
+    detached_expr = detach_ctes(
+        original,
+        platform="snowflake",
+        cte_mapping={"__cte_0": "_my_cte_table"},
+    )
+    detached = detached_expr.sql(dialect="snowflake")
+
+    assert (
+        detached
+        == "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN _my_cte_table AS tablealias ON table2.id = tablealias.id"
+    )
+
+
+def test_detach_ctes_with_multipart_replacement():
+    original = "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN __cte_0 ON table2.id = __cte_0.id"
+    detached_expr = detach_ctes(
+        original,
+        platform="snowflake",
+        cte_mapping={"__cte_0": "my_db.my_schema.my_table"},
+    )
+    detached = detached_expr.sql(dialect="snowflake")
+
+    assert (
+        detached
+        == "WITH __cte_0 AS (SELECT * FROM table1) SELECT * FROM table2 JOIN my_db.my_schema.my_table ON table2.id = my_db.my_schema.my_table.id"
+    )
+
+
 def test_select_max():
     # The COL2 should get normalized to col2.
     assert_sql_result(
@@ -630,6 +678,84 @@ def test_snowflake_column_cast():
     )
 
 
+def test_snowflake_unused_cte():
+    # For this, we expect table level lineage to include table1, but CLL should not.
+    assert_sql_result(
+        """
+WITH cte1 AS (
+    SELECT col1, col2
+    FROM table1
+    WHERE col1 = 'value1'
+), cte2 AS (
+    SELECT col3, col4
+    FROM table2
+    WHERE col2 = 'value2'
+)
+SELECT cte1.col1, table3.col6
+FROM cte1
+JOIN table3 ON table3.col5 = cte1.col2
+""",
+        dialect="snowflake",
+        expected_file=RESOURCE_DIR / "test_snowflake_unused_cte.json",
+    )
+
+
+def test_snowflake_cte_name_collision():
+    # In this example, output col1 should come from table3 and not table1, since the cte is unused.
+    # We'll still generate table-level lineage that includes table1.
+    assert_sql_result(
+        """
+WITH cte_alias AS (
+    SELECT col1, col2
+    FROM table1
+)
+SELECT table2.col2, cte_alias.col1
+FROM table2
+JOIN table3 AS cte_alias ON cte_alias.col2 = cte_alias.col2
+""",
+        dialect="snowflake",
+        default_db="my_db",
+        default_schema="my_schema",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table1,PROD)": {
+                "col1": "NUMBER(38,0)",
+                "col2": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table2,PROD)": {
+                "col2": "VARCHAR(16777216)",
+            },
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.table3,PROD)": {
+                "col1": "VARCHAR(16777216)",
+                "col2": "VARCHAR(16777216)",
+            },
+        },
+        expected_file=RESOURCE_DIR / "test_snowflake_cte_name_collision.json",
+    )
+
+
+def test_snowflake_full_table_name_col_reference():
+    assert_sql_result(
+        """
+SELECT
+    my_db.my_schema.my_table.id,
+    case when my_db.my_schema.my_table.id > 100 then 1 else 0 end as id_gt_100,
+    my_db.my_schema.my_table.struct_field.field1 as struct_field1,
+FROM my_db.my_schema.my_table
+""",
+        dialect="snowflake",
+        default_db="my_db",
+        default_schema="my_schema",
+        schemas={
+            "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_db.my_schema.my_db.my_schema.my_table,PROD)": {
+                "id": "NUMBER(38,0)",
+                "struct_field": "struct",
+            },
+        },
+        expected_file=RESOURCE_DIR
+        / "test_snowflake_full_table_name_col_reference.json",
+    )
+
+
 # TODO: Add a test for setting platform_instance or env
 
 
diff --git a/metadata-ingestion/tests/unit/test_topological_sort.py b/metadata-ingestion/tests/unit/test_topological_sort.py
new file mode 100644
index 0000000000000..4300816b6c48f
--- /dev/null
+++ b/metadata-ingestion/tests/unit/test_topological_sort.py
@@ -0,0 +1,33 @@
+import pytest
+
+from datahub.utilities.topological_sort import topological_sort
+
+
+def test_topological_sort_valid():
+    nodes = ["a", "b", "c", "d", "e", "f"]
+    edges = [
+        ("a", "d"),
+        ("f", "b"),
+        ("b", "d"),
+        ("f", "a"),
+        ("d", "c"),
+    ]
+
+    # This isn't the only valid topological sort order.
+    expected_order = ["e", "f", "b", "a", "d", "c"]
+    assert list(topological_sort(nodes, edges)) == expected_order
+
+
+def test_topological_sort_invalid():
+    nodes = ["a", "b", "c", "d", "e", "f"]
+    edges = [
+        ("a", "d"),
+        ("f", "b"),
+        ("b", "d"),
+        ("f", "a"),
+        ("d", "c"),
+        ("c", "f"),
+    ]
+
+    with pytest.raises(ValueError, match="cycle"):
+        list(topological_sort(nodes, edges))

From 906a5b91a8adbd3165c01607e9301c84b886106f Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 13 Nov 2023 21:51:11 -0500
Subject: [PATCH 204/259] chore(ingest): cleanup various methods (#9221)

---
 metadata-events/mxe-schemas/rename-namespace.sh       |  2 +-
 metadata-ingestion/scripts/docgen.py                  | 11 +++--------
 .../datahub/ingestion/source/looker/looker_common.py  |  2 +-
 .../src/datahub/ingestion/source/mode.py              |  2 ++
 4 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/metadata-events/mxe-schemas/rename-namespace.sh b/metadata-events/mxe-schemas/rename-namespace.sh
index 6402e09b65c07..ef04868a6bd15 100755
--- a/metadata-events/mxe-schemas/rename-namespace.sh
+++ b/metadata-events/mxe-schemas/rename-namespace.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 SCRIPT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]:-$0}" )" >/dev/null && pwd )"
 
diff --git a/metadata-ingestion/scripts/docgen.py b/metadata-ingestion/scripts/docgen.py
index 1a4db09e961ce..3e4595650d46a 100644
--- a/metadata-ingestion/scripts/docgen.py
+++ b/metadata-ingestion/scripts/docgen.py
@@ -7,11 +7,10 @@
 import sys
 import textwrap
 from importlib.metadata import metadata, requires
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional
 
 import click
 from pydantic import BaseModel, Field
-from pydantic.dataclasses import dataclass
 
 from datahub.configuration.common import ConfigModel
 from datahub.ingestion.api.decorators import (
@@ -94,7 +93,6 @@ class Component(BaseModel):
 
     @staticmethod
     def map_field_path_to_components(field_path: str) -> List[Component]:
-
         m = re.match(FieldRow._V2_FIELD_PATH_TOKEN_MATCHER_PREFIX, field_path)
         v = re.match(FieldRow._V2_FIELD_PATH_FIELD_NAME_MATCHER, field_path)
         components: List[FieldRow.Component] = []
@@ -197,7 +195,7 @@ def get_checkbox(self) -> str:
             # Using a non-breaking space to prevent the checkbox from being
             # broken into a new line.
             if not self.parent:  # None and empty string both count
-                return f'&nbsp;<abbr title="Required">✅</abbr>'
+                return '&nbsp;<abbr title="Required">✅</abbr>'
             else:
                 return f'&nbsp;<abbr title="Required if {self.parent} is set">❓</abbr>'
         else:
@@ -356,7 +354,6 @@ def priority_value(path: str) -> str:
 
 
 def gen_md_table_from_struct(schema_dict: Dict[str, Any]) -> List[str]:
-
     from datahub.ingestion.extractor.json_schema_util import JsonSchemaTranslator
 
     # we don't want default field values to be injected into the description of the field
@@ -460,7 +457,6 @@ def get_additional_deps_for_extra(extra_name: str) -> List[str]:
 
 
 def relocate_path(orig_path: str, relative_path: str, relocated_path: str) -> str:
-
     newPath = os.path.join(os.path.dirname(orig_path), relative_path)
     assert os.path.exists(newPath)
 
@@ -515,7 +511,6 @@ def generate(
 
     if extra_docs:
         for path in glob.glob(f"{extra_docs}/**/*[.md|.yaml|.yml]", recursive=True):
-
             m = re.search("/docs/sources/(.*)/(.*).md", path)
             if m:
                 platform_name = m.group(1).lower()
@@ -741,7 +736,7 @@ def generate(
             i += 1
             f.write(f"---\nsidebar_position: {i}\n---\n\n")
             f.write(
-                f"import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n"
+                "import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n"
             )
             f.write(f"# {platform_docs['name']}\n")
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
index 7ca5ce49019ab..e440750cba0d0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -828,7 +828,7 @@ def from_api(  # noqa: C901
                 )
             else:
                 logger.warning(
-                    f"Failed to extract explore {explore_name} from model {model}.", e
+                    f"Failed to extract explore {explore_name} from model {model}: {e}"
                 )
 
         except AssertionError:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py
index c46b56da422d9..e4ea3b2ed099f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mode.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py
@@ -218,6 +218,8 @@ def construct_dashboard(
         if creator is not None:
             modified_actor = builder.make_user_urn(creator)
             if report_info.get("last_saved_at") is None:
+                # Sometimes mode returns null for last_saved_at.
+                # In that case, we use the created_at timestamp instead.
                 report_info["last_saved_at"] = report_info.get("created_at")
 
             modified_ts = int(

From f1b6aa782277e3611eeb64edf6c123598de109df Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Mon, 13 Nov 2023 21:51:27 -0500
Subject: [PATCH 205/259] docs: clarify how to disable telemetry (#9236)

---
 docs/deploy/telemetry.md | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/docs/deploy/telemetry.md b/docs/deploy/telemetry.md
index c5458cc5df05e..a1b25337de767 100644
--- a/docs/deploy/telemetry.md
+++ b/docs/deploy/telemetry.md
@@ -4,7 +4,25 @@
 
 To effectively build and maintain the DataHub Project, we must understand how end-users work within DataHub. Beginning in version 0.8.35, DataHub collects anonymous usage statistics and errors to inform our roadmap priorities and to enable us to proactively address errors.
 
-Deployments are assigned a UUID which is sent along with event details, Java version, OS, and timestamp; telemetry collection is enabled by default and can be disabled by setting `DATAHUB_TELEMETRY_ENABLED=false` in your Docker Compose config.
+Both the DataHub backend and the ingestion framework collect telemetry.
 
+## DataHub Backend Telemetry
 
-The source code is available [here.](../../metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java)
\ No newline at end of file
+Deployments are assigned a UUID which is sent along with event details, Java version, OS, and timestamp.
+The source code is available [here](../../metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java).
+
+## Ingestion Framework Telemetry
+
+The ingestion framework collects telemetry including CLI invocations, source/sink types, error types, versions, and timestamps. If you run with `datahub --debug`, all telemetry calls will be logged.
+
+On first invocation, the CLI will generate a randomized UUID, which will be sent alongside every telemetry event. This config is stored in `~/.datahub/telemetry-config.json`.
+
+The source code is available [here](../../metadata-ingestion/src/datahub/telemetry/telemetry.py).
+
+## Disabling Telemetry
+
+Telemetry is enabled by default. While we are careful to anonymize all telemetry data and encourage users to keep it enabled so that we can improve DataHub, we understand that some users may wish to disable it.
+
+You can disable backend telemetry by setting the `DATAHUB_TELEMETRY_ENABLED` environment variable to `false`. You'll need to set this on both the datahub-gms and datahub-actions containers.
+
+If you're using the DataHub CLI, ingestion framework telemetry will be disabled when the `DATAHUB_TELEMETRY_ENABLED` environment variable is set to `false`. To persist this change for your machine, run `datahub telemetry disable`.

From cfeecd799dd58793e12deb941cb51d590f7a6f4f Mon Sep 17 00:00:00 2001
From: Tony Ouyang <tony@terratrue.com>
Date: Tue, 14 Nov 2023 09:12:39 -0800
Subject: [PATCH 206/259] feat(ingest/mongodb): support AWS DocumentDB for
 MongoDB (#9201)

---
 .../src/datahub/ingestion/source/mongodb.py   | 38 ++++++++++++++++---
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
index ce2b9ce2981e0..2aa8b1d37d477 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py
@@ -1,5 +1,6 @@
 import logging
 from dataclasses import dataclass, field
+from enum import Enum
 from typing import Dict, Iterable, List, Optional, Tuple, Type, Union, ValuesView
 
 import bson.timestamp
@@ -74,6 +75,12 @@
 DENY_DATABASE_LIST = set(["admin", "config", "local"])
 
 
+class HostingEnvironment(Enum):
+    SELF_HOSTED = "SELF_HOSTED"
+    ATLAS = "ATLAS"
+    AWS_DOCUMENTDB = "AWS_DOCUMENTDB"
+
+
 class MongoDBConfig(
     PlatformInstanceConfigMixin, EnvConfigMixin, StatefulIngestionConfigBase
 ):
@@ -108,6 +115,11 @@ class MongoDBConfig(
     # errors out with "16793600" as the maximum size supported.
     maxDocumentSize: Optional[PositiveInt] = Field(default=16793600, description="")
 
+    hostingEnvironment: Optional[HostingEnvironment] = Field(
+        default=HostingEnvironment.SELF_HOSTED,
+        description="Hosting environment of MongoDB, default is SELF_HOSTED, currently support `SELF_HOSTED`, `ATLAS`, `AWS_DOCUMENTDB`",
+    )
+
     database_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
         description="regex patterns for databases to filter in ingestion.",
@@ -176,7 +188,7 @@ def construct_schema_pymongo(
     delimiter: str,
     use_random_sampling: bool,
     max_document_size: int,
-    is_version_gte_4_4: bool,
+    should_add_document_size_filter: bool,
     sample_size: Optional[int] = None,
 ) -> Dict[Tuple[str, ...], SchemaDescription]:
     """
@@ -191,15 +203,19 @@ def construct_schema_pymongo(
             the PyMongo collection
         delimiter:
             string to concatenate field names by
+        use_random_sampling:
+            boolean to indicate if random sampling should be added to aggregation
+        max_document_size:
+            maximum size of the document that will be considered for generating the schema.
+        should_add_document_size_filter:
+            boolean to indicate if document size filter should be added to aggregation
         sample_size:
             number of items in the collection to sample
             (reads entire collection if not provided)
-        max_document_size:
-            maximum size of the document that will be considered for generating the schema.
     """
 
     aggregations: List[Dict] = []
-    if is_version_gte_4_4:
+    if should_add_document_size_filter:
         doc_size_field = "temporary_doc_size_field"
         # create a temporary field to store the size of the document. filter on it and then remove it.
         aggregations = [
@@ -381,7 +397,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                         delimiter=".",
                         use_random_sampling=self.config.useRandomSampling,
                         max_document_size=self.config.maxDocumentSize,
-                        is_version_gte_4_4=self.is_server_version_gte_4_4(),
+                        should_add_document_size_filter=self.should_add_document_size_filter(),
                         sample_size=self.config.schemaSamplingSize,
                     )
 
@@ -475,6 +491,18 @@ def is_server_version_gte_4_4(self) -> bool:
 
         return False
 
+    def is_hosted_on_aws_documentdb(self) -> bool:
+        return self.config.hostingEnvironment == HostingEnvironment.AWS_DOCUMENTDB
+
+    def should_add_document_size_filter(self) -> bool:
+        # the operation $bsonsize is only available in server version greater than 4.4
+        # and is not supported by AWS DocumentDB, we should only add this operation to
+        # aggregation for mongodb that doesn't run on AWS DocumentDB and version is greater than 4.4
+        # https://docs.aws.amazon.com/documentdb/latest/developerguide/mongo-apis.html
+        return (
+            self.is_server_version_gte_4_4() and not self.is_hosted_on_aws_documentdb()
+        )
+
     def get_report(self) -> MongoDBSourceReport:
         return self.report
 

From ec13847f54fb167571359bb233489b8b353bad02 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 14 Nov 2023 14:25:26 -0500
Subject: [PATCH 207/259] feat(airflow): make RUN_IN_THREAD configurable
 (#9226)

---
 docs/lineage/airflow.md                                   | 1 +
 .../src/datahub_airflow_plugin/datahub_listener.py        | 8 ++++++--
 metadata-ingestion/src/datahub/cli/docker_cli.py          | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md
index 19ed1598d4c5a..3a13aefa834a4 100644
--- a/docs/lineage/airflow.md
+++ b/docs/lineage/airflow.md
@@ -193,6 +193,7 @@ In order to use this example, you must first configure the Datahub hook. Like in
 If you're not seeing lineage in DataHub, check the following:
 
 - Validate that the plugin is loaded in Airflow. Go to Admin -> Plugins and check that the DataHub plugin is listed.
+- With the v2 plugin, it should also print a log line like `INFO  [datahub_airflow_plugin.datahub_listener] DataHub plugin v2 using DataHubRestEmitter: configured to talk to <datahub_url>` during Airflow startup, and the `airflow plugins` command should list `datahub_plugin` with a listener enabled.
 - If using the v2 plugin's automatic lineage, ensure that the `enable_extractors` config is set to true and that automatic lineage is supported for your operator.
 - If using manual lineage annotation, ensure that you're using the `datahub_airflow_plugin.entities.Dataset` or `datahub_airflow_plugin.entities.Urn` classes for your inlets and outlets.
 
diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
index d00b10bbe1756..c39eef2635658 100644
--- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
+++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py
@@ -1,6 +1,7 @@
 import copy
 import functools
 import logging
+import os
 import threading
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, TypeVar, cast
 
@@ -55,7 +56,10 @@ def hookimpl(f: _F) -> _F:  # type: ignore[misc] # noqa: F811
 
 _airflow_listener_initialized = False
 _airflow_listener: Optional["DataHubListener"] = None
-_RUN_IN_THREAD = True
+_RUN_IN_THREAD = os.getenv("DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD", "true").lower() in (
+    "true",
+    "1",
+)
 _RUN_IN_THREAD_TIMEOUT = 30
 
 
@@ -133,7 +137,7 @@ def __init__(self, config: DatahubLineageConfig):
 
         self._emitter = config.make_emitter_hook().make_emitter()
         self._graph: Optional[DataHubGraph] = None
-        logger.info(f"DataHub plugin using {repr(self._emitter)}")
+        logger.info(f"DataHub plugin v2 using {repr(self._emitter)}")
 
         # See discussion here https://github.com/OpenLineage/OpenLineage/pull/508 for
         # why we need to keep track of tasks ourselves.
diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py
index 77e3285d359ef..08f3faae8abb2 100644
--- a/metadata-ingestion/src/datahub/cli/docker_cli.py
+++ b/metadata-ingestion/src/datahub/cli/docker_cli.py
@@ -766,7 +766,7 @@ def quickstart(  # noqa: C901
                 logger.debug("docker compose up timed out, sending SIGTERM")
                 up_process.terminate()
                 try:
-                    up_process.wait(timeout=3)
+                    up_process.wait(timeout=8)
                 except subprocess.TimeoutExpired:
                     logger.debug("docker compose up still running, sending SIGKILL")
                     up_process.kill()

From 486e394cb8727470e6efe564ef04359f4ec6d1b1 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Tue, 14 Nov 2023 14:06:33 -0600
Subject: [PATCH 208/259] fix(signup): prevent invalid email signup (#9234)

---
 .../app/auth/NativeAuthenticationConfigs.java    | 16 ++++++++++++----
 .../controllers/AuthenticationController.java    |  8 ++++++++
 datahub-frontend/conf/application.conf           |  4 ++++
 .../authentication/user/NativeUserService.java   | 10 +++++++++-
 .../user/NativeUserServiceTest.java              | 15 ++++++++++++++-
 .../authentication/AuthServiceController.java    |  8 ++++++++
 .../factory/auth/NativeUserServiceFactory.java   |  7 ++++++-
 7 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java
index db17313d67f9a..3114da92d7d79 100644
--- a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java
+++ b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java
@@ -6,18 +6,26 @@
 public class NativeAuthenticationConfigs {
 
   public static final String NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH = "auth.native.enabled";
+  public static final String NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH = "auth.native.signUp.enforceValidEmail";
 
   private Boolean _isEnabled = true;
+  private Boolean _isEnforceValidEmailEnabled = true;
 
   public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) {
-    if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)
-        && Boolean.FALSE.equals(
-        Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString()))) {
-      _isEnabled = false;
+    if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)) {
+      _isEnabled = Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString());
+    }
+    if (configs.hasPath(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)) {
+      _isEnforceValidEmailEnabled =
+          Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH).toString());
     }
   }
 
   public boolean isNativeAuthenticationEnabled() {
     return _isEnabled;
   }
+
+  public boolean isEnforceValidEmailEnabled() {
+    return _isEnforceValidEmailEnabled;
+  }
 }
diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java
index 4f89f4f67e149..e28d4ba2ee37e 100644
--- a/datahub-frontend/app/controllers/AuthenticationController.java
+++ b/datahub-frontend/app/controllers/AuthenticationController.java
@@ -27,6 +27,7 @@
 import org.pac4j.play.store.PlaySessionStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import play.data.validation.Constraints;
 import play.libs.Json;
 import play.mvc.Controller;
 import play.mvc.Http;
@@ -203,6 +204,13 @@ public Result signUp(Http.Request request) {
             JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty.");
             return Results.badRequest(invalidCredsJson);
         }
+        if (_nativeAuthenticationConfigs.isEnforceValidEmailEnabled()) {
+            Constraints.EmailValidator emailValidator = new Constraints.EmailValidator();
+            if (!emailValidator.isValid(email)) {
+                JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty.");
+                return Results.badRequest(invalidCredsJson);
+            }
+        }
 
         if (StringUtils.isBlank(password)) {
             JsonNode invalidCredsJson = Json.newObject().put("message", "Password must not be empty.");
diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf
index 1a62c8547e721..0f4ddb7c497e6 100644
--- a/datahub-frontend/conf/application.conf
+++ b/datahub-frontend/conf/application.conf
@@ -196,6 +196,10 @@ auth.oidc.preferredJwsAlgorithm = ${?AUTH_OIDC_PREFERRED_JWS_ALGORITHM} # Which
 #
 auth.jaas.enabled = ${?AUTH_JAAS_ENABLED}
 auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
+
+# Enforces the usage of a valid email for user sign up
+auth.native.signUp.enforceValidEmail = true
+auth.native.signUp.enforceValidEmail = ${?ENFORCE_VALID_EMAIL}
 #
 # To disable all authentication to the app, and proxy all users through a master "datahub" account, make sure that,
 # jaas, native and oidc auth are disabled:
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java
index 7f0c16d28f121..bff675ddd9cb2 100644
--- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java
+++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java
@@ -1,7 +1,9 @@
 package com.datahub.authentication.user;
 
 import com.datahub.authentication.Authentication;
+import com.datahub.authentication.AuthenticationConfiguration;
 import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.CorpuserUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.events.metadata.ChangeType;
@@ -34,6 +36,7 @@ public class NativeUserService {
   private final EntityService _entityService;
   private final EntityClient _entityClient;
   private final SecretService _secretService;
+  private final AuthenticationConfiguration _authConfig;
 
   public void createNativeUser(@Nonnull String userUrnString, @Nonnull String fullName, @Nonnull String email,
       @Nonnull String title, @Nonnull String password, @Nonnull Authentication authentication) throws Exception {
@@ -45,7 +48,12 @@ public void createNativeUser(@Nonnull String userUrnString, @Nonnull String full
     Objects.requireNonNull(authentication, "authentication must not be null!");
 
     final Urn userUrn = Urn.createFromString(userUrnString);
-    if (_entityService.exists(userUrn) || userUrn.toString().equals(SYSTEM_ACTOR)) {
+    if (_entityService.exists(userUrn)
+        // Should never fail these due to Controller level check, but just in case more usages get put in
+        || userUrn.toString().equals(SYSTEM_ACTOR)
+        || userUrn.toString().equals(new CorpuserUrn(_authConfig.getSystemClientId()).toString())
+        || userUrn.toString().equals(DATAHUB_ACTOR)
+        || userUrn.toString().equals(UNKNOWN_ACTOR)) {
       throw new RuntimeException("This user already exists! Cannot create a new user.");
     }
     updateCorpUserInfo(userUrn, fullName, email, title, authentication);
diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java
index b0b10af82155a..0102311ff3b61 100644
--- a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java
+++ b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java
@@ -3,6 +3,7 @@
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.ActorType;
 import com.datahub.authentication.Authentication;
+import com.datahub.authentication.AuthenticationConfiguration;
 import com.linkedin.common.urn.CorpuserUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.client.EntityClient;
@@ -48,8 +49,10 @@ public void setupTest() throws Exception {
     _entityService = mock(EntityService.class);
     _entityClient = mock(EntityClient.class);
     _secretService = mock(SecretService.class);
+    AuthenticationConfiguration authenticationConfiguration = new AuthenticationConfiguration();
+    authenticationConfiguration.setSystemClientId("someCustomId");
 
-    _nativeUserService = new NativeUserService(_entityService, _entityClient, _secretService);
+    _nativeUserService = new NativeUserService(_entityService, _entityClient, _secretService, authenticationConfiguration);
   }
 
   @Test
@@ -74,6 +77,16 @@ public void testCreateNativeUserUserAlreadyExists() throws Exception {
     _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, TITLE, PASSWORD, SYSTEM_AUTHENTICATION);
   }
 
+  @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "This user already exists! Cannot create a new user.")
+  public void testCreateNativeUserUserDatahub() throws Exception {
+    _nativeUserService.createNativeUser(DATAHUB_ACTOR, FULL_NAME, EMAIL, TITLE, PASSWORD, SYSTEM_AUTHENTICATION);
+  }
+
+  @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "This user already exists! Cannot create a new user.")
+  public void testCreateNativeUserUserSystemUser() throws Exception {
+    _nativeUserService.createNativeUser(SYSTEM_ACTOR, FULL_NAME, EMAIL, TITLE, PASSWORD, SYSTEM_AUTHENTICATION);
+  }
+
   @Test
   public void testCreateNativeUserPasses() throws Exception {
     when(_entityService.exists(any())).thenReturn(false);
diff --git a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java
index b5ce99902108a..34354a47b7f04 100644
--- a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java
+++ b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java
@@ -12,6 +12,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.linkedin.common.urn.CorpuserUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import java.util.concurrent.CompletableFuture;
@@ -28,6 +29,8 @@
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.client.HttpClientErrorException;
 
+import static com.linkedin.metadata.Constants.*;
+
 
 @Slf4j
 @RestController
@@ -177,6 +180,11 @@ CompletableFuture<ResponseEntity<String>> signUp(final HttpEntity<String> httpEn
     }
 
     String userUrnString = userUrn.asText();
+    String systemClientUser = new CorpuserUrn(_configProvider.getAuthentication().getSystemClientId()).toString();
+
+    if (userUrnString.equals(systemClientUser) || userUrnString.equals(DATAHUB_ACTOR) || userUrnString.equals(UNKNOWN_ACTOR)) {
+      return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST));
+    }
     String fullNameString = fullName.asText();
     String emailString = email.asText();
     String titleString = title.asText();
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java
index 3df499ea9392e..a0df661852935 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java
@@ -3,6 +3,7 @@
 package com.linkedin.gms.factory.auth;
 
 import com.datahub.authentication.user.NativeUserService;
+import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.metadata.client.JavaEntityClient;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import com.linkedin.metadata.entity.EntityService;
@@ -31,10 +32,14 @@ public class NativeUserServiceFactory {
   @Qualifier("dataHubSecretService")
   private SecretService _secretService;
 
+  @Autowired
+  private ConfigurationProvider _configurationProvider;
+
   @Bean(name = "nativeUserService")
   @Scope("singleton")
   @Nonnull
   protected NativeUserService getInstance() throws Exception {
-    return new NativeUserService(this._entityService, this._javaEntityClient, this._secretService);
+    return new NativeUserService(_entityService, _javaEntityClient, _secretService,
+        _configurationProvider.getAuthentication());
   }
 }
\ No newline at end of file

From f70d8a45b5237fde7f4be8bc859b8bef0785752e Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Tue, 14 Nov 2023 19:00:22 -0600
Subject: [PATCH 209/259] chore(security): version adjustments for security
 vulns (#9243)

---
 build.gradle                                  | 27 ++++++++++---------
 datahub-upgrade/build.gradle                  | 10 ++++++-
 metadata-events/mxe-registration/build.gradle |  2 +-
 metadata-events/mxe-utils-avro/build.gradle   |  2 +-
 metadata-io/build.gradle                      | 12 +++++++--
 metadata-service/factories/build.gradle       |  1 +
 metadata-service/restli-api/build.gradle      |  3 +++
 7 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/build.gradle b/build.gradle
index 54802917d05a5..9eecb1696bb19 100644
--- a/build.gradle
+++ b/build.gradle
@@ -19,7 +19,7 @@ buildscript {
   ext.logbackClassic = '1.2.12'
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '2.3.0'
-  ext.hazelcastVersion = '5.3.1'
+  ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
 
   ext.docker_registry = 'linkedin'
@@ -53,7 +53,7 @@ project.ext.spec = [
         'pegasus' : [
             'd2' : 'com.linkedin.pegasus:d2:' + pegasusVersion,
             'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
-            'dataAvro1_6' : 'com.linkedin.pegasus:data-avro-1_6:' + pegasusVersion,
+            'dataAvro': 'com.linkedin.pegasus:data-avro:' + pegasusVersion,
             'generator': 'com.linkedin.pegasus:generator:' + pegasusVersion,
             'restliCommon' : 'com.linkedin.pegasus:restli-common:' + pegasusVersion,
             'restliClient' : 'com.linkedin.pegasus:restli-client:' + pegasusVersion,
@@ -71,22 +71,21 @@ project.ext.externalDependency = [
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
-    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
-    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
-    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
-    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0',
+    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
+    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.9',
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
     'awsRds':'software.amazon.awssdk:rds:2.18.24',
-    'cacheApi' : 'javax.cache:cache-api:1.1.0',
+    'cacheApi': 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
     'commonsText': 'org.apache.commons:commons-text:1.10.0',
     'commonsCollections': 'commons-collections:commons-collections:3.2.2',
-    'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
     'datastaxOssCore': 'com.datastax.oss:java-driver-core:4.14.1',
     'datastaxOssQueryBuilder': 'com.datastax.oss:java-driver-query-builder:4.14.1',
-    'dgraph4j' : 'io.dgraph:dgraph4j:21.03.1',
+    'dgraph4j' : 'io.dgraph:dgraph4j:21.12.0',
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
@@ -131,7 +130,7 @@ project.ext.externalDependency = [
     'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
-    'json': 'org.json:json:20230227',
+    'json': 'org.json:json:20231013',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
@@ -140,7 +139,7 @@ project.ext.externalDependency = [
     'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
     'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion",
-    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.3',
+    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.4',
     'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
     'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
     'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
@@ -164,6 +163,7 @@ project.ext.externalDependency = [
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
+    'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
     'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
@@ -178,6 +178,7 @@ project.ext.externalDependency = [
     'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
     'postgresql': 'org.postgresql:postgresql:42.3.8',
     'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
+    'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.9',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
@@ -201,7 +202,7 @@ project.ext.externalDependency = [
     'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion",
     'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion",
     'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
-    'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11',
+    'springKafka': 'org.springframework.kafka:spring-kafka:2.9.13',
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
@@ -263,7 +264,7 @@ subprojects {
   plugins.withType(JavaPlugin) {
     dependencies {
       constraints {
-        implementation('io.netty:netty-all:4.1.86.Final')
+        implementation('io.netty:netty-all:4.1.100.Final')
         implementation('org.apache.commons:commons-compress:1.21')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')
diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
index 81e6e79c2a5e5..3356445cda7e1 100644
--- a/datahub-upgrade/build.gradle
+++ b/datahub-upgrade/build.gradle
@@ -66,7 +66,9 @@ dependencies {
   runtimeOnly externalDependency.mysqlConnector
   runtimeOnly externalDependency.postgresql
 
-  implementation externalDependency.awsMskIamAuth
+  implementation(externalDependency.awsMskIamAuth) {
+    exclude group: 'software.amazon.awssdk', module: 'third-party-jackson-core'
+  }
 
   annotationProcessor externalDependency.lombok
   annotationProcessor externalDependency.picocli
@@ -75,6 +77,12 @@ dependencies {
   testImplementation externalDependency.mockito
   testImplementation externalDependency.testng
   testRuntimeOnly externalDependency.logbackClassic
+
+  constraints {
+    implementation(implementation externalDependency.parquetHadoop) {
+      because("CVE-2022-42003")
+    }
+  }
 }
 
 bootJar {
diff --git a/metadata-events/mxe-registration/build.gradle b/metadata-events/mxe-registration/build.gradle
index 032870d93329f..2842dd935c7ee 100644
--- a/metadata-events/mxe-registration/build.gradle
+++ b/metadata-events/mxe-registration/build.gradle
@@ -7,7 +7,7 @@ configurations {
 dependencies {
   implementation project(':metadata-events:mxe-avro')
   implementation project(':metadata-models')
-  implementation spec.product.pegasus.dataAvro1_6
+  implementation spec.product.pegasus.dataAvro
 
   testImplementation project(':test-models')
   testImplementation project(path: ':test-models', configuration: 'testDataTemplate')
diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle
index a7bf287ab224d..3493797ab4f97 100644
--- a/metadata-events/mxe-utils-avro/build.gradle
+++ b/metadata-events/mxe-utils-avro/build.gradle
@@ -3,7 +3,7 @@ apply plugin: 'java-library'
 dependencies {
   api project(':metadata-events:mxe-avro')
   api project(':metadata-models')
-  api spec.product.pegasus.dataAvro1_6
+  api spec.product.pegasus.dataAvro
 
   testImplementation externalDependency.testng
   testImplementation project(':test-models')
diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle
index 4b36f533476f7..48f80f06d07c2 100644
--- a/metadata-io/build.gradle
+++ b/metadata-io/build.gradle
@@ -22,13 +22,18 @@ dependencies {
   implementation externalDependency.guava
   implementation externalDependency.reflections
   implementation externalDependency.jsonPatch
-  api externalDependency.dgraph4j exclude group: 'com.google.guava', module: 'guava'
+  api(externalDependency.dgraph4j) {
+    exclude group: 'com.google.guava', module: 'guava'
+    exclude group: 'io.grpc', module: 'grpc-protobuf'
+  }
   implementation externalDependency.slf4jApi
   runtimeOnly externalDependency.logbackClassic
   compileOnly externalDependency.lombok
   implementation externalDependency.commonsCollections
   api externalDependency.datastaxOssNativeProtocol
-  api externalDependency.datastaxOssCore
+  api(externalDependency.datastaxOssCore) {
+    exclude group: 'com.fasterxml.jackson.core'
+  }
   api externalDependency.datastaxOssQueryBuilder
   api externalDependency.elasticSearchRest
   api externalDependency.elasticSearchJava
@@ -101,6 +106,9 @@ dependencies {
     implementation(externalDependency.snappy) {
       because("previous versions are vulnerable to CVE-2023-34453 through CVE-2023-34455")
     }
+    implementation(externalDependency.grpcProtobuf) {
+      because("CVE-2023-1428, CVE-2023-32731")
+    }
   }
 }
 
diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle
index 2e99def17c3c5..86644e3b034da 100644
--- a/metadata-service/factories/build.gradle
+++ b/metadata-service/factories/build.gradle
@@ -63,4 +63,5 @@ dependencies {
 configurations.all{
   exclude group: "commons-io", module:"commons-io"
   exclude group: "jline", module:"jline"
+  exclude group: 'software.amazon.awssdk', module: 'third-party-jackson-core'
 }
diff --git a/metadata-service/restli-api/build.gradle b/metadata-service/restli-api/build.gradle
index f182d11b6baeb..352738d01f8da 100644
--- a/metadata-service/restli-api/build.gradle
+++ b/metadata-service/restli-api/build.gradle
@@ -13,5 +13,8 @@ dependencies {
     restClientCompile(externalDependency.zookeeper) {
       because("CVE-2023-44981")
     }
+    restClientCompile(externalDependency.grpcProtobuf) {
+      because("CVE-2023-1428, CVE-2023-32731")
+    }
   }
 }
\ No newline at end of file

From 8475fc92b41e9a67841d7e5ecf114fd073499cb0 Mon Sep 17 00:00:00 2001
From: Pierre Guivarch <3973133+PGuiv@users.noreply.github.com>
Date: Tue, 14 Nov 2023 21:42:13 -0500
Subject: [PATCH 210/259] docs(ingest): fix typo in snowflake ingestion docs
 (#9239)

---
 .../src/datahub/ingestion/source_config/sql/snowflake.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
index 2e9a15063661e..ccc4e115729a2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
+++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py
@@ -59,7 +59,7 @@ class BaseSnowflakeConfig(ConfigModel):
     )
     private_key: Optional[str] = pydantic.Field(
         default=None,
-        description="Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n' if using key pair authentication. Encrypted version of private key will be in a form of '-----BEGIN ENCRYPTED PRIVATE KEY-----\\nencrypted-private-key\\n-----END ECNCRYPTED PRIVATE KEY-----\\n' See: https://docs.snowflake.com/en/user-guide/key-pair-auth.html",
+        description="Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n' if using key pair authentication. Encrypted version of private key will be in a form of '-----BEGIN ENCRYPTED PRIVATE KEY-----\\nencrypted-private-key\\n-----END ENCRYPTED PRIVATE KEY-----\\n' See: https://docs.snowflake.com/en/user-guide/key-pair-auth.html",
     )
 
     private_key_path: Optional[str] = pydantic.Field(

From 4201e541ca48ad617f1cfa63c4a6487288973589 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Wed, 15 Nov 2023 06:28:40 -0600
Subject: [PATCH 211/259] chore(security): jre to headless, removes x11
 dependency (#9245)

---
 docker/datahub-frontend/Dockerfile     | 2 +-
 docker/datahub-gms/Dockerfile          | 2 +-
 docker/datahub-mae-consumer/Dockerfile | 2 +-
 docker/datahub-mce-consumer/Dockerfile | 2 +-
 docker/datahub-upgrade/Dockerfile      | 2 +-
 docker/kafka-setup/Dockerfile          | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile
index 9c13e73078042..aaace5ae38ca3 100644
--- a/docker/datahub-frontend/Dockerfile
+++ b/docker/datahub-frontend/Dockerfile
@@ -9,7 +9,7 @@ RUN addgroup -S datahub && adduser -S datahub -G datahub
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl sqlite libc6-compat java-snappy \
-    && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
+    && apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/
 
 ENV LD_LIBRARY_PATH="/lib:/lib64"
diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
index e271188a703cc..c5696bbd2d1d2 100644
--- a/docker/datahub-gms/Dockerfile
+++ b/docker/datahub-gms/Dockerfile
@@ -19,7 +19,7 @@ ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
-    && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
+    && apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
index ec3da4de71d15..07af7c66a7783 100644
--- a/docker/datahub-mae-consumer/Dockerfile
+++ b/docker/datahub-mae-consumer/Dockerfile
@@ -19,7 +19,7 @@ ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash coreutils sqlite libc6-compat java-snappy \
-    && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
+    && apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
     && wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
index f9c47f77a98f5..97861d6be3141 100644
--- a/docker/datahub-mce-consumer/Dockerfile
+++ b/docker/datahub-mce-consumer/Dockerfile
@@ -19,7 +19,7 @@ ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash sqlite libc6-compat java-snappy \
-    && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
+    && apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
     && wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
index f08e7268e4018..fa8e65009662b 100644
--- a/docker/datahub-upgrade/Dockerfile
+++ b/docker/datahub-upgrade/Dockerfile
@@ -19,7 +19,7 @@ ENV JMX_VERSION=0.18.0
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
-    && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
+    && apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
     && curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile
index a9c75521fead1..e7f084739a576 100644
--- a/docker/kafka-setup/Dockerfile
+++ b/docker/kafka-setup/Dockerfile
@@ -18,7 +18,7 @@ ENV SCALA_VERSION 2.13
 LABEL name="kafka" version=${KAFKA_VERSION}
 
 RUN apk add --no-cache bash coreutils
-RUN apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community
+RUN apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community
 
 RUN apk add --no-cache -t .build-deps git curl ca-certificates jq gcc musl-dev libffi-dev zip
 RUN mkdir -p /opt \

From 6655918923deefad5486d037a1be158ec1a1856c Mon Sep 17 00:00:00 2001
From: Pedro Silva <pedro@acryl.io>
Date: Wed, 15 Nov 2023 19:53:50 +0000
Subject: [PATCH 212/259] feat(recomendations): Make top platforms account only
 for searchable entities (#9240)

---
 .../elasticsearch/ElasticSearchService.java   |  8 ++---
 .../elasticsearch/query/ESSearchDAO.java      | 14 ++++----
 .../metadata/search/TestEntityTestBase.java   | 15 +++++----
 .../EntitySearchAggregationSource.java        |  7 +++-
 .../candidatesource/TopPlatformsSource.java   | 32 +++++++++++++------
 .../metadata/search/EntitySearchService.java  |  6 ++--
 6 files changed, 51 insertions(+), 31 deletions(-)

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
index 024cf2b0abec2..9b43642d7621c 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
@@ -142,11 +142,11 @@ public AutoCompleteResult autoComplete(@Nonnull String entityName, @Nonnull Stri
 
   @Nonnull
   @Override
-  public Map<String, Long> aggregateByValue(@Nullable String entityName, @Nonnull String field,
+  public Map<String, Long> aggregateByValue(@Nullable List<String> entityNames, @Nonnull String field,
       @Nullable Filter requestParams, int limit) {
-    log.debug("Aggregating by value: {}, field: {}, requestParams: {}, limit: {}", entityName, field, requestParams,
-        limit);
-    return esSearchDAO.aggregateByValue(entityName, field, requestParams, limit);
+    log.debug("Aggregating by value: {}, field: {}, requestParams: {}, limit: {}", entityNames.toString(), field,
+        requestParams, limit);
+    return esSearchDAO.aggregateByValue(entityNames, field, requestParams, limit);
   }
 
   @Nonnull
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
index 290e8c60deb00..960a5b38826b1 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
@@ -31,6 +31,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.RequiredArgsConstructor;
@@ -263,17 +264,16 @@ public AutoCompleteResult autoComplete(@Nonnull String entityName, @Nonnull Stri
    * @return
    */
   @Nonnull
-  public Map<String, Long> aggregateByValue(@Nullable String entityName, @Nonnull String field,
+  public Map<String, Long> aggregateByValue(@Nullable List<String> entityNames, @Nonnull String field,
       @Nullable Filter requestParams, int limit) {
     final SearchRequest searchRequest = SearchRequestHandler.getAggregationRequest(field, transformFilterForEntities(requestParams, indexConvention), limit);
-    String indexName;
-    if (entityName == null) {
-      indexName = indexConvention.getAllEntityIndicesPattern();
+    if (entityNames == null) {
+      String indexName = indexConvention.getAllEntityIndicesPattern();
+      searchRequest.indices(indexName);
     } else {
-      EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName);
-      indexName = indexConvention.getIndexName(entitySpec);
+      Stream<String> stream = entityNames.stream().map(entityRegistry::getEntitySpec).map(indexConvention::getIndexName);
+      searchRequest.indices(stream.toArray(String[]::new));
     }
-    searchRequest.indices(indexName);
 
     try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "aggregateByValue_search").time()) {
       final SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java
index d358c03c612d0..a4c359b3595c2 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java
@@ -3,6 +3,7 @@
 import com.datahub.test.Snapshot;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.TestEntityUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
@@ -99,7 +100,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception {
     BrowseResult browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10);
     assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 0);
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 0);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textField", null, 10).size(), 0);
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textField", null, 10).size(), 0);
 
     Urn urn = new TestEntityUrn("test", "urn1", "VALUE_1");
     ObjectNode document = JsonNodeFactory.instance.objectNode();
@@ -124,7 +125,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception {
     assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 1);
     assertEquals(browseResult.getGroups().get(0).getName(), "b");
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 1);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textFieldOverride", null, 10),
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textFieldOverride", null, 10),
         ImmutableMap.of("textFieldOverride", 1L));
 
     Urn urn2 = new TestEntityUrn("test2", "urn2", "VALUE_2");
@@ -147,7 +148,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception {
     assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 1);
     assertEquals(browseResult.getGroups().get(0).getName(), "b");
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 2);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textFieldOverride", null, 10),
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textFieldOverride", null, 10),
         ImmutableMap.of("textFieldOverride", 1L, "textFieldOverride2", 1L));
 
     _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString());
@@ -158,7 +159,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception {
     browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10);
     assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 0);
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 0);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textField", null, 10).size(), 0);
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textField", null, 10).size(), 0);
   }
 
   @Test
@@ -181,7 +182,7 @@ public void testElasticSearchServiceFulltext() throws Exception {
     assertEquals(searchResult.getEntities().get(0).getEntity(), urn);
 
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 1);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textFieldOverride", null, 10),
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textFieldOverride", null, 10),
             ImmutableMap.of("textFieldOverride", 1L));
 
     Urn urn2 = new TestEntityUrn("test2", "urn2", "VALUE_2");
@@ -198,7 +199,7 @@ public void testElasticSearchServiceFulltext() throws Exception {
     assertEquals(searchResult.getEntities().get(0).getEntity(), urn2);
 
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 2);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textFieldOverride", null, 10),
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textFieldOverride", null, 10),
             ImmutableMap.of("textFieldOverride", 1L, "textFieldOverride2", 1L));
 
     _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString());
@@ -208,6 +209,6 @@ public void testElasticSearchServiceFulltext() throws Exception {
     assertEquals(searchResult.getNumEntities().intValue(), 0);
 
     assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 0);
-    assertEquals(_elasticSearchService.aggregateByValue(ENTITY_NAME, "textField", null, 10).size(), 0);
+    assertEquals(_elasticSearchService.aggregateByValue(ImmutableList.of(ENTITY_NAME), "textField", null, 10).size(), 0);
   }
 }
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java
index 9fb0c18f1b621..e1ebc6d5e97be 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java
@@ -82,7 +82,7 @@ protected <T> boolean isValidCandidate(T candidate) {
   public List<RecommendationContent> getRecommendations(@Nonnull Urn userUrn,
       @Nullable RecommendationRequestContext requestContext) {
     Map<String, Long> aggregationResult =
-        _entitySearchService.aggregateByValue(null, getSearchFieldName(), null, getMaxContent());
+        _entitySearchService.aggregateByValue(getEntityNames(), getSearchFieldName(), null, getMaxContent());
 
     if (aggregationResult.isEmpty()) {
       return Collections.emptyList();
@@ -116,6 +116,11 @@ public List<RecommendationContent> getRecommendations(@Nonnull Urn userUrn,
         .collect(Collectors.toList());
   }
 
+  protected List<String> getEntityNames() {
+    // By default, no list is applied which means searching across entities.
+    return null;
+  }
+
   // Get top K entries with the most count
   private <T> List<Map.Entry<T, Long>> getTopKValues(Map<T, Long> countMap) {
     final PriorityQueue<Map.Entry<T, Long>> queue =
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java
index f81a91be0660a..9562440889f63 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java
@@ -1,15 +1,16 @@
 package com.linkedin.metadata.recommendation.candidatesource;
 
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.dataplatform.DataPlatformInfo;
+import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.recommendation.RecommendationRenderType;
 import com.linkedin.metadata.recommendation.RecommendationRequestContext;
 import com.linkedin.metadata.recommendation.ScenarioType;
 import com.linkedin.metadata.search.EntitySearchService;
-import java.util.Set;
+import java.util.List;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 
@@ -18,12 +19,24 @@
 public class TopPlatformsSource extends EntitySearchAggregationSource {
 
   /**
-   * TODO: Remove this once we permit specifying set of entities in aggregation API (filter out assertions)
+   * Set of entities that we want to consider for defining the top platform sources.
+   * This must match SearchUtils.SEARCHABLE_ENTITY_TYPES
    */
-  private static final Set<String> FILTERED_DATA_PLATFORM_URNS = ImmutableSet.of(
-      "urn:li:dataPlatform:great-expectations"
+  private static final List<String> SEARCHABLE_ENTITY_TYPES = ImmutableList.of(
+      Constants.DATASET_ENTITY_NAME,
+      Constants.DASHBOARD_ENTITY_NAME,
+      Constants.CHART_ENTITY_NAME,
+      Constants.ML_MODEL_ENTITY_NAME,
+      Constants.ML_MODEL_GROUP_ENTITY_NAME,
+      Constants.ML_FEATURE_TABLE_ENTITY_NAME,
+      Constants.ML_FEATURE_ENTITY_NAME,
+      Constants.ML_PRIMARY_KEY_ENTITY_NAME,
+      Constants.DATA_FLOW_ENTITY_NAME,
+      Constants.DATA_JOB_ENTITY_NAME,
+      Constants.TAG_ENTITY_NAME,
+      Constants.CONTAINER_ENTITY_NAME,
+      Constants.NOTEBOOK_ENTITY_NAME
   );
-
   private final EntityService _entityService;
   private static final String PLATFORM = "platform";
 
@@ -52,6 +65,10 @@ public boolean isEligible(@Nonnull Urn userUrn, @Nonnull RecommendationRequestCo
     return requestContext.getScenario() == ScenarioType.HOME;
   }
 
+  protected List<String> getEntityNames() {
+    return SEARCHABLE_ENTITY_TYPES;
+  }
+
   @Override
   protected String getSearchFieldName() {
     return PLATFORM;
@@ -69,9 +86,6 @@ protected boolean isValueUrn() {
 
   @Override
   protected boolean isValidCandidateUrn(Urn urn) {
-    if (FILTERED_DATA_PLATFORM_URNS.contains(urn.toString())) {
-      return false;
-    }
     RecordTemplate dataPlatformInfo = _entityService.getLatestAspect(urn, "dataPlatformInfo");
     if (dataPlatformInfo == null) {
       return false;
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
index cbfeeaef860d3..9cd865bd888e2 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java
@@ -131,15 +131,15 @@ AutoCompleteResult autoComplete(@Nonnull String entityName, @Nonnull String quer
   /**
    * Returns number of documents per field value given the field and filters
    *
-   * @param entityName name of the entity, if empty aggregate over all entities
+   * @param entityNames list of name of entities to aggregate across, if empty aggregate over all entities
    * @param field the field name for aggregate
    * @param requestParams filters to apply before aggregating
    * @param limit the number of aggregations to return
    * @return
    */
   @Nonnull
-  Map<String, Long> aggregateByValue(@Nullable String entityName, @Nonnull String field, @Nullable Filter requestParams,
-      int limit);
+  Map<String, Long> aggregateByValue(@Nullable List<String> entityNames, @Nonnull String field,
+      @Nullable Filter requestParams, int limit);
 
   /**
    * Gets a list of groups/entities that match given browse request.

From 5dd09dd62e1df262ac4962f0a420fac04e47288b Mon Sep 17 00:00:00 2001
From: gaurav2733 <77378510+gaurav2733@users.noreply.github.com>
Date: Thu, 16 Nov 2023 22:53:07 +0530
Subject: [PATCH 213/259] Feature/prd 770 (#9224)

---
 .../src/app/domain/EmptyDomainDescription.tsx | 39 +++++++++++
 .../src/app/domain/EmptyDomainsSection.tsx    | 69 +++++++++++++++++++
 .../nestedDomains/ManageDomainsPageV2.tsx     |  2 +-
 .../app/domain/nestedDomains/RootDomains.tsx  | 16 ++++-
 4 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 datahub-web-react/src/app/domain/EmptyDomainDescription.tsx
 create mode 100644 datahub-web-react/src/app/domain/EmptyDomainsSection.tsx

diff --git a/datahub-web-react/src/app/domain/EmptyDomainDescription.tsx b/datahub-web-react/src/app/domain/EmptyDomainDescription.tsx
new file mode 100644
index 0000000000000..6a5f304e565be
--- /dev/null
+++ b/datahub-web-react/src/app/domain/EmptyDomainDescription.tsx
@@ -0,0 +1,39 @@
+import { Typography } from 'antd';
+import React from 'react';
+import styled from 'styled-components/macro';
+import { ANTD_GRAY } from '../entity/shared/constants';
+
+const StyledParagraph = styled(Typography.Paragraph)`
+    text-align: justify;
+    text-justify: inter-word;
+    margin: 40px 0;
+    font-size: 15px;
+`;
+
+function EmptyDomainDescription() {
+    return (
+        <>
+            <StyledParagraph type="secondary">
+                <strong style={{ color: ANTD_GRAY[8] }}>Welcome to your Data Domains!</strong> It looks like this space
+                is ready to be transformed into a well-organized data universe. Start by creating your first domain - a
+                high-level category for your data assets.
+            </StyledParagraph>
+            <StyledParagraph type="secondary">
+                <strong style={{ color: ANTD_GRAY[8] }}> Create Nested Domains:</strong> Want to dive deeper? You can
+                also create nested domains to add granularity and structure. Just like nesting Russian dolls, its all
+                about refining your organization.
+            </StyledParagraph>
+            <StyledParagraph type="secondary">
+                <strong style={{ color: ANTD_GRAY[8] }}>Build Data Products</strong>: Once your domains are set, go a
+                step further! Organize your data assets into data products to realize a data mesh architecture. Data
+                products empower you to treat data as a product, making it more accessible and manageable.
+            </StyledParagraph>
+            <StyledParagraph type="secondary">
+                Ready to embark on this data adventure? Click the Create Domain button to begin shaping your data
+                landscape!
+            </StyledParagraph>
+        </>
+    );
+}
+
+export default EmptyDomainDescription;
diff --git a/datahub-web-react/src/app/domain/EmptyDomainsSection.tsx b/datahub-web-react/src/app/domain/EmptyDomainsSection.tsx
new file mode 100644
index 0000000000000..f232d259c20da
--- /dev/null
+++ b/datahub-web-react/src/app/domain/EmptyDomainsSection.tsx
@@ -0,0 +1,69 @@
+import { PlusOutlined } from '@ant-design/icons';
+import { Button, Empty, Typography } from 'antd';
+import React from 'react';
+import styled from 'styled-components/macro';
+import { ANTD_GRAY } from '../entity/shared/constants';
+
+const EmptyDomainContainer = styled.div`
+    display: flex;
+    justify-content: center;
+    align-items: center;
+`;
+
+const StyledEmpty = styled(Empty)`
+    width: 35vw;
+    @media screen and (max-width: 1300px) {
+        width: 50vw;
+    }
+    @media screen and (max-width: 896px) {
+        overflow-y: auto;
+        max-height: 75vh;
+        &::-webkit-scrollbar {
+            width: 5px;
+            background: #d6d6d6;
+        }
+    }
+    padding: 60px 40px;
+    .ant-empty-image {
+        display: none;
+    }
+`;
+
+const StyledButton = styled(Button)`
+    margin: 18px 8px 0 0;
+`;
+
+const IconContainer = styled.span`
+    color: ${ANTD_GRAY[7]};
+    font-size: 40px;
+`;
+
+interface Props {
+    title?: string;
+    setIsCreatingDomain: React.Dispatch<React.SetStateAction<boolean>>;
+    description?: React.ReactNode;
+    icon?: React.ReactNode;
+}
+
+function EmptyDomainsSection(props: Props) {
+    const { title, description, setIsCreatingDomain, icon } = props;
+    return (
+        <EmptyDomainContainer>
+            <StyledEmpty
+                description={
+                    <>
+                        <IconContainer>{icon}</IconContainer>
+                        <Typography.Title level={4}>{title}</Typography.Title>
+                        {description}
+                    </>
+                }
+            >
+                <StyledButton onClick={() => setIsCreatingDomain(true)}>
+                    <PlusOutlined /> Create Domain
+                </StyledButton>
+            </StyledEmpty>
+        </EmptyDomainContainer>
+    );
+}
+
+export default EmptyDomainsSection;
diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx
index b69f0c5458b5d..f5fc0cba2d8ec 100644
--- a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx
+++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx
@@ -51,7 +51,7 @@ export default function ManageDomainsPageV2() {
                     <PlusOutlined /> New Domain
                 </Button>
             </Header>
-            <RootDomains />
+            <RootDomains setIsCreatingDomain={setIsCreatingDomain} />
             {isCreatingDomain && (
                 <CreateDomainModal
                     onClose={() => setIsCreatingDomain(false)}
diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx
index 757119919e336..75c38cd4951ef 100644
--- a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx
+++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx
@@ -1,17 +1,23 @@
 import React from 'react';
 import styled from 'styled-components';
+import { ReadOutlined } from '@ant-design/icons';
 import { Message } from '../../shared/Message';
 import { ResultWrapper } from '../../search/SearchResultList';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import { EntityType } from '../../../types.generated';
 import useListDomains from '../useListDomains';
+import EmptyDomainsSection from '../EmptyDomainsSection';
+import EmptyDomainDescription from '../EmptyDomainDescription';
 
 const DomainsWrapper = styled.div`
     overflow: auto;
     padding: 0 28px 16px 28px;
 `;
 
-export default function RootDomains() {
+interface Props {
+    setIsCreatingDomain: React.Dispatch<React.SetStateAction<boolean>>;
+}
+export default function RootDomains({ setIsCreatingDomain }: Props) {
     const entityRegistry = useEntityRegistry();
     const { loading, error, data, sortedDomains } = useListDomains({});
 
@@ -19,6 +25,14 @@ export default function RootDomains() {
         <>
             {!data && loading && <Message type="loading" content="Loading domains..." />}
             {error && <Message type="error" content="Failed to load domains. An unexpected error occurred." />}
+            {!loading && (!data || !data?.listDomains?.domains?.length) && (
+                <EmptyDomainsSection
+                    icon={<ReadOutlined />}
+                    title="Organize your data"
+                    description={<EmptyDomainDescription />}
+                    setIsCreatingDomain={setIsCreatingDomain}
+                />
+            )}
             <DomainsWrapper>
                 {sortedDomains?.map((domain) => (
                     <ResultWrapper showUpdatedStyles>

From 8451c758e700e147d9653f2ce7bddc9eb2ef314c Mon Sep 17 00:00:00 2001
From: Salman-Apptware <101426513+Salman-Apptware@users.noreply.github.com>
Date: Thu, 16 Nov 2023 22:54:47 +0530
Subject: [PATCH 214/259] fix(search): fix search on paginated lists (#9198)

---
 datahub-web-react/src/app/identity/group/GroupList.tsx   | 5 ++++-
 datahub-web-react/src/app/identity/user/UserList.tsx     | 5 ++++-
 datahub-web-react/src/app/ingest/secret/SecretsList.tsx  | 9 ++++++---
 .../src/app/ingest/source/IngestionSourceList.tsx        | 9 ++++++---
 .../src/app/permissions/policy/ManagePolicies.tsx        | 7 +++++--
 .../src/app/permissions/roles/ManageRoles.tsx            | 7 +++++--
 6 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/datahub-web-react/src/app/identity/group/GroupList.tsx b/datahub-web-react/src/app/identity/group/GroupList.tsx
index db9901a53b26b..5ef77b4dfc8a8 100644
--- a/datahub-web-react/src/app/identity/group/GroupList.tsx
+++ b/datahub-web-react/src/app/identity/group/GroupList.tsx
@@ -92,7 +92,10 @@ export const GroupList = () => {
                             fontSize: 12,
                         }}
                         onSearch={() => null}
-                        onQueryChange={(q) => setQuery(q)}
+                        onQueryChange={(q) => {
+                            setPage(1);
+                            setQuery(q);
+                        }}
                         entityRegistry={entityRegistry}
                         hideRecommendations
                     />
diff --git a/datahub-web-react/src/app/identity/user/UserList.tsx b/datahub-web-react/src/app/identity/user/UserList.tsx
index 55ef27b8458fa..e50005b08377e 100644
--- a/datahub-web-react/src/app/identity/user/UserList.tsx
+++ b/datahub-web-react/src/app/identity/user/UserList.tsx
@@ -135,7 +135,10 @@ export const UserList = () => {
                             fontSize: 12,
                         }}
                         onSearch={() => null}
-                        onQueryChange={(q) => setQuery(q)}
+                        onQueryChange={(q) => {
+                            setPage(1);
+                            setQuery(q);
+                        }}
                         entityRegistry={entityRegistry}
                         hideRecommendations
                     />
diff --git a/datahub-web-react/src/app/ingest/secret/SecretsList.tsx b/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
index 8e5b601e2a809..2728fff0ccba3 100644
--- a/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
+++ b/datahub-web-react/src/app/ingest/secret/SecretsList.tsx
@@ -54,10 +54,10 @@ export const SecretsList = () => {
             input: {
                 start,
                 count: pageSize,
-                query: query && query.length > 0 ? query : undefined,
+                query: (query?.length && query) || undefined,
             },
         },
-        fetchPolicy: query && query.length > 0 ? 'no-cache' : 'cache-first',
+        fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',
     });
 
     const totalSecrets = data?.listSecrets?.total || 0;
@@ -197,7 +197,10 @@ export const SecretsList = () => {
                             fontSize: 12,
                         }}
                         onSearch={() => null}
-                        onQueryChange={(q) => setQuery(q)}
+                        onQueryChange={(q) => {
+                            setPage(1);
+                            setQuery(q);
+                        }}
                         entityRegistry={entityRegistry}
                         hideRecommendations
                     />
diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
index 13af19b0b6ac2..6188845694f9e 100644
--- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
+++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -107,10 +107,10 @@ export const IngestionSourceList = () => {
             input: {
                 start,
                 count: pageSize,
-                query,
+                query: (query?.length && query) || undefined,
             },
         },
-        fetchPolicy: 'cache-first',
+        fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',
     });
     const [createIngestionSource] = useCreateIngestionSourceMutation();
     const [updateIngestionSource] = useUpdateIngestionSourceMutation();
@@ -399,7 +399,10 @@ export const IngestionSourceList = () => {
                                 fontSize: 12,
                             }}
                             onSearch={() => null}
-                            onQueryChange={(q) => setQuery(q)}
+                            onQueryChange={(q) => {
+                                setPage(1);
+                                setQuery(q);
+                            }}
                             entityRegistry={entityRegistry}
                             hideRecommendations
                         />
diff --git a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx
index 08327d40a7165..49b0ec922fd57 100644
--- a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx
+++ b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx
@@ -166,7 +166,6 @@ export const ManagePolicies = () => {
         data: policiesData,
         refetch: policiesRefetch,
     } = useListPoliciesQuery({
-        fetchPolicy: 'no-cache',
         variables: {
             input: {
                 start,
@@ -174,6 +173,7 @@ export const ManagePolicies = () => {
                 query,
             },
         },
+        fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',
     });
 
     // Any time a policy is removed, edited, or created, refetch the list.
@@ -476,7 +476,10 @@ export const ManagePolicies = () => {
                             fontSize: 12,
                         }}
                         onSearch={() => null}
-                        onQueryChange={(q) => setQuery(q)}
+                        onQueryChange={(q) => {
+                            setPage(1);
+                            setQuery(q);
+                        }}
                         entityRegistry={entityRegistry}
                         hideRecommendations
                     />
diff --git a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
index ccdfb7002c67d..011109e2eb915 100644
--- a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
+++ b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
@@ -72,7 +72,6 @@ export const ManageRoles = () => {
         data: rolesData,
         refetch: rolesRefetch,
     } = useListRolesQuery({
-        fetchPolicy: 'cache-first',
         variables: {
             input: {
                 start,
@@ -80,6 +79,7 @@ export const ManageRoles = () => {
                 query,
             },
         },
+        fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',
     });
 
     const totalRoles = rolesData?.listRoles?.total || 0;
@@ -238,7 +238,10 @@ export const ManageRoles = () => {
                             fontSize: 12,
                         }}
                         onSearch={() => null}
-                        onQueryChange={(q) => setQuery(q)}
+                        onQueryChange={(q) => {
+                            setPage(1);
+                            setQuery(q);
+                        }}
                         entityRegistry={entityRegistry}
                     />
                     {isBatchAddRolesModalVisible && (

From ee9ee406662018aabce0f7859f5d14d235bfe7f1 Mon Sep 17 00:00:00 2001
From: gaurav2733 <77378510+gaurav2733@users.noreply.github.com>
Date: Thu, 16 Nov 2023 23:05:48 +0530
Subject: [PATCH 215/259] fix(): increase the search bar highlight border to
 double the width (#9251)

Co-authored-by: John Joyce <john@acryl.io>
---
 datahub-web-react/src/app/search/SearchBar.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
index a23ead83caf54..15457c006c61b 100644
--- a/datahub-web-react/src/app/search/SearchBar.tsx
+++ b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -45,7 +45,7 @@ const StyledSearchBar = styled(Input)`
         border: 2px solid transparent;
 
         &:focus-within {
-            border: 1.5px solid ${REDESIGN_COLORS.BLUE};
+            border: 2px solid ${REDESIGN_COLORS.BLUE};
         }
     }
     > .ant-input::placeholder {

From e6305c0d95b4ed2960a5be89dcf1e97a3b3d3787 Mon Sep 17 00:00:00 2001
From: Sumit Patil <91715217+sumitappt@users.noreply.github.com>
Date: Thu, 16 Nov 2023 23:07:23 +0530
Subject: [PATCH 216/259] feat: Add loading indicator to Manage Domains sidebar
 (#9142)

Co-authored-by: apptware <ext_sumit.patil@workpac.com>
---
 .../src/app/domain/DomainSearch.tsx           | 93 +++++++------------
 .../src/app/domain/DomainSearchResultItem.tsx | 68 ++++++++++++++
 .../domainNavigator/DomainNavigator.tsx       | 28 ++++--
 .../profile/sidebar/Domain/SetDomainModal.tsx | 36 ++++++-
 4 files changed, 151 insertions(+), 74 deletions(-)
 create mode 100644 datahub-web-react/src/app/domain/DomainSearchResultItem.tsx

diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx
index e82dae9c2c9e6..5036a795ea5e4 100644
--- a/datahub-web-react/src/app/domain/DomainSearch.tsx
+++ b/datahub-web-react/src/app/domain/DomainSearch.tsx
@@ -1,17 +1,12 @@
-import React, { CSSProperties, useRef, useState } from 'react';
-import { Link } from 'react-router-dom';
+import React, { useRef, useState } from 'react';
+import { LoadingOutlined } from '@ant-design/icons';
 import styled from 'styled-components/macro';
-import Highlight from 'react-highlighter';
 import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated';
 import { EntityType } from '../../types.generated';
-import { IconStyleType } from '../entity/Entity';
-import { ANTD_GRAY } from '../entity/shared/constants';
 import { SearchBar } from '../search/SearchBar';
 import ClickOutside from '../shared/ClickOutside';
 import { useEntityRegistry } from '../useEntityRegistry';
-import DomainIcon from './DomainIcon';
-import ParentEntities from '../search/filters/ParentEntities';
-import { getParentDomains } from './utils';
+import DomainSearchResultItem from './DomainSearchResultItem';
 
 const DomainSearchWrapper = styled.div`
     position: relative;
@@ -33,34 +28,19 @@ const ResultsWrapper = styled.div`
     z-index: 1;
 `;
 
-const SearchResult = styled(Link)`
-    color: #262626;
+const LoadingWrapper = styled.div`
     display: flex;
     align-items: center;
-    gap: 8px;
-    height: 100%;
-    padding: 6px 8px;
-    width: 100%;
-    &:hover {
-        background-color: ${ANTD_GRAY[3]};
-        color: #262626;
-    }
+    justify-content: center;
+    height: 350px;
+    font-size: 30px;
 `;
 
-const IconWrapper = styled.span``;
-
-const highlightMatchStyle: CSSProperties = {
-    fontWeight: 'bold',
-    background: 'none',
-    padding: 0,
-};
-
 function DomainSearch() {
     const [query, setQuery] = useState('');
     const [isSearchBarFocused, setIsSearchBarFocused] = useState(false);
     const entityRegistry = useEntityRegistry();
-
-    const { data } = useGetSearchResultsForMultipleQuery({
+    const { data, loading } = useGetSearchResultsForMultipleQuery({
         variables: {
             input: {
                 types: [EntityType.Domain],
@@ -69,11 +49,11 @@ function DomainSearch() {
                 count: 50,
             },
         },
-        skip: !query,
     });
 
     const searchResults = data?.searchAcrossEntities?.searchResults;
     const timerRef = useRef(-1);
+
     const handleQueryChange = (q: string) => {
         window.clearTimeout(timerRef.current);
         timerRef.current = window.setTimeout(() => {
@@ -81,6 +61,26 @@ function DomainSearch() {
         }, 250);
     };
 
+    const renderLoadingIndicator = () => (
+        <LoadingWrapper>
+            <LoadingOutlined />
+        </LoadingWrapper>
+    );
+
+    const renderSearchResults = () => (
+        <ResultsWrapper>
+            {searchResults?.map((result) => (
+                <DomainSearchResultItem
+                    key={result.entity.urn}
+                    entity={result.entity}
+                    entityRegistry={entityRegistry}
+                    query={query}
+                    onResultClick={() => setIsSearchBarFocused(false)}
+                />
+            ))}
+        </ResultsWrapper>
+    );
+
     return (
         <DomainSearchWrapper>
             <ClickOutside onClickOutside={() => setIsSearchBarFocused(false)}>
@@ -102,39 +102,8 @@ function DomainSearch() {
                     entityRegistry={entityRegistry}
                     onFocus={() => setIsSearchBarFocused(true)}
                 />
-                {isSearchBarFocused && searchResults && !!searchResults.length && (
-                    <ResultsWrapper>
-                        {searchResults.map((result) => {
-                            return (
-                                <SearchResult
-                                    to={entityRegistry.getEntityUrl(result.entity.type, result.entity.urn)}
-                                    onClick={() => setIsSearchBarFocused(false)}
-                                >
-                                    <IconWrapper>
-                                        {result.entity.type === EntityType.Domain ? (
-                                            <DomainIcon
-                                                style={{
-                                                    fontSize: 16,
-                                                    color: '#BFBFBF',
-                                                }}
-                                            />
-                                        ) : (
-                                            entityRegistry.getIcon(result.entity.type, 12, IconStyleType.ACCENT)
-                                        )}
-                                    </IconWrapper>
-                                    <div>
-                                        <ParentEntities
-                                            parentEntities={getParentDomains(result.entity, entityRegistry)}
-                                        />
-                                        <Highlight matchStyle={highlightMatchStyle} search={query}>
-                                            {entityRegistry.getDisplayName(result.entity.type, result.entity)}
-                                        </Highlight>
-                                    </div>
-                                </SearchResult>
-                            );
-                        })}
-                    </ResultsWrapper>
-                )}
+                {loading && renderLoadingIndicator()}
+                {!loading && isSearchBarFocused && !!searchResults?.length && renderSearchResults()}
             </ClickOutside>
         </DomainSearchWrapper>
     );
diff --git a/datahub-web-react/src/app/domain/DomainSearchResultItem.tsx b/datahub-web-react/src/app/domain/DomainSearchResultItem.tsx
new file mode 100644
index 0000000000000..dc33ea173e0ae
--- /dev/null
+++ b/datahub-web-react/src/app/domain/DomainSearchResultItem.tsx
@@ -0,0 +1,68 @@
+// Create a new component called SearchResultItem.js
+import React from 'react';
+import { Link } from 'react-router-dom';
+import Highlight from 'react-highlighter';
+import styled from 'styled-components/macro';
+import { Entity, EntityType } from '../../types.generated';
+import { IconStyleType } from '../entity/Entity';
+import { ANTD_GRAY } from '../entity/shared/constants';
+import DomainIcon from './DomainIcon';
+import ParentEntities from '../search/filters/ParentEntities';
+import { getParentDomains } from './utils';
+import EntityRegistry from '../entity/EntityRegistry';
+
+type Props = {
+    entity: Entity;
+    entityRegistry: EntityRegistry;
+    query: string;
+    onResultClick: () => void;
+};
+
+const SearchResult = styled(Link)`
+    color: #262626;
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    height: 100%;
+    padding: 6px 8px;
+    width: 100%;
+    &:hover {
+        background-color: ${ANTD_GRAY[3]};
+        color: #262626;
+    }
+`;
+
+const IconWrapper = styled.span``;
+
+const highlightMatchStyle = {
+    fontWeight: 'bold',
+    background: 'none',
+    padding: 0,
+};
+
+function DomainSearchResultItem({ entity, entityRegistry, query, onResultClick }: Props) {
+    return (
+        <SearchResult to={entityRegistry.getEntityUrl(entity.type, entity.urn)} onClick={onResultClick}>
+            <IconWrapper>
+                {entity.type === EntityType.Domain ? (
+                    <DomainIcon
+                        style={{
+                            fontSize: 16,
+                            color: '#BFBFBF',
+                        }}
+                    />
+                ) : (
+                    entityRegistry.getIcon(entity.type, 12, IconStyleType.ACCENT)
+                )}
+            </IconWrapper>
+            <div>
+                <ParentEntities parentEntities={getParentDomains(entity, entityRegistry)} />
+                <Highlight matchStyle={highlightMatchStyle} search={query}>
+                    {entityRegistry.getDisplayName(entity.type, entity)}
+                </Highlight>
+            </div>
+        </SearchResult>
+    );
+}
+
+export default DomainSearchResultItem;
diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx
index 0fbcffb9a260c..8decc2840a379 100644
--- a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx
+++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx
@@ -1,9 +1,10 @@
-import { Alert } from 'antd';
+import { Alert, Empty } from 'antd';
 import React from 'react';
 import styled from 'styled-components';
 import useListDomains from '../../useListDomains';
 import DomainNode from './DomainNode';
 import { Domain } from '../../../../types.generated';
+import { ANTD_GRAY } from '../../../entity/shared/constants';
 
 const NavigatorWrapper = styled.div`
     font-size: 14px;
@@ -19,19 +20,28 @@ interface Props {
 
 export default function DomainNavigator({ domainUrnToHide, selectDomainOverride }: Props) {
     const { sortedDomains, error } = useListDomains({});
+    const noDomainsFound: boolean = !sortedDomains || sortedDomains.length === 0;
 
     return (
         <NavigatorWrapper>
             {error && <Alert message="Loading Domains failed." showIcon type="error" />}
-            {sortedDomains?.map((domain) => (
-                <DomainNode
-                    key={domain.urn}
-                    domain={domain as Domain}
-                    numDomainChildren={domain.children?.total || 0}
-                    domainUrnToHide={domainUrnToHide}
-                    selectDomainOverride={selectDomainOverride}
+            {noDomainsFound && (
+                <Empty
+                    description="No Domains Found"
+                    image={Empty.PRESENTED_IMAGE_SIMPLE}
+                    style={{ color: ANTD_GRAY[7] }}
                 />
-            ))}
+            )}
+            {!noDomainsFound &&
+                sortedDomains?.map((domain) => (
+                    <DomainNode
+                        key={domain.urn}
+                        domain={domain as Domain}
+                        numDomainChildren={domain.children?.total || 0}
+                        domainUrnToHide={domainUrnToHide}
+                        selectDomainOverride={selectDomainOverride}
+                    />
+                ))}
         </NavigatorWrapper>
     );
 }
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx
index 405442e8d7f50..3d9a7d7f08425 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx
@@ -1,6 +1,8 @@
 import React, { useRef, useState } from 'react';
-import { Button, Form, message, Modal, Select } from 'antd';
+import { Button, Form, message, Modal, Select, Empty } from 'antd';
+import { LoadingOutlined } from '@ant-design/icons';
 
+import styled from 'styled-components/macro';
 import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/search.generated';
 import { Domain, Entity, EntityType } from '../../../../../../../types.generated';
 import { useBatchSetDomainMutation } from '../../../../../../../graphql/mutations.generated';
@@ -12,6 +14,7 @@ import { tagRender } from '../tagRenderer';
 import { BrowserWrapper } from '../../../../../../shared/tags/AddTagsTermsModal';
 import DomainNavigator from '../../../../../../domain/nestedDomains/domainNavigator/DomainNavigator';
 import ClickOutside from '../../../../../../shared/ClickOutside';
+import { ANTD_GRAY } from '../../../../constants';
 
 type Props = {
     urns: string[];
@@ -28,6 +31,18 @@ type SelectedDomain = {
     urn: string;
 };
 
+const LoadingWrapper = styled.div`
+    padding: 8px;
+    display: flex;
+    justify-content: center;
+
+    svg {
+        height: 15px;
+        width: 15px;
+        color: ${ANTD_GRAY[8]};
+    }
+`;
+
 export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOkOverride, titleOverride }: Props) => {
     const entityRegistry = useEntityRegistry();
     const [isFocusedOnInput, setIsFocusedOnInput] = useState(false);
@@ -41,7 +56,7 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk
               }
             : undefined,
     );
-    const [domainSearch, { data: domainSearchData }] = useGetSearchResultsLazyQuery();
+    const [domainSearch, { data: domainSearchData, loading }] = useGetSearchResultsLazyQuery();
     const domainSearchResults =
         domainSearchData?.search?.searchResults?.map((searchResult) => searchResult.entity) || [];
     const [batchSetDomainMutation] = useBatchSetDomainMutation();
@@ -206,8 +221,23 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk
                             onBlur={handleBlur}
                             onFocus={() => setIsFocusedOnInput(true)}
                             dropdownStyle={isShowingDomainNavigator ? { display: 'none' } : {}}
+                            notFoundContent={
+                                <Empty
+                                    description="No Domains Found"
+                                    image={Empty.PRESENTED_IMAGE_SIMPLE}
+                                    style={{ color: ANTD_GRAY[7] }}
+                                />
+                            }
                         >
-                            {domainSearchOptions}
+                            {loading ? (
+                                <Select.Option value="loading">
+                                    <LoadingWrapper>
+                                        <LoadingOutlined />
+                                    </LoadingWrapper>
+                                </Select.Option>
+                            ) : (
+                                domainSearchOptions
+                            )}
                         </Select>
                         <BrowserWrapper isHidden={!isShowingDomainNavigator}>
                             <DomainNavigator selectDomainOverride={selectDomainFromBrowser} />

From 15efa72728bb7ab48fb8609236f32f80f56c4817 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20L=C3=BCdin?=
 <13187726+Masterchen09@users.noreply.github.com>
Date: Thu, 16 Nov 2023 18:39:14 +0100
Subject: [PATCH 217/259] fix(ui): show external url also in entity profile of
 containers (#8834)

---
 datahub-web-react/src/app/entity/container/ContainerEntity.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
index 9aecf6900f634..6c683a27295bd 100644
--- a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
+++ b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
@@ -167,6 +167,7 @@ export class ContainerEntity implements Entity<Container> {
     getOverridePropertiesFromEntity = (data: Container) => {
         return {
             name: this.displayName(data),
+            externalUrl: data.properties?.externalUrl,
             entityCount: data.entities?.total,
         };
     };

From 78abeb9bebee0c01a9d5725b6933016061a5009b Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Thu, 16 Nov 2023 12:41:12 -0500
Subject: [PATCH 218/259] feat(ingest/unity): Support specifying catalogs
 directly; pass env correctly (#9110)

---
 docs/how/updating-datahub.md                  |  3 +++
 .../datahub/ingestion/source/unity/config.py  | 10 ++++++-
 .../datahub/ingestion/source/unity/proxy.py   |  9 +++++++
 .../datahub/ingestion/source/unity/source.py  | 27 ++++++++++++++++---
 4 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 90b53161950e8..4a82e30103f81 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -68,6 +68,9 @@ qualified dataset name, i.e. `<project_name>.<dataset_name>`. We attempt to supp
 pattern format by prepending `.*\\.` to dataset patterns lacking a period, so in most cases this
 should not cause any issues. However, if you have a complex dataset pattern, we recommend you
 manually convert it to the fully qualified format to avoid any potential issues.
+- #9110 - The Unity Catalog source will now generate urns based on `env` properly. If you have
+been setting `env` in your recipe to something besides `PROD`, we will now generate urns
+with that new env variable, invalidating your existing urns.
 
 ### Potential Downtime
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
index 7073830318abe..4e3deedddbc43 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py
@@ -1,7 +1,7 @@
 import logging
 import os
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 import pydantic
 from pydantic import Field
@@ -132,6 +132,14 @@ class UnityCatalogSourceConfig(
 
     _metastore_id_pattern_removed = pydantic_removed_field("metastore_id_pattern")
 
+    catalogs: Optional[List[str]] = pydantic.Field(
+        default=None,
+        description=(
+            "Fixed list of catalogs to ingest."
+            " If not specified, catalogs will be ingested based on `catalog_pattern`."
+        ),
+    )
+
     catalog_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern.allow_all(),
         description="Regex patterns for catalogs to filter in ingestion. Specify regex to match the full `metastore.catalog` name.",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
index 3fb77ce512ed2..375c76db8e971 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py
@@ -112,6 +112,15 @@ def catalogs(self, metastore: Optional[Metastore]) -> Iterable[Catalog]:
         for catalog in response:
             yield self._create_catalog(metastore, catalog)
 
+    def catalog(
+        self, catalog_name: str, metastore: Optional[Metastore]
+    ) -> Optional[Catalog]:
+        response = self._workspace_client.catalogs.get(catalog_name)
+        if not response:
+            logger.info(f"Catalog {catalog_name} not found")
+            return None
+        return self._create_catalog(metastore, response)
+
     def schemas(self, catalog: Catalog) -> Iterable[Schema]:
         response = self._workspace_client.schemas.list(catalog_name=catalog.name)
         if not response:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
index b63cf65d55dc8..44b5bbbcb0ceb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py
@@ -188,9 +188,10 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        self.report.report_ingestion_stage_start("Start warehouse")
+        self.report.report_ingestion_stage_start("Ingestion Setup")
         wait_on_warehouse = None
         if self.config.is_profiling_enabled():
+            self.report.report_ingestion_stage_start("Start warehouse")
             # Can take several minutes, so start now and wait later
             wait_on_warehouse = self.unity_catalog_api_proxy.start_warehouse()
             if wait_on_warehouse is None:
@@ -200,8 +201,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 )
                 return
 
-        self.report.report_ingestion_stage_start("Ingest service principals")
-        self.build_service_principal_map()
+        if self.config.include_ownership:
+            self.report.report_ingestion_stage_start("Ingest service principals")
+            self.build_service_principal_map()
         if self.config.include_notebooks:
             self.report.report_ingestion_stage_start("Ingest notebooks")
             yield from self.process_notebooks()
@@ -317,7 +319,7 @@ def process_metastores(self) -> Iterable[MetadataWorkUnit]:
     def process_catalogs(
         self, metastore: Optional[Metastore]
     ) -> Iterable[MetadataWorkUnit]:
-        for catalog in self.unity_catalog_api_proxy.catalogs(metastore=metastore):
+        for catalog in self._get_catalogs(metastore):
             if not self.config.catalog_pattern.allowed(catalog.id):
                 self.report.catalogs.dropped(catalog.id)
                 continue
@@ -327,6 +329,17 @@ def process_catalogs(
 
             self.report.catalogs.processed(catalog.id)
 
+    def _get_catalogs(self, metastore: Optional[Metastore]) -> Iterable[Catalog]:
+        if self.config.catalogs:
+            for catalog_name in self.config.catalogs:
+                catalog = self.unity_catalog_api_proxy.catalog(
+                    catalog_name, metastore=metastore
+                )
+                if catalog:
+                    yield catalog
+        else:
+            yield from self.unity_catalog_api_proxy.catalogs(metastore=metastore)
+
     def process_schemas(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]:
         for schema in self.unity_catalog_api_proxy.schemas(catalog=catalog):
             if not self.config.schema_pattern.allowed(schema.id):
@@ -509,6 +522,7 @@ def gen_dataset_urn(self, table_ref: TableReference) -> str:
             platform=self.platform,
             platform_instance=self.platform_instance_name,
             name=str(table_ref),
+            env=self.config.env,
         )
 
     def gen_notebook_urn(self, notebook: Union[Notebook, NotebookId]) -> str:
@@ -576,6 +590,7 @@ def gen_schema_key(self, schema: Schema) -> ContainerKey:
                 instance=self.config.platform_instance,
                 catalog=schema.catalog.name,
                 metastore=schema.catalog.metastore.name,
+                env=self.config.env,
             )
         else:
             return UnitySchemaKey(
@@ -583,6 +598,7 @@ def gen_schema_key(self, schema: Schema) -> ContainerKey:
                 platform=self.platform,
                 instance=self.config.platform_instance,
                 catalog=schema.catalog.name,
+                env=self.config.env,
             )
 
     def gen_metastore_key(self, metastore: Metastore) -> MetastoreKey:
@@ -590,6 +606,7 @@ def gen_metastore_key(self, metastore: Metastore) -> MetastoreKey:
             metastore=metastore.name,
             platform=self.platform,
             instance=self.config.platform_instance,
+            env=self.config.env,
         )
 
     def gen_catalog_key(self, catalog: Catalog) -> ContainerKey:
@@ -600,12 +617,14 @@ def gen_catalog_key(self, catalog: Catalog) -> ContainerKey:
                 metastore=catalog.metastore.name,
                 platform=self.platform,
                 instance=self.config.platform_instance,
+                env=self.config.env,
             )
         else:
             return CatalogKey(
                 catalog=catalog.name,
                 platform=self.platform,
                 instance=self.config.platform_instance,
+                env=self.config.env,
             )
 
     def _gen_domain_urn(self, dataset_name: str) -> Optional[str]:

From e15e28e2d62111e00e9e1f0aa4cd0356d8d09f9f Mon Sep 17 00:00:00 2001
From: Patrick Franco Braz <patrickfbraz@poli.ufrj.br>
Date: Thu, 16 Nov 2023 14:49:40 -0300
Subject: [PATCH 219/259] refactor(datahub-web-react): allows proxying to
 external datahub-frontend servers (#9250)

---
 datahub-web-react/.env              | 3 ++-
 datahub-web-react/README.md         | 8 ++++++++
 datahub-web-react/src/setupProxy.js | 8 +++++---
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/datahub-web-react/.env b/datahub-web-react/.env
index d503159ecaf97..e5529bbdaa56d 100644
--- a/datahub-web-react/.env
+++ b/datahub-web-react/.env
@@ -1,4 +1,5 @@
 PUBLIC_URL=/assets
 REACT_APP_THEME_CONFIG=theme_light.config.json
 SKIP_PREFLIGHT_CHECK=true
-BUILD_PATH=build/yarn
\ No newline at end of file
+BUILD_PATH=build/yarn
+REACT_APP_PROXY_TARGET=http://localhost:9002
\ No newline at end of file
diff --git a/datahub-web-react/README.md b/datahub-web-react/README.md
index 8bf592b11a0ae..560f5315b2c71 100644
--- a/datahub-web-react/README.md
+++ b/datahub-web-react/README.md
@@ -51,6 +51,14 @@ need to be deployed, still at `http://localhost:9002`, to service GraphQL API re
 
 Optionally you could also start the app with the mock server without running the docker containers by executing `yarn start:mock`. See [here](src/graphql-mock/fixtures/searchResult/userSearchResult.ts#L6) for available login users.
 
+### Testing your customizations
+
+There is two options to test your customizations:
+* **Option 1**: Initialize the docker containers with the `quickstart.sh` script (or if any custom docker-compose file) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at `http://localhost:9002` to fetch real data.
+* **Option 2**: Change the environment variable `REACT_APP_PROXY_TARGET` in the `.env` file to point to your `datahub-frontend` server (ex: https://my_datahub_host.com) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at some domain to fetch real data.
+
+The option 2 is useful if you want to test your React customizations without having to run the hole DataHub stack locally. However, if you changed other components of the DataHub stack, you will need to run the hole stack locally (building the docker images) and use the option 1.
+
 ### Functional testing
 
 In order to start a server and run frontend unit tests using react-testing-framework, run:
diff --git a/datahub-web-react/src/setupProxy.js b/datahub-web-react/src/setupProxy.js
index 478c015705a13..165e394a507f3 100644
--- a/datahub-web-react/src/setupProxy.js
+++ b/datahub-web-react/src/setupProxy.js
@@ -2,6 +2,8 @@ const logInFilter = function (pathname, req) {
   return pathname.match('^/logIn') && req.method === 'POST';
 };
 
+const proxyTarget = process.env.REACT_APP_PROXY_TARGET || 'http://localhost:9002';
+
 if (process.env.REACT_APP_MOCK === 'true' || process.env.REACT_APP_MOCK === 'cy') {
     // no proxy needed, MirageJS will intercept all http requests
     module.exports = function () {};
@@ -13,21 +15,21 @@ if (process.env.REACT_APP_MOCK === 'true' || process.env.REACT_APP_MOCK === 'cy'
         app.use(
             '/logIn',
             createProxyMiddleware(logInFilter, {
-                target: 'http://localhost:9002',
+                target: proxyTarget,
                 changeOrigin: true,
             }),
         );
         app.use(
             '/authenticate',
             createProxyMiddleware({
-                target: 'http://localhost:9002',
+                target: proxyTarget,
                 changeOrigin: true,
             }),
         );
         app.use(
             '/api/v2/graphql',
             createProxyMiddleware({
-                target: 'http://localhost:9002',
+                target: proxyTarget,
                 changeOrigin: true,
             }),
         );

From cd789b8d769b25a10870b75d10047aa09ca0977a Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Thu, 16 Nov 2023 12:09:28 -0600
Subject: [PATCH 220/259] chore(node): update node to non-EOL version (#9252)

---
 datahub-web-react/build.gradle | 4 ++--
 datahub-web-react/package.json | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle
index ae96ed130c1d1..13eabe90ee509 100644
--- a/datahub-web-react/build.gradle
+++ b/datahub-web-react/build.gradle
@@ -16,10 +16,10 @@ node {
   }
 
   // Version of node to use.
-  version = '16.8.0'
+  version = '21.2.0'
 
   // Version of Yarn to use.
-  yarnVersion = '1.22.0'
+  yarnVersion = '1.22.1'
 
   // Base URL for fetching node distributions (set nodeDistBaseUrl if you have a mirror).
   if (project.hasProperty('nodeDistBaseUrl')) {
diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index 019295f3e6ffe..5afbc8fa5892d 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -95,7 +95,7 @@
         "start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start",
         "start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start",
         "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start",
-        "build": "yarn run generate && CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/",
+        "build": "yarn run generate && NODE_OPTIONS=--openssl-legacy-provider CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/",
         "test": "craco test",
         "pretest:e2e:ci": "yarn generate",
         "test:e2e": "start-server-and-test start:e2e 3010",

From 9d41a8f9f096c2809afd5026c6f40cf4368669af Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 16 Nov 2023 13:33:35 -0500
Subject: [PATCH 221/259] fix(ingest): drop redshift-legacy and
 redshift-usage-legacy sources (#9244)

---
 docs/how/updating-datahub.md                  |    2 +
 metadata-ingestion/setup.py                   |    6 -
 .../ingestion/source/redshift/query.py        |    2 +-
 .../ingestion/source/source_registry.py       |   13 +-
 .../datahub/ingestion/source/sql/redshift.py  | 1198 -----------------
 .../ingestion/source/usage/redshift_usage.py  |  397 ------
 6 files changed, 4 insertions(+), 1614 deletions(-)
 delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py
 delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 4a82e30103f81..54f5775d8331f 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -6,6 +6,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
+- #9244: The `redshift-legacy` and `redshift-legacy-usage` sources, which have been deprecated for >6 months, have been removed. The new `redshift` source is a superset of the functionality provided by those legacy sources.
+
 ### Potential Downtime
 
 ### Deprecations
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index ebe180703051f..04ae03cd440ac 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -366,8 +366,6 @@
     | usage_common
     | {"redshift-connector"}
     | sqlglot_lib,
-    "redshift-legacy": sql_common | redshift_common | sqlglot_lib,
-    "redshift-usage-legacy": sql_common | redshift_common | sqlglot_lib | usage_common,
     "s3": {*s3_base, *data_lake_profiling},
     "gcs": {*s3_base, *data_lake_profiling},
     "sagemaker": aws_common,
@@ -510,8 +508,6 @@
             "presto",
             "redash",
             "redshift",
-            "redshift-legacy",
-            "redshift-usage-legacy",
             "s3",
             "snowflake",
             "tableau",
@@ -608,8 +604,6 @@
         "postgres = datahub.ingestion.source.sql.postgres:PostgresSource",
         "redash = datahub.ingestion.source.redash:RedashSource",
         "redshift = datahub.ingestion.source.redshift.redshift:RedshiftSource",
-        "redshift-legacy = datahub.ingestion.source.sql.redshift:RedshiftSource",
-        "redshift-usage-legacy = datahub.ingestion.source.usage.redshift_usage:RedshiftUsageSource",
         "snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source",
         "superset = datahub.ingestion.source.superset:SupersetSource",
         "tableau = datahub.ingestion.source.tableau:TableauSource",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
index 0b57c41131714..a96171caf9835 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from datahub.ingestion.source.sql.redshift import redshift_datetime_format
+redshift_datetime_format = "%Y-%m-%d %H:%M:%S"
 
 
 class RedshiftQuery:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/source_registry.py b/metadata-ingestion/src/datahub/ingestion/source/source_registry.py
index 37f088bcd7b50..c3fbab3f9a012 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/source_registry.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/source_registry.py
@@ -1,6 +1,3 @@
-import warnings
-
-from datahub.configuration.common import ConfigurationWarning
 from datahub.ingestion.api.registry import PluginRegistry
 from datahub.ingestion.api.source import Source
 
@@ -8,15 +5,7 @@
 source_registry.register_from_entrypoint("datahub.ingestion.source.plugins")
 
 # Deprecations.
-source_registry.register_alias(
-    "redshift-usage",
-    "redshift-usage-legacy",
-    lambda: warnings.warn(
-        "source type redshift-usage is deprecated, use redshift source instead as usage was merged into the main source",
-        ConfigurationWarning,
-        stacklevel=3,
-    ),
-)
+# source_registry.register_alias(<new_name>, <old_name>, <deprecation_message>)
 
 # The MSSQL source has two possible sets of dependencies. We alias
 # the second to the first so that we maintain the 1:1 mapping between
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py
deleted file mode 100644
index 33d517c8589e9..0000000000000
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py
+++ /dev/null
@@ -1,1198 +0,0 @@
-import logging
-from collections import defaultdict
-from dataclasses import dataclass, field
-from enum import Enum
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
-from urllib.parse import urlparse
-
-# These imports verify that the dependencies are available.
-import psycopg2  # noqa: F401
-import sqlalchemy
-import sqlalchemy_redshift  # noqa: F401
-from pydantic.fields import Field
-from sqlalchemy import create_engine, inspect
-from sqlalchemy.engine import Connection, reflection
-from sqlalchemy.engine.reflection import Inspector
-from sqlalchemy_redshift.dialect import RedshiftDialect, RelationKey
-from sqllineage.runner import LineageRunner
-
-import datahub.emitter.mce_builder as builder
-from datahub.configuration import ConfigModel
-from datahub.configuration.source_common import DatasetLineageProviderConfigBase
-from datahub.configuration.time_window_config import BaseTimeWindowConfig
-from datahub.emitter import mce_builder
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.api.decorators import (
-    SourceCapability,
-    SupportStatus,
-    capability,
-    config_class,
-    platform_name,
-    support_status,
-)
-from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.aws.s3_util import strip_s3_prefix
-from datahub.ingestion.source.data_lake_common.path_spec import PathSpec
-from datahub.ingestion.source.sql.postgres import BasePostgresConfig
-from datahub.ingestion.source.sql.sql_common import (
-    SQLAlchemySource,
-    SQLSourceReport,
-    SqlWorkUnit,
-)
-
-# TRICKY: it's necessary to import the Postgres source because
-# that module has some side effects that we care about here.
-from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage
-from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
-from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
-from datahub.metadata.schema_classes import (
-    ChangeTypeClass,
-    DatasetLineageTypeClass,
-    DatasetPropertiesClass,
-    DatasetSnapshotClass,
-    UpstreamClass,
-)
-
-logger: logging.Logger = logging.getLogger(__name__)
-
-
-class LineageMode(Enum):
-    SQL_BASED = "sql_based"
-    STL_SCAN_BASED = "stl_scan_based"
-    MIXED = "mixed"
-
-
-class LineageCollectorType(Enum):
-    QUERY_SCAN = "query_scan"
-    QUERY_SQL_PARSER = "query_sql_parser"
-    VIEW = "view"
-    NON_BINDING_VIEW = "non-binding-view"
-    COPY = "copy"
-    UNLOAD = "unload"
-
-
-class LineageDatasetPlatform(Enum):
-    S3 = "s3"
-    REDSHIFT = "redshift"
-
-
-@dataclass(frozen=True, eq=True)
-class LineageDataset:
-    platform: LineageDatasetPlatform
-    path: str
-
-
-@dataclass
-class LineageItem:
-    dataset: LineageDataset
-    upstreams: Set[LineageDataset]
-    collector_type: LineageCollectorType
-    dataset_lineage_type: str = field(init=False)
-    query_parser_failed_sqls: List[str]
-
-    def __post_init__(self):
-        if self.collector_type == LineageCollectorType.COPY:
-            self.dataset_lineage_type = DatasetLineageTypeClass.COPY
-        elif self.collector_type in [
-            LineageCollectorType.VIEW,
-            LineageCollectorType.NON_BINDING_VIEW,
-        ]:
-            self.dataset_lineage_type = DatasetLineageTypeClass.VIEW
-        else:
-            self.dataset_lineage_type = DatasetLineageTypeClass.TRANSFORMED
-
-
-class S3LineageProviderConfig(ConfigModel):
-    """
-    Any source that produces s3 lineage from/to Datasets should inherit this class.
-    """
-
-    path_specs: List[PathSpec] = Field(
-        description="List of PathSpec. See below the details about PathSpec"
-    )
-
-
-class DatasetS3LineageProviderConfigBase(ConfigModel):
-    """
-    Any source that produces s3 lineage from/to Datasets should inherit this class.
-    """
-
-    s3_lineage_config: Optional[S3LineageProviderConfig] = Field(
-        default=None, description="Common config for S3 lineage generation"
-    )
-
-
-class RedshiftConfig(
-    BasePostgresConfig,
-    BaseTimeWindowConfig,
-    DatasetLineageProviderConfigBase,
-    DatasetS3LineageProviderConfigBase,
-):
-    def get_identifier(self, schema: str, table: str) -> str:
-        regular = f"{schema}.{table}"
-        if self.database_alias:
-            return f"{self.database_alias}.{regular}"
-        if self.database:
-            return f"{self.database}.{regular}"
-        return regular
-
-    # Although Amazon Redshift is compatible with Postgres's wire format,
-    # we actually want to use the sqlalchemy-redshift package and dialect
-    # because it has better caching behavior. In particular, it queries
-    # the full table, column, and constraint information in a single larger
-    # query, and then simply pulls out the relevant information as needed.
-    # Because of this behavior, it uses dramatically fewer round trips for
-    # large Redshift warehouses. As an example, see this query for the columns:
-    # https://github.com/sqlalchemy-redshift/sqlalchemy-redshift/blob/60b4db04c1d26071c291aeea52f1dcb5dd8b0eb0/sqlalchemy_redshift/dialect.py#L745.
-    scheme: str = Field(
-        default="redshift+psycopg2",
-        description="",
-        hidden_from_docs=True,
-    )
-
-    default_schema: str = Field(
-        default="public",
-        description="The default schema to use if the sql parser fails to parse the schema with `sql_based` lineage collector",
-    )
-
-    include_table_lineage: Optional[bool] = Field(
-        default=True, description="Whether table lineage should be ingested."
-    )
-    include_copy_lineage: Optional[bool] = Field(
-        default=True,
-        description="Whether lineage should be collected from copy commands",
-    )
-    include_unload_lineage: Optional[bool] = Field(
-        default=True,
-        description="Whether lineage should be collected from unload commands",
-    )
-    capture_lineage_query_parser_failures: Optional[bool] = Field(
-        default=False,
-        description="Whether to capture lineage query parser errors with dataset properties for debuggings",
-    )
-
-    table_lineage_mode: Optional[LineageMode] = Field(
-        default=LineageMode.STL_SCAN_BASED,
-        description="Which table lineage collector mode to use. Available modes are: [stl_scan_based, sql_based, mixed]",
-    )
-
-
-# reflection.cache uses eval and other magic to partially rewrite the function.
-# mypy can't handle it, so we ignore it for now.
-@reflection.cache  # type: ignore
-def _get_all_table_comments(self, connection, **kw):
-    COMMENT_SQL = """
-        SELECT n.nspname as schema,
-               c.relname as table_name,
-               pgd.description as table_comment
-        FROM pg_catalog.pg_class c
-            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-            LEFT JOIN pg_catalog.pg_description pgd ON pgd.objsubid = 0 AND pgd.objoid = c.oid
-        WHERE c.relkind in ('r', 'v', 'm', 'f', 'p')
-          AND pgd.description IS NOT NULL
-        ORDER BY "schema", "table_name";
-    """
-
-    all_table_comments: Dict[RelationKey, str] = {}
-
-    result = connection.execute(COMMENT_SQL)
-    for table in result:
-        key = RelationKey(table.table_name, table.schema, connection)
-        all_table_comments[key] = table.table_comment
-
-    return all_table_comments
-
-
-@reflection.cache  # type: ignore
-def get_table_comment(self, connection, table_name, schema=None, **kw):
-    all_table_comments = self._get_all_table_comments(connection, **kw)
-    key = RelationKey(table_name, schema, connection)
-    if key not in all_table_comments.keys():
-        key = key.unquoted()
-    return {"text": all_table_comments.get(key)}
-
-
-# gets all the relations for internal schemas and external schemas
-# by UNION of internal schemas (excluding namespaces starting with pg_)
-# and external schemas
-@reflection.cache  # type: ignore
-def _get_all_relation_info(self, connection, **kw):
-    result = connection.execute(
-        """
-        SELECT c.relkind,
-            n.oid AS "schema_oid",
-            n.nspname AS "schema",
-            c.oid AS "rel_oid",
-            c.relname,
-            CASE c.reldiststyle
-                WHEN 0 THEN 'EVEN'
-                WHEN 1 THEN 'KEY'
-                WHEN 8 THEN 'ALL'
-            END AS "diststyle",
-            c.relowner AS "owner_id",
-            u.usename AS "owner_name",
-            TRIM(TRAILING ';' FROM pg_catalog.pg_get_viewdef (c.oid,TRUE)) AS "view_definition",
-            pg_catalog.array_to_string(c.relacl,'\n') AS "privileges"
-        FROM pg_catalog.pg_class c
-        LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-        JOIN pg_catalog.pg_user u ON u.usesysid = c.relowner
-        WHERE c.relkind IN ('r','v','m','S','f')
-        AND   n.nspname !~ '^pg_'
-        AND   n.nspname != 'information_schema'
-        UNION
-        SELECT 'r' AS "relkind",
-            NULL AS "schema_oid",
-            schemaname AS "schema",
-            NULL AS "rel_oid",
-            tablename AS "relname",
-            NULL AS "diststyle",
-            NULL AS "owner_id",
-            NULL AS "owner_name",
-            NULL AS "view_definition",
-            NULL AS "privileges"
-        FROM pg_catalog.svv_external_tables
-        ORDER BY "schema",
-                "relname";"""
-    )
-    relations = {}
-    for rel in result:
-        key = RelationKey(rel.relname, rel.schema, connection)
-        relations[key] = rel
-    return relations
-
-
-# workaround to get external tables
-# Rewriting some external table types to match redshift type based on
-# this redshift-sqlalchemy pull request:
-#   https://github.com/sqlalchemy-redshift/sqlalchemy-redshift/pull/163/files
-# The mapping of external types to redshift types:
-#   (https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html):
-# External type -> Redshift type
-#   int -> integer
-#   decimal -> numeric
-#   char -> character
-#   float -> real
-#   double -> float
-@reflection.cache  # type: ignore
-def _get_schema_column_info(self, connection, schema=None, **kw):
-    schema_clause = "AND schema = '{schema}'".format(schema=schema) if schema else ""
-    all_columns = defaultdict(list)
-
-    with connection.connect() as cc:
-        result = cc.execute(
-            """
-            SELECT
-              n.nspname as "schema",
-              c.relname as "table_name",
-              att.attname as "name",
-              format_encoding(att.attencodingtype::integer) as "encode",
-              format_type(att.atttypid, att.atttypmod) as "type",
-              att.attisdistkey as "distkey",
-              att.attsortkeyord as "sortkey",
-              att.attnotnull as "notnull",
-              pg_catalog.col_description(att.attrelid, att.attnum)
-                as "comment",
-              adsrc,
-              attnum,
-              pg_catalog.format_type(att.atttypid, att.atttypmod),
-              pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS DEFAULT,
-              n.oid as "schema_oid",
-              c.oid as "table_oid"
-            FROM pg_catalog.pg_class c
-            LEFT JOIN pg_catalog.pg_namespace n
-              ON n.oid = c.relnamespace
-            JOIN pg_catalog.pg_attribute att
-              ON att.attrelid = c.oid
-            LEFT JOIN pg_catalog.pg_attrdef ad
-              ON (att.attrelid, att.attnum) = (ad.adrelid, ad.adnum)
-            WHERE n.nspname !~ '^pg_'
-              AND att.attnum > 0
-              AND NOT att.attisdropped
-              {schema_clause}
-            UNION
-            SELECT
-              view_schema as "schema",
-              view_name as "table_name",
-              col_name as "name",
-              null as "encode",
-              col_type as "type",
-              null as "distkey",
-              0 as "sortkey",
-              null as "notnull",
-              null as "comment",
-              null as "adsrc",
-              null as "attnum",
-              col_type as "format_type",
-              null as "default",
-              null as "schema_oid",
-              null as "table_oid"
-            FROM pg_get_late_binding_view_cols() cols(
-              view_schema name,
-              view_name name,
-              col_name name,
-              col_type varchar,
-              col_num int)
-            WHERE 1 {schema_clause}
-            UNION
-            SELECT
-              schemaname as "schema",
-              tablename as "table_name",
-              columnname as "name",
-              null as "encode",
-              -- Spectrum represents data types differently.
-              -- Standardize, so we can infer types.
-              CASE
-                WHEN external_type = 'int' THEN 'integer'
-                 ELSE
-                   regexp_replace(
-                   replace(
-                   replace(
-                   replace(
-                   replace(
-                   replace(
-                   replace(external_type, 'decimal', 'numeric'),
-                    'varchar', 'character varying'),
-                    'string', 'character varying'),
-                    'char(', 'character('),
-                    'float', 'real'),
-                    'double', 'float'),
-                    '^array<(.*)>$', '$1[]', 1, 'p')
-                 END AS "type",
-              null as "distkey",
-              0 as "sortkey",
-              null as "notnull",
-              null as "comment",
-              null as "adsrc",
-              null as "attnum",
-              CASE
-                 WHEN external_type = 'int' THEN 'integer'
-                 ELSE
-                   regexp_replace(
-                   replace(
-                   replace(
-                   replace(
-                   replace(
-                   replace(
-                   replace(external_type, 'decimal', 'numeric'),
-                    'varchar', 'character varying'),
-                    'string', 'character varying'),
-                    'char(', 'character('),
-                    'float', 'real'),
-                    'double', 'float'),
-                    '^array<(.*)>$', '$1[]', 1, 'p')
-                 END AS "format_type",
-              null as "default",
-              null as "schema_oid",
-              null as "table_oid"
-            FROM SVV_EXTERNAL_COLUMNS
-            WHERE 1 {schema_clause}
-            ORDER BY "schema", "table_name", "attnum"
-        """.format(
-                schema_clause=schema_clause
-            )
-        )
-        for col in result:
-            key = RelationKey(col.table_name, col.schema, connection)
-            all_columns[key].append(col)
-    return dict(all_columns)
-
-
-def _get_external_db_mapping(connection):
-    # SQL query to get mapping of external schemas in redshift to its external database.
-    return connection.execute(
-        """
-        select * from svv_external_schemas
-        """
-    )
-
-
-# This monkey-patching enables us to batch fetch the table descriptions, rather than
-# fetching them one at a time.
-RedshiftDialect._get_all_table_comments = _get_all_table_comments
-RedshiftDialect.get_table_comment = get_table_comment
-RedshiftDialect._get_all_relation_info = _get_all_relation_info
-RedshiftDialect._get_schema_column_info = _get_schema_column_info
-
-redshift_datetime_format = "%Y-%m-%d %H:%M:%S"
-
-
-@dataclass
-class RedshiftReport(SQLSourceReport):
-    # https://forums.aws.amazon.com/ann.jspa?annID=9105
-    saas_version: str = ""
-    upstream_lineage: Dict[str, List[str]] = field(default_factory=dict)
-
-
-@platform_name("Redshift")
-@config_class(RedshiftConfig)
-@support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
-@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
-@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
-@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
-@capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration")
-@capability(
-    SourceCapability.USAGE_STATS,
-    "Not provided by this module, use `redshift-usage` for that.",
-    supported=False,
-)
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
-class RedshiftSource(SQLAlchemySource):
-    """
-    This plugin extracts the following:
-
-    - Metadata for databases, schemas, views and tables
-    - Column types associated with each table
-    - Also supports PostGIS extensions
-    - Table, row, and column statistics via optional SQL profiling
-    - Table lineage
-
-    :::tip
-
-    You can also get fine-grained usage statistics for Redshift using the `redshift-usage` source described below.
-
-    :::
-
-    ### Prerequisites
-
-    This source needs to access system tables that require extra permissions.
-    To grant these permissions, please alter your datahub Redshift user the following way:
-    ```sql
-    ALTER USER datahub_user WITH SYSLOG ACCESS UNRESTRICTED;
-    GRANT SELECT ON pg_catalog.svv_table_info to datahub_user;
-    GRANT SELECT ON pg_catalog.svl_user_info to datahub_user;
-    ```
-    :::note
-
-    Giving a user unrestricted access to system tables gives the user visibility to data generated by other users. For example, STL_QUERY and STL_QUERYTEXT contain the full text of INSERT, UPDATE, and DELETE statements.
-
-    :::
-
-    ### Lineage
-
-    There are multiple lineage collector implementations as Redshift does not support table lineage out of the box.
-
-    #### stl_scan_based
-    The stl_scan based collector uses Redshift's [stl_insert](https://docs.aws.amazon.com/redshift/latest/dg/r_STL_INSERT.html) and [stl_scan](https://docs.aws.amazon.com/redshift/latest/dg/r_STL_SCAN.html) system tables to
-    discover lineage between tables.
-    Pros:
-    - Fast
-    - Reliable
-
-    Cons:
-    - Does not work with Spectrum/external tables because those scans do not show up in stl_scan table.
-    - If a table is depending on a view then the view won't be listed as dependency. Instead the table will be connected with the view's dependencies.
-
-    #### sql_based
-    The sql_based based collector uses Redshift's [stl_insert](https://docs.aws.amazon.com/redshift/latest/dg/r_STL_INSERT.html) to discover all the insert queries
-    and uses sql parsing to discover the dependecies.
-
-    Pros:
-    - Works with Spectrum tables
-    - Views are connected properly if a table depends on it
-
-    Cons:
-    - Slow.
-    - Less reliable as the query parser can fail on certain queries
-
-    #### mixed
-    Using both collector above and first applying the sql based and then the stl_scan based one.
-
-    Pros:
-    - Works with Spectrum tables
-    - Views are connected properly if a table depends on it
-    - A bit more reliable than the sql_based one only
-
-    Cons:
-    - Slow
-    - May be incorrect at times as the query parser can fail on certain queries
-
-    :::note
-
-    The redshift stl redshift tables which are used for getting data lineage only retain approximately two to five days of log history. This means you cannot extract lineage from queries issued outside that window.
-
-    :::
-
-    """
-
-    eskind_to_platform = {1: "glue", 2: "hive", 3: "postgres", 4: "redshift"}
-
-    def __init__(self, config: RedshiftConfig, ctx: PipelineContext):
-        super().__init__(config, ctx, "redshift")
-        self.catalog_metadata: Dict = {}
-        self.config: RedshiftConfig = config
-        self._lineage_map: Optional[Dict[str, LineageItem]] = None
-        self._all_tables_set: Optional[Set[str]] = None
-        self.report: RedshiftReport = RedshiftReport()
-
-    @classmethod
-    def create(cls, config_dict, ctx):
-        config = RedshiftConfig.parse_obj(config_dict)
-        return cls(config, ctx)
-
-    def get_catalog_metadata(self, conn: Connection) -> None:
-        try:
-            catalog_metadata = _get_external_db_mapping(conn)
-        except Exception as e:
-            self.error(logger, "external-svv_external_schemas", f"Error was {e}")
-            return
-
-        db_name = self.get_db_name()
-
-        external_schema_mapping = {}
-        for rel in catalog_metadata:
-            if rel.eskind != 1:
-                logger.debug(
-                    f"Skipping {rel.schemaname} for mapping to external database as currently we only "
-                    f"support glue"
-                )
-                continue
-            external_schema_mapping[rel.schemaname] = {
-                "eskind": rel.eskind,
-                "external_database": rel.databasename,
-                "esoptions": rel.esoptions,
-                "esoid": rel.esoid,
-                "esowner": rel.esowner,
-            }
-        self.catalog_metadata[db_name] = external_schema_mapping
-
-    def get_inspectors(self) -> Iterable[Inspector]:
-        # This method can be overridden in the case that you want to dynamically
-        # run on multiple databases.
-        engine = self.get_metadata_engine()
-        with engine.connect() as conn:
-            self.get_catalog_metadata(conn)
-            inspector = inspect(conn)
-            yield inspector
-
-    def get_metadata_engine(self) -> sqlalchemy.engine.Engine:
-        url = self.config.get_sql_alchemy_url()
-        logger.debug(f"sql_alchemy_url={url}")
-        return create_engine(url, **self.config.options)
-
-    def inspect_version(self) -> Any:
-        db_engine = self.get_metadata_engine()
-        logger.info("Checking current version")
-        for db_row in db_engine.execute("select version()"):
-            self.report.saas_version = db_row[0]
-
-    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
-        try:
-            self.inspect_version()
-        except Exception as e:
-            self.report.report_failure("version", f"Error: {e}")
-            return
-
-        for wu in super().get_workunits_internal():
-            yield wu
-            if (
-                isinstance(wu, SqlWorkUnit)
-                and isinstance(wu.metadata, MetadataChangeEvent)
-                and isinstance(wu.metadata.proposedSnapshot, DatasetSnapshot)
-            ):
-                lineage_mcp = None
-                lineage_properties_aspect: Optional[DatasetPropertiesClass] = None
-
-                dataset_snapshot: DatasetSnapshotClass = wu.metadata.proposedSnapshot
-                assert dataset_snapshot
-
-                if self.config.include_table_lineage:
-                    lineage_mcp, lineage_properties_aspect = self.get_lineage_mcp(
-                        wu.metadata.proposedSnapshot.urn
-                    )
-
-                if lineage_mcp is not None:
-                    yield lineage_mcp.as_workunit()
-
-                if lineage_properties_aspect:
-                    aspects = dataset_snapshot.aspects
-                    if aspects is None:
-                        aspects = []
-
-                    dataset_properties_aspect: Optional[DatasetPropertiesClass] = None
-
-                    for aspect in aspects:
-                        if isinstance(aspect, DatasetPropertiesClass):
-                            dataset_properties_aspect = aspect
-
-                    if dataset_properties_aspect is None:
-                        dataset_properties_aspect = DatasetPropertiesClass()
-                        aspects.append(dataset_properties_aspect)
-
-                    custom_properties = (
-                        {
-                            **dataset_properties_aspect.customProperties,
-                            **lineage_properties_aspect.customProperties,
-                        }
-                        if dataset_properties_aspect.customProperties
-                        else lineage_properties_aspect.customProperties
-                    )
-                    dataset_properties_aspect.customProperties = custom_properties
-                    dataset_snapshot.aspects = aspects
-
-                    dataset_snapshot.aspects.append(dataset_properties_aspect)
-
-    def _get_all_tables(self) -> Set[str]:
-        all_tables_query: str = """
-        select
-            table_schema as schemaname,
-            table_name as tablename
-        from
-            pg_catalog.svv_tables
-        where
-            table_type = 'BASE TABLE'
-            and table_schema not in ('information_schema', 'pg_catalog', 'pg_internal')
-        union
-        select
-            distinct schemaname,
-            tablename
-        from
-            svv_external_tables
-        union
-        SELECT
-            n.nspname AS schemaname
-            ,c.relname AS tablename
-        FROM
-            pg_catalog.pg_class AS c
-        INNER JOIN
-            pg_catalog.pg_namespace AS n
-            ON c.relnamespace = n.oid
-        WHERE relkind = 'v'
-        and
-        n.nspname not in ('pg_catalog', 'information_schema')
-
-        """
-        db_name = self.get_db_name()
-        all_tables_set = set()
-
-        engine = self.get_metadata_engine()
-        for db_row in engine.execute(all_tables_query):
-            all_tables_set.add(
-                f'{db_name}.{db_row["schemaname"]}.{db_row["tablename"]}'
-            )
-
-        return all_tables_set
-
-    def _get_sources_from_query(self, db_name: str, query: str) -> List[LineageDataset]:
-        sources = list()
-
-        parser = LineageRunner(query)
-
-        for table in parser.source_tables:
-            source_schema, source_table = str(table).split(".")
-            if source_schema == "<default>":
-                source_schema = str(self.config.default_schema)
-
-            source = LineageDataset(
-                platform=LineageDatasetPlatform.REDSHIFT,
-                path=f"{db_name}.{source_schema}.{source_table}",
-            )
-            sources.append(source)
-
-        return sources
-
-    def get_db_name(self, inspector: Optional[Inspector] = None) -> str:
-        db_name = self.config.database
-        db_alias = self.config.database_alias
-        if db_alias:
-            db_name = db_alias
-        assert db_name
-        return db_name
-
-    def _get_s3_path(self, path: str) -> str:
-        if self.config.s3_lineage_config:
-            for path_spec in self.config.s3_lineage_config.path_specs:
-                if path_spec.allowed(path):
-                    table_name, table_path = path_spec.extract_table_name_and_path(path)
-                    return table_path
-        return path
-
-    def _build_s3_path_from_row(self, db_row):
-        path = db_row["filename"].strip()
-        if urlparse(path).scheme != "s3":
-            raise ValueError(
-                f"Only s3 source supported with copy/unload. The source was: {path}"
-            )
-        return strip_s3_prefix(self._get_s3_path(path))
-
-    def _populate_lineage_map(
-        self, query: str, lineage_type: LineageCollectorType
-    ) -> None:
-        """
-        This method generate table level lineage based with the given query.
-        The query should return the following columns: target_schema, target_table, source_table, source_schema
-        source_table and source_schema can be omitted if the sql_field is set because then it assumes the source_table
-        and source_schema will be extracted from the sql_field by sql parsing.
-
-        :param query: The query to run to extract lineage.
-        :type query: str
-        :param lineage_type: The way the lineage should be processed
-        :type lineage_type: LineageType
-        return: The method does not return with anything as it directly modify the self._lineage_map property.
-        :rtype: None
-        """
-        assert self._lineage_map is not None
-
-        if not self._all_tables_set:
-            self._all_tables_set = self._get_all_tables()
-
-        engine = self.get_metadata_engine()
-
-        db_name = self.get_db_name()
-
-        try:
-            for db_row in engine.execute(query):
-                if lineage_type != LineageCollectorType.UNLOAD:
-                    if not self.config.schema_pattern.allowed(
-                        db_row["target_schema"]
-                    ) or not self.config.table_pattern.allowed(db_row["target_table"]):
-                        continue
-
-                # Target
-                if lineage_type == LineageCollectorType.UNLOAD:
-                    try:
-                        target_platform = LineageDatasetPlatform.S3
-                        # Following call requires 'filename' key in db_row
-                        target_path = self._build_s3_path_from_row(db_row)
-                    except ValueError as e:
-                        self.warn(logger, "non-s3-lineage", str(e))
-                        continue
-                else:
-                    target_platform = LineageDatasetPlatform.REDSHIFT
-                    target_path = (
-                        f'{db_name}.{db_row["target_schema"]}.{db_row["target_table"]}'
-                    )
-
-                target = LineageItem(
-                    dataset=LineageDataset(platform=target_platform, path=target_path),
-                    upstreams=set(),
-                    collector_type=lineage_type,
-                    query_parser_failed_sqls=list(),
-                )
-
-                # Source
-                sources: List[LineageDataset] = list()
-                if lineage_type in {
-                    lineage_type.QUERY_SQL_PARSER,
-                    lineage_type.NON_BINDING_VIEW,
-                }:
-                    try:
-                        sources = self._get_sources_from_query(
-                            db_name=db_name, query=db_row["ddl"]
-                        )
-                    except Exception as e:
-                        target.query_parser_failed_sqls.append(db_row["ddl"])
-                        self.warn(
-                            logger,
-                            "parsing-query",
-                            f'Error parsing query {db_row["ddl"]} for getting lineage .'
-                            f"\nError was {e}.",
-                        )
-                else:
-                    if lineage_type == lineage_type.COPY:
-                        try:
-                            platform = LineageDatasetPlatform.S3
-                            # Following call requires 'filename' key in db_row
-                            path = self._build_s3_path_from_row(db_row)
-                        except ValueError as e:
-                            self.warn(logger, "non-s3-lineage", str(e))
-                            continue
-                    else:
-                        platform = LineageDatasetPlatform.REDSHIFT
-                        path = f'{db_name}.{db_row["source_schema"]}.{db_row["source_table"]}'
-
-                    sources = [
-                        LineageDataset(
-                            platform=platform,
-                            path=path,
-                        )
-                    ]
-
-                for source in sources:
-                    # Filtering out tables which does not exist in Redshift
-                    # It was deleted in the meantime or query parser did not capture well the table name
-                    if (
-                        source.platform == LineageDatasetPlatform.REDSHIFT
-                        and source.path not in self._all_tables_set
-                    ):
-                        self.warn(
-                            logger, "missing-table", f"{source.path} missing table"
-                        )
-                        continue
-
-                    target.upstreams.add(source)
-
-                # Merging downstreams if dataset already exists and has downstreams
-                if target.dataset.path in self._lineage_map:
-                    self._lineage_map[
-                        target.dataset.path
-                    ].upstreams = self._lineage_map[
-                        target.dataset.path
-                    ].upstreams.union(
-                        target.upstreams
-                    )
-
-                else:
-                    self._lineage_map[target.dataset.path] = target
-
-                logger.info(
-                    f"Lineage[{target}]:{self._lineage_map[target.dataset.path]}"
-                )
-
-        except Exception as e:
-            self.warn(logger, f"extract-{lineage_type.name}", f"Error was {e}")
-
-    def _populate_lineage(self) -> None:
-        stl_scan_based_lineage_query: str = """
-            select
-                distinct cluster,
-                target_schema,
-                target_table,
-                username as username,
-                source_schema,
-                source_table
-            from
-                    (
-                select
-                    distinct tbl as target_table_id,
-                    sti.schema as target_schema,
-                    sti.table as target_table,
-                    sti.database as cluster,
-                    query,
-                    starttime
-                from
-                    stl_insert
-                join SVV_TABLE_INFO sti on
-                    sti.table_id = tbl
-                where starttime >= '{start_time}'
-                and starttime < '{end_time}'
-                and cluster = '{db_name}'
-                    ) as target_tables
-            join ( (
-                select
-                    sui.usename as username,
-                    ss.tbl as source_table_id,
-                    sti.schema as source_schema,
-                    sti.table as source_table,
-                    scan_type,
-                    sq.query as query
-                from
-                    (
-                    select
-                        distinct userid,
-                        query,
-                        tbl,
-                        type as scan_type
-                    from
-                        stl_scan
-                ) ss
-                join SVV_TABLE_INFO sti on
-                    sti.table_id = ss.tbl
-                left join stl_query sq on
-                    ss.query = sq.query
-                left join svl_user_info sui on
-                    sq.userid = sui.usesysid
-                where
-                    sui.usename <> 'rdsdb')
-            ) as source_tables
-                    using (query)
-            where
-                scan_type in (1, 2, 3)
-            order by cluster, target_schema, target_table, starttime asc
-        """.format(
-            # We need the original database name for filtering
-            db_name=self.config.database,
-            start_time=self.config.start_time.strftime(redshift_datetime_format),
-            end_time=self.config.end_time.strftime(redshift_datetime_format),
-        )
-        view_lineage_query = """
-            select
-                distinct
-                srcnsp.nspname as source_schema
-                ,
-                srcobj.relname as source_table
-                ,
-                tgtnsp.nspname as target_schema
-                ,
-                tgtobj.relname as target_table
-            from
-                pg_catalog.pg_class as srcobj
-            inner join
-                pg_catalog.pg_depend as srcdep
-                    on
-                srcobj.oid = srcdep.refobjid
-            inner join
-                pg_catalog.pg_depend as tgtdep
-                    on
-                srcdep.objid = tgtdep.objid
-            join
-                pg_catalog.pg_class as tgtobj
-                    on
-                tgtdep.refobjid = tgtobj.oid
-                and srcobj.oid <> tgtobj.oid
-            left outer join
-                pg_catalog.pg_namespace as srcnsp
-                    on
-                srcobj.relnamespace = srcnsp.oid
-            left outer join
-                pg_catalog.pg_namespace tgtnsp
-                    on
-                tgtobj.relnamespace = tgtnsp.oid
-            where
-                tgtdep.deptype = 'i'
-                --dependency_internal
-                and tgtobj.relkind = 'v'
-                --i=index, v=view, s=sequence
-                and tgtnsp.nspname not in ('pg_catalog', 'information_schema')
-                order by target_schema, target_table asc
-        """
-
-        list_late_binding_views_query = """
-        SELECT
-            n.nspname AS target_schema
-            ,c.relname AS target_table
-            , COALESCE(pg_get_viewdef(c.oid, TRUE), '') AS ddl
-        FROM
-            pg_catalog.pg_class AS c
-        INNER JOIN
-            pg_catalog.pg_namespace AS n
-            ON c.relnamespace = n.oid
-        WHERE relkind = 'v'
-        and ddl ilike '%%with no schema binding%%'
-        and
-        n.nspname not in ('pg_catalog', 'information_schema')
-        """
-
-        list_insert_create_queries_sql = """
-        select
-            distinct cluster,
-            target_schema,
-            target_table,
-            username,
-            querytxt as ddl
-        from
-                (
-            select
-                distinct tbl as target_table_id,
-                sti.schema as target_schema,
-                sti.table as target_table,
-                sti.database as cluster,
-                sui.usename as username,
-                querytxt,
-                si.starttime as starttime
-            from
-                stl_insert as si
-            join SVV_TABLE_INFO sti on
-                sti.table_id = tbl
-            left join svl_user_info sui on
-                si.userid = sui.usesysid
-            left join stl_query sq on
-                si.query = sq.query
-            left join stl_load_commits slc on
-                slc.query = si.query
-            where
-                sui.usename <> 'rdsdb'
-                and sq.aborted = 0
-                and slc.query IS NULL
-                and cluster = '{db_name}'
-                and si.starttime >= '{start_time}'
-                and si.starttime < '{end_time}'
-            ) as target_tables
-            order by cluster, target_schema, target_table, starttime asc
-        """.format(
-            # We need the original database name for filtering
-            db_name=self.config.database,
-            start_time=self.config.start_time.strftime(redshift_datetime_format),
-            end_time=self.config.end_time.strftime(redshift_datetime_format),
-        )
-
-        list_copy_commands_sql = """
-        select
-            distinct
-                "schema" as target_schema,
-                "table" as target_table,
-                filename
-        from
-            stl_insert as si
-        join stl_load_commits as c on
-            si.query = c.query
-        join SVV_TABLE_INFO sti on
-            sti.table_id = tbl
-        where
-            database = '{db_name}'
-            and si.starttime >= '{start_time}'
-            and si.starttime < '{end_time}'
-        order by target_schema, target_table, starttime asc
-        """.format(
-            # We need the original database name for filtering
-            db_name=self.config.database,
-            start_time=self.config.start_time.strftime(redshift_datetime_format),
-            end_time=self.config.end_time.strftime(redshift_datetime_format),
-        )
-
-        list_unload_commands_sql = """
-        select
-            distinct
-                sti.database as cluster,
-                sti.schema as source_schema,
-                sti."table" as source_table,
-                unl.path as filename
-        from
-            stl_unload_log unl
-        join stl_scan sc on
-            sc.query = unl.query and
-            sc.starttime >= '{start_time}' and
-            sc.endtime < '{end_time}'
-        join SVV_TABLE_INFO sti on
-            sti.table_id = sc.tbl
-        where
-            unl.start_time >= '{start_time}' and
-            unl.end_time < '{end_time}' and
-            sti.database = '{db_name}'
-          and sc.type in (1, 2, 3)
-        order by cluster, source_schema, source_table, filename, unl.start_time asc
-        """.format(
-            # We need the original database name for filtering
-            db_name=self.config.database,
-            start_time=self.config.start_time.strftime(redshift_datetime_format),
-            end_time=self.config.end_time.strftime(redshift_datetime_format),
-        )
-
-        if not self._lineage_map:
-            self._lineage_map = defaultdict()
-
-        if self.config.table_lineage_mode == LineageMode.STL_SCAN_BASED:
-            # Populate table level lineage by getting upstream tables from stl_scan redshift table
-            self._populate_lineage_map(
-                query=stl_scan_based_lineage_query,
-                lineage_type=LineageCollectorType.QUERY_SCAN,
-            )
-        elif self.config.table_lineage_mode == LineageMode.SQL_BASED:
-            # Populate table level lineage by parsing table creating sqls
-            self._populate_lineage_map(
-                query=list_insert_create_queries_sql,
-                lineage_type=LineageCollectorType.QUERY_SQL_PARSER,
-            )
-        elif self.config.table_lineage_mode == LineageMode.MIXED:
-            # Populate table level lineage by parsing table creating sqls
-            self._populate_lineage_map(
-                query=list_insert_create_queries_sql,
-                lineage_type=LineageCollectorType.QUERY_SQL_PARSER,
-            )
-            # Populate table level lineage by getting upstream tables from stl_scan redshift table
-            self._populate_lineage_map(
-                query=stl_scan_based_lineage_query,
-                lineage_type=LineageCollectorType.QUERY_SCAN,
-            )
-
-        if self.config.include_views:
-            # Populate table level lineage for views
-            self._populate_lineage_map(
-                query=view_lineage_query, lineage_type=LineageCollectorType.VIEW
-            )
-
-            # Populate table level lineage for late binding views
-            self._populate_lineage_map(
-                query=list_late_binding_views_query,
-                lineage_type=LineageCollectorType.NON_BINDING_VIEW,
-            )
-        if self.config.include_copy_lineage:
-            self._populate_lineage_map(
-                query=list_copy_commands_sql, lineage_type=LineageCollectorType.COPY
-            )
-        if self.config.include_unload_lineage:
-            self._populate_lineage_map(
-                query=list_unload_commands_sql, lineage_type=LineageCollectorType.UNLOAD
-            )
-
-    def get_lineage_mcp(
-        self, dataset_urn: str
-    ) -> Tuple[
-        Optional[MetadataChangeProposalWrapper], Optional[DatasetPropertiesClass]
-    ]:
-        dataset_key = mce_builder.dataset_urn_to_key(dataset_urn)
-        if dataset_key is None:
-            return None, None
-
-        if self._lineage_map is None:
-            logger.debug("Populating lineage")
-            self._populate_lineage()
-        assert self._lineage_map is not None
-
-        upstream_lineage: List[UpstreamClass] = []
-        custom_properties: Dict[str, str] = {}
-
-        if dataset_key.name in self._lineage_map:
-            item = self._lineage_map[dataset_key.name]
-            if (
-                self.config.capture_lineage_query_parser_failures
-                and item.query_parser_failed_sqls
-            ):
-                custom_properties["lineage_sql_parser_failed_queries"] = ",".join(
-                    item.query_parser_failed_sqls
-                )
-            for upstream in item.upstreams:
-                upstream_table = UpstreamClass(
-                    dataset=builder.make_dataset_urn_with_platform_instance(
-                        upstream.platform.value,
-                        upstream.path,
-                        platform_instance=self.config.platform_instance_map.get(
-                            upstream.platform.value
-                        )
-                        if self.config.platform_instance_map
-                        else None,
-                        env=self.config.env,
-                    ),
-                    type=item.dataset_lineage_type,
-                )
-                upstream_lineage.append(upstream_table)
-
-        dataset_params = dataset_key.name.split(".")
-        db_name = dataset_params[0]
-        schemaname = dataset_params[1]
-        tablename = dataset_params[2]
-        if db_name in self.catalog_metadata:
-            if schemaname in self.catalog_metadata[db_name]:
-                external_db_params = self.catalog_metadata[db_name][schemaname]
-                upstream_platform = self.eskind_to_platform[
-                    external_db_params["eskind"]
-                ]
-                catalog_upstream = UpstreamClass(
-                    mce_builder.make_dataset_urn_with_platform_instance(
-                        upstream_platform,
-                        "{database}.{table}".format(
-                            database=external_db_params["external_database"],
-                            table=tablename,
-                        ),
-                        platform_instance=self.config.platform_instance_map.get(
-                            upstream_platform
-                        )
-                        if self.config.platform_instance_map
-                        else None,
-                        env=self.config.env,
-                    ),
-                    DatasetLineageTypeClass.COPY,
-                )
-                upstream_lineage.append(catalog_upstream)
-
-        properties = None
-        if custom_properties:
-            properties = DatasetPropertiesClass(customProperties=custom_properties)
-
-        if upstream_lineage:
-            self.report.upstream_lineage[dataset_urn] = [
-                u.dataset for u in upstream_lineage
-            ]
-        else:
-            return None, properties
-
-        mcp = MetadataChangeProposalWrapper(
-            entityType="dataset",
-            changeType=ChangeTypeClass.UPSERT,
-            entityUrn=dataset_urn,
-            aspectName="upstreamLineage",
-            aspect=UpstreamLineage(upstreams=upstream_lineage),
-        )
-
-        return mcp, properties
diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py
deleted file mode 100644
index 691eaa8211054..0000000000000
--- a/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py
+++ /dev/null
@@ -1,397 +0,0 @@
-import collections
-import dataclasses
-import logging
-import time
-from datetime import datetime
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set
-
-from pydantic.fields import Field
-from pydantic.main import BaseModel
-from sqlalchemy import create_engine
-from sqlalchemy.engine import Engine
-
-import datahub.emitter.mce_builder as builder
-from datahub.configuration.source_common import EnvConfigMixin
-from datahub.configuration.time_window_config import get_time_bucket
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.api.decorators import (
-    SourceCapability,
-    SupportStatus,
-    capability,
-    config_class,
-    platform_name,
-    support_status,
-)
-from datahub.ingestion.api.source import Source, SourceReport
-from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.sql.redshift import RedshiftConfig
-from datahub.ingestion.source.usage.usage_common import (
-    BaseUsageConfig,
-    GenericAggregatedDataset,
-)
-from datahub.metadata.schema_classes import OperationClass, OperationTypeClass
-
-logger = logging.getLogger(__name__)
-
-if TYPE_CHECKING:
-    try:
-        from sqlalchemy.engine import Row  # type: ignore
-    except ImportError:
-        # See https://github.com/python/mypy/issues/1153.
-        from sqlalchemy.engine.result import RowProxy as Row  # type: ignore
-
-REDSHIFT_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
-
-
-# Add this join to the sql query for more metrics on completed queries
-# LEFT JOIN svl_query_metrics_summary sqms ON ss.query = sqms.query
-# Reference: https://docs.aws.amazon.com/redshift/latest/dg/r_SVL_QUERY_METRICS_SUMMARY.html
-
-# this sql query joins stl_scan over table info,
-# querytext, and user info to get usage stats
-# using non-LEFT joins here to limit the results to
-# queries run by the user on user-defined tables.
-REDSHIFT_USAGE_QUERY_TEMPLATE: str = """
-SELECT DISTINCT ss.userid as userid,
-       ss.query as query,
-       sui.usename as username,
-       ss.tbl as tbl,
-       sq.querytxt as querytxt,
-       sti.database as database,
-       sti.schema as schema,
-       sti.table as table,
-       sq.starttime as starttime,
-       sq.endtime as endtime
-FROM stl_scan ss
-  JOIN svv_table_info sti ON ss.tbl = sti.table_id
-  JOIN stl_query sq ON ss.query = sq.query
-  JOIN svl_user_info sui ON sq.userid = sui.usesysid
-WHERE ss.starttime >= '{start_time}'
-AND ss.starttime < '{end_time}'
-AND sti.database = '{database}'
-AND sq.aborted = 0
-ORDER BY ss.endtime DESC;
-""".strip()
-
-REDSHIFT_OPERATION_ASPECT_QUERY_TEMPLATE: str = """
-  (SELECT
-      DISTINCT si.userid AS userid,
-      si.query AS query,
-      si.rows AS rows,
-      sui.usename AS username,
-      si.tbl AS tbl,
-      sq.querytxt AS querytxt,
-      sti.database AS database,
-      sti.schema AS schema,
-      sti.table AS table,
-      sq.starttime AS starttime,
-      sq.endtime AS endtime,
-      'insert' AS operation_type
-    FROM
-      stl_insert si
-      JOIN svv_table_info sti ON si.tbl = sti.table_id
-      JOIN stl_query sq ON si.query = sq.query
-      JOIN svl_user_info sui ON sq.userid = sui.usesysid
-    WHERE
-      si.starttime >= '{start_time}'
-      AND si.starttime < '{end_time}'
-      AND si.rows > 0
-      AND sq.aborted = 0)
-UNION
-  (SELECT
-      DISTINCT sd.userid AS userid,
-      sd.query AS query,
-      sd.rows AS ROWS,
-      sui.usename AS username,
-      sd.tbl AS tbl,
-      sq.querytxt AS querytxt,
-      sti.database AS database,
-      sti.schema AS schema,
-      sti.table AS table,
-      sq.starttime AS starttime,
-      sq.endtime AS endtime,
-      'delete' AS operation_type
-    FROM
-      stl_delete sd
-      JOIN svv_table_info sti ON sd.tbl = sti.table_id
-      JOIN stl_query sq ON sd.query = sq.query
-      JOIN svl_user_info sui ON sq.userid = sui.usesysid
-    WHERE
-      sd.starttime >= '{start_time}'
-      AND sd.starttime < '{end_time}'
-      AND sd.rows > 0
-      AND sq.aborted = 0)
-ORDER BY
-  endtime DESC
-""".strip()
-
-RedshiftTableRef = str
-AggregatedDataset = GenericAggregatedDataset[RedshiftTableRef]
-AggregatedAccessEvents = Dict[datetime, Dict[RedshiftTableRef, AggregatedDataset]]
-
-
-class RedshiftAccessEvent(BaseModel):
-    userid: int
-    username: str
-    query: int
-    tbl: int
-    text: Optional[str] = Field(None, alias="querytxt")
-    database: str
-    schema_: str = Field(alias="schema")
-    table: str
-    operation_type: Optional[str] = None
-    starttime: datetime
-    endtime: datetime
-
-
-class RedshiftUsageConfig(RedshiftConfig, BaseUsageConfig, EnvConfigMixin):
-    email_domain: str = Field(
-        description="Email domain of your organisation so users can be displayed on UI appropriately."
-    )
-    options: Dict = Field(
-        default={},
-        description="Any options specified here will be passed to SQLAlchemy's create_engine as kwargs."
-        "See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine for details.",
-    )
-
-    def get_sql_alchemy_url(self):
-        return super().get_sql_alchemy_url()
-
-
-@dataclasses.dataclass
-class RedshiftUsageSourceReport(SourceReport):
-    filtered: Set[str] = dataclasses.field(default_factory=set)
-    num_usage_workunits_emitted: Optional[int] = None
-    num_operational_stats_workunits_emitted: Optional[int] = None
-
-    def report_dropped(self, key: str) -> None:
-        self.filtered.add(key)
-
-
-@platform_name("Redshift")
-@config_class(RedshiftUsageConfig)
-@support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
-class RedshiftUsageSource(Source):
-    """
-    This plugin extracts usage statistics for datasets in Amazon Redshift.
-
-    Note: Usage information is computed by querying the following system tables -
-    1. stl_scan
-    2. svv_table_info
-    3. stl_query
-    4. svl_user_info
-
-    To grant access this plugin for all system tables, please alter your datahub Redshift user the following way:
-    ```sql
-    ALTER USER datahub_user WITH SYSLOG ACCESS UNRESTRICTED;
-    ```
-    This plugin has the below functionalities -
-    1. For a specific dataset this plugin ingests the following statistics -
-       1. top n queries.
-       2. top users.
-    2. Aggregation of these statistics into buckets, by day or hour granularity.
-
-    :::note
-
-    This source only does usage statistics. To get the tables, views, and schemas in your Redshift warehouse, ingest using the `redshift` source described above.
-
-    :::
-
-    :::note
-
-    Redshift system tables have some latency in getting data from queries. In addition, these tables only maintain logs for 2-5 days. You can find more information from the official documentation [here](https://aws.amazon.com/premiumsupport/knowledge-center/logs-redshift-database-cluster/).
-
-    :::
-
-    """
-
-    def __init__(self, config: RedshiftUsageConfig, ctx: PipelineContext):
-        super().__init__(ctx)
-        self.config: RedshiftUsageConfig = config
-        self.report: RedshiftUsageSourceReport = RedshiftUsageSourceReport()
-
-    @classmethod
-    def create(cls, config_dict: Dict, ctx: PipelineContext) -> "RedshiftUsageSource":
-        config = RedshiftUsageConfig.parse_obj(config_dict)
-        return cls(config, ctx)
-
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        """Gets Redshift usage stats as work units"""
-        engine: Engine = self._make_sql_engine()
-        if self.config.include_operational_stats:
-            # Generate operation aspect workunits
-            yield from self._gen_operation_aspect_workunits(engine)
-
-        # Generate aggregate events
-        query: str = REDSHIFT_USAGE_QUERY_TEMPLATE.format(
-            start_time=self.config.start_time.strftime(REDSHIFT_DATETIME_FORMAT),
-            end_time=self.config.end_time.strftime(REDSHIFT_DATETIME_FORMAT),
-            database=self.config.database,
-        )
-        access_events_iterable: Iterable[
-            RedshiftAccessEvent
-        ] = self._gen_access_events_from_history_query(query, engine)
-
-        aggregated_events: AggregatedAccessEvents = self._aggregate_access_events(
-            access_events_iterable
-        )
-        # Generate usage workunits from aggregated events.
-        self.report.num_usage_workunits_emitted = 0
-        for time_bucket in aggregated_events.values():
-            for aggregate in time_bucket.values():
-                yield self._make_usage_stat(aggregate)
-                self.report.num_usage_workunits_emitted += 1
-
-    def _gen_operation_aspect_workunits(
-        self, engine: Engine
-    ) -> Iterable[MetadataWorkUnit]:
-        # Generate access events
-        query: str = REDSHIFT_OPERATION_ASPECT_QUERY_TEMPLATE.format(
-            start_time=self.config.start_time.strftime(REDSHIFT_DATETIME_FORMAT),
-            end_time=self.config.end_time.strftime(REDSHIFT_DATETIME_FORMAT),
-        )
-        access_events_iterable: Iterable[
-            RedshiftAccessEvent
-        ] = self._gen_access_events_from_history_query(query, engine)
-
-        # Generate operation aspect work units from the access events
-        yield from self._gen_operation_aspect_workunits_from_access_events(
-            access_events_iterable
-        )
-
-    def _make_sql_engine(self) -> Engine:
-        url: str = self.config.get_sql_alchemy_url()
-        logger.debug(f"sql_alchemy_url = {url}")
-        return create_engine(url, **self.config.options)
-
-    def _should_process_row(self, row: "Row") -> bool:
-        # Check for mandatory proerties being present first.
-        missing_props: List[str] = [
-            prop
-            for prop in ["database", "schema", "table", "username"]
-            if not row[prop]
-        ]
-        if missing_props:
-            logging.info(
-                f"Access event parameter(s):[{','.join(missing_props)}] missing. Skipping ...."
-            )
-            return False
-        # Check schema/table allow/deny patterns
-        full_table_name: str = f"{row['database']}.{row['schema']}.{row['table']}"
-        if not self.config.schema_pattern.allowed(row["schema"]):
-            logger.debug(f"Filtering out {full_table_name} due to schema_pattern.")
-            self.report.report_dropped(full_table_name)
-            return False
-        if not self.config.table_pattern.allowed(full_table_name):
-            logger.debug(f"Filtering out {full_table_name} due to table_pattern.")
-            self.report.report_dropped(full_table_name)
-            return False
-        # Passed all checks.
-        return True
-
-    def _gen_access_events_from_history_query(
-        self, query: str, engine: Engine
-    ) -> Iterable[RedshiftAccessEvent]:
-        results = engine.execute(query)
-        for row in results:
-            if not self._should_process_row(row):
-                continue
-            row = row._asdict()
-            access_event = RedshiftAccessEvent(**dict(row.items()))
-            # Replace database name with the alias name if one is provided in the config.
-            if self.config.database_alias:
-                access_event.database = self.config.database_alias
-            yield access_event
-
-    def _gen_operation_aspect_workunits_from_access_events(
-        self,
-        events_iterable: Iterable[RedshiftAccessEvent],
-    ) -> Iterable[MetadataWorkUnit]:
-        self.report.num_operational_stats_workunits_emitted = 0
-        for event in events_iterable:
-            if not (
-                event.database
-                and event.username
-                and event.schema_
-                and event.table
-                and event.endtime
-                and event.operation_type
-            ):
-                continue
-
-            assert event.operation_type in ["insert", "delete"]
-
-            resource: str = f"{event.database}.{event.schema_}.{event.table}"
-            reported_time: int = int(time.time() * 1000)
-            last_updated_timestamp: int = int(event.endtime.timestamp() * 1000)
-            user_email: str = event.username
-            operation_aspect = OperationClass(
-                timestampMillis=reported_time,
-                lastUpdatedTimestamp=last_updated_timestamp,
-                actor=builder.make_user_urn(user_email.split("@")[0]),
-                operationType=(
-                    OperationTypeClass.INSERT
-                    if event.operation_type == "insert"
-                    else OperationTypeClass.DELETE
-                ),
-            )
-            yield MetadataChangeProposalWrapper(
-                entityUrn=builder.make_dataset_urn_with_platform_instance(
-                    "redshift",
-                    resource.lower(),
-                    self.config.platform_instance,
-                    self.config.env,
-                ),
-                aspect=operation_aspect,
-            ).as_workunit()
-            self.report.num_operational_stats_workunits_emitted += 1
-
-    def _aggregate_access_events(
-        self, events_iterable: Iterable[RedshiftAccessEvent]
-    ) -> AggregatedAccessEvents:
-        datasets: AggregatedAccessEvents = collections.defaultdict(dict)
-        for event in events_iterable:
-            floored_ts: datetime = get_time_bucket(
-                event.starttime, self.config.bucket_duration
-            )
-            resource: str = f"{event.database}.{event.schema_}.{event.table}"
-            # Get a reference to the bucket value(or initialize not yet in dict) and update it.
-            agg_bucket: AggregatedDataset = datasets[floored_ts].setdefault(
-                resource,
-                AggregatedDataset(
-                    bucket_start_time=floored_ts,
-                    resource=resource,
-                ),
-            )
-            # current limitation in user stats UI, we need to provide email to show users
-            user_email: str = f"{event.username if event.username else 'unknown'}"
-            if "@" not in user_email:
-                user_email += f"@{self.config.email_domain}"
-            logger.info(f"user_email: {user_email}")
-            agg_bucket.add_read_entry(
-                user_email,
-                event.text,
-                [],  # TODO: not currently supported by redshift; find column level changes
-                user_email_pattern=self.config.user_email_pattern,
-            )
-        return datasets
-
-    def _make_usage_stat(self, agg: AggregatedDataset) -> MetadataWorkUnit:
-        return agg.make_usage_workunit(
-            self.config.bucket_duration,
-            lambda resource: builder.make_dataset_urn_with_platform_instance(
-                "redshift",
-                resource.lower(),
-                self.config.platform_instance,
-                self.config.env,
-            ),
-            self.config.top_n_queries,
-            self.config.format_sql_queries,
-            self.config.include_top_n_queries,
-            self.config.queries_character_limit,
-        )
-
-    def get_report(self) -> RedshiftUsageSourceReport:
-        return self.report

From 417ffb12d8093a809ef11e9362786b91b6a9f1f6 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Thu, 16 Nov 2023 13:34:18 -0500
Subject: [PATCH 222/259] feat(ingest): support advanced configs for aws
 (#9237)

---
 .../ingestion/source/aws/aws_common.py        | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
index d61975694f541..0fb211a5d7b16 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
@@ -1,8 +1,8 @@
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import boto3
 from boto3.session import Session
-from botocore.config import Config
+from botocore.config import DEFAULT_TIMEOUT, Config
 from botocore.utils import fix_s3_host
 from pydantic.fields import Field
 
@@ -104,6 +104,16 @@ class AwsConnectionConfig(ConfigModel):
         description="A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.",
     )
 
+    read_timeout: float = Field(
+        default=DEFAULT_TIMEOUT,
+        description="The timeout for reading from the connection (in seconds).",
+    )
+
+    aws_advanced_config: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).",
+    )
+
     def _normalized_aws_roles(self) -> List[AwsAssumeRoleConfig]:
         if not self.aws_role:
             return []
@@ -167,13 +177,20 @@ def get_credentials(self) -> Dict[str, str]:
             }
         return {}
 
+    def _aws_config(self) -> Config:
+        return Config(
+            proxies=self.aws_proxy,
+            read_timeout=self.read_timeout,
+            **self.aws_advanced_config,
+        )
+
     def get_s3_client(
         self, verify_ssl: Optional[Union[bool, str]] = None
     ) -> "S3Client":
         return self.get_session().client(
             "s3",
             endpoint_url=self.aws_endpoint_url,
-            config=Config(proxies=self.aws_proxy),
+            config=self._aws_config(),
             verify=verify_ssl,
         )
 
@@ -183,7 +200,7 @@ def get_s3_resource(
         resource = self.get_session().resource(
             "s3",
             endpoint_url=self.aws_endpoint_url,
-            config=Config(proxies=self.aws_proxy),
+            config=self._aws_config(),
             verify=verify_ssl,
         )
         # according to: https://stackoverflow.com/questions/32618216/override-s3-endpoint-using-boto3-configuration-file
@@ -195,10 +212,10 @@ def get_s3_resource(
         return resource
 
     def get_glue_client(self) -> "GlueClient":
-        return self.get_session().client("glue")
+        return self.get_session().client("glue", config=self._aws_config())
 
     def get_sagemaker_client(self) -> "SageMakerClient":
-        return self.get_session().client("sagemaker")
+        return self.get_session().client("sagemaker", config=self._aws_config())
 
 
 class AwsSourceConfig(EnvConfigMixin, AwsConnectionConfig):

From d0fa5de9f051a77c729f19f2e2df74f8581ccefd Mon Sep 17 00:00:00 2001
From: Adriano Vega Llobell <Starkie@users.noreply.github.com>
Date: Thu, 16 Nov 2023 19:58:49 +0100
Subject: [PATCH 223/259] fix(sql-parser): convert platform instance to
 lowercase when building table urns (#9181)

---
 .../src/datahub/utilities/sqlglot_lineage.py  |  6 +++-
 .../unit/sql_parsing/test_schemaresolver.py   | 33 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py

diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
index d1209f3ec7b75..efe2d26aae3d9 100644
--- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
+++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
@@ -360,8 +360,12 @@ def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
         table_name = ".".join(
             filter(None, [table.database, table.db_schema, table.table])
         )
+
+        platform_instance = self.platform_instance
+
         if lower:
             table_name = table_name.lower()
+            platform_instance = platform_instance.lower() if platform_instance else None
 
         if self.platform == "bigquery":
             # Normalize shard numbers and other BigQuery weirdness.
@@ -372,7 +376,7 @@ def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
 
         urn = make_dataset_urn_with_platform_instance(
             platform=self.platform,
-            platform_instance=self.platform_instance,
+            platform_instance=platform_instance,
             env=self.env,
             name=table_name,
         )
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py b/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
new file mode 100644
index 0000000000000..3fd5d72b4d41a
--- /dev/null
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
@@ -0,0 +1,33 @@
+from datahub.utilities.sqlglot_lineage import SchemaResolver, _TableName
+
+
+def test_get_urn_for_table_lowercase():
+    schema_resolver = SchemaResolver(
+        platform="mssql",
+        platform_instance="Uppercased-Instance",
+        env="PROD",
+        graph=None,
+    )
+
+    table = _TableName(database="Database", db_schema="DataSet", table="Table")
+
+    assert (
+        schema_resolver.get_urn_for_table(table=table, lower=True)
+        == "urn:li:dataset:(urn:li:dataPlatform:mssql,uppercased-instance.database.dataset.table,PROD)"
+    )
+
+
+def test_get_urn_for_table_not_lower_should_keep_capital_letters():
+    schema_resolver = SchemaResolver(
+        platform="mssql",
+        platform_instance="Uppercased-Instance",
+        env="PROD",
+        graph=None,
+    )
+
+    table = _TableName(database="Database", db_schema="DataSet", table="Table")
+
+    assert (
+        schema_resolver.get_urn_for_table(table=table, lower=False)
+        == "urn:li:dataset:(urn:li:dataPlatform:mssql,Uppercased-Instance.Database.DataSet.Table,PROD)"
+    )

From 5cb04981b26a7a5045fe43ee074c2a99f30c015f Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Thu, 16 Nov 2023 17:18:17 -0500
Subject: [PATCH 224/259] test(ingest/unity): Update goldens (#9254)

---
 .../unity/unity_catalog_mces_golden.json      | 499 ++++++++++++------
 1 file changed, 337 insertions(+), 162 deletions(-)

diff --git a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json
index 0c14096345d7e..2e92215d70b99 100644
--- a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json
+++ b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json
@@ -8,6 +8,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore"
             },
             "externalUrl": "https://dummy.cloud.databricks.com/explore/data",
@@ -16,7 +17,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -31,7 +33,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -46,7 +49,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -63,7 +67,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -87,7 +92,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -102,7 +108,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,6 +121,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "main"
             },
@@ -124,7 +132,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -139,7 +148,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -154,7 +164,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -171,7 +182,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -195,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +223,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -230,7 +244,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -242,6 +257,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "main",
                 "unity_schema": "default"
@@ -253,7 +269,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -268,7 +285,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -283,7 +301,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -300,7 +319,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -324,7 +344,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -339,7 +360,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -363,7 +385,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -378,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -420,7 +444,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -437,7 +462,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -494,7 +520,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -516,7 +543,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -540,7 +568,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -568,7 +597,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -580,6 +610,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "main",
                 "unity_schema": "information_schema"
@@ -591,7 +622,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -606,7 +638,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -621,7 +654,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -638,7 +672,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -662,7 +697,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -677,7 +713,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -701,7 +738,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -716,7 +754,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -758,7 +797,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -775,7 +815,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -832,7 +873,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -854,7 +896,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -878,7 +921,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -906,7 +950,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -918,6 +963,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "main",
                 "unity_schema": "quickstart_schema"
@@ -929,7 +975,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -944,7 +991,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -959,7 +1007,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -976,7 +1025,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1000,7 +1050,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1015,7 +1066,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1039,7 +1091,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1054,7 +1107,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1096,7 +1150,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1113,7 +1168,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1170,7 +1226,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1192,7 +1249,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1216,7 +1274,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1244,7 +1303,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1256,6 +1316,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "quickstart_catalog"
             },
@@ -1266,7 +1327,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1281,7 +1343,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1296,7 +1359,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1313,7 +1377,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1337,7 +1402,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1352,7 +1418,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1372,7 +1439,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1384,6 +1452,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "quickstart_catalog",
                 "unity_schema": "default"
@@ -1395,7 +1464,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1410,7 +1480,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1425,7 +1496,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1442,7 +1514,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1466,7 +1539,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1481,7 +1555,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1505,7 +1580,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1520,7 +1596,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1562,7 +1639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1579,7 +1657,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1636,7 +1715,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1658,7 +1738,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1682,7 +1763,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1710,7 +1792,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1722,6 +1805,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "quickstart_catalog",
                 "unity_schema": "information_schema"
@@ -1733,7 +1817,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1748,7 +1833,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1763,7 +1849,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1780,7 +1867,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1804,7 +1892,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1819,7 +1908,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1843,7 +1933,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1858,7 +1949,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1900,7 +1992,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1917,7 +2010,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1974,7 +2068,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -1996,7 +2091,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2020,7 +2116,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2048,7 +2145,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2060,6 +2158,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "quickstart_catalog",
                 "unity_schema": "quickstart_schema"
@@ -2071,7 +2170,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2086,7 +2186,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2101,7 +2202,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2118,7 +2220,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2142,7 +2245,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2157,7 +2261,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2181,7 +2286,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2196,7 +2302,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2238,7 +2345,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2255,7 +2363,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2312,7 +2421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2334,7 +2444,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2358,7 +2469,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2386,7 +2498,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2398,6 +2511,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "system"
             },
@@ -2408,7 +2522,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2423,7 +2538,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2438,7 +2554,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2455,7 +2572,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2479,7 +2597,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2494,7 +2613,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2514,7 +2634,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2526,6 +2647,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "system",
                 "unity_schema": "default"
@@ -2537,7 +2659,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2552,7 +2675,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2567,7 +2691,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2584,7 +2709,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2608,7 +2734,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2623,7 +2750,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2647,7 +2775,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2662,7 +2791,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2704,7 +2834,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2721,7 +2852,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2778,7 +2910,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2800,7 +2933,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2824,7 +2958,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2852,7 +2987,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2864,6 +3000,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "system",
                 "unity_schema": "information_schema"
@@ -2875,7 +3012,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2890,7 +3028,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2905,7 +3044,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2922,7 +3062,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2946,7 +3087,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2961,7 +3103,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -2985,7 +3128,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3000,7 +3144,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3042,7 +3187,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3059,7 +3205,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3116,7 +3263,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3138,7 +3286,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3162,7 +3311,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3190,7 +3340,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3202,6 +3353,7 @@
         "json": {
             "customProperties": {
                 "platform": "databricks",
+                "env": "PROD",
                 "metastore": "acryl metastore",
                 "catalog": "system",
                 "unity_schema": "quickstart_schema"
@@ -3213,7 +3365,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3228,7 +3381,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3243,7 +3397,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3260,7 +3415,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3284,7 +3440,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3299,7 +3456,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3323,7 +3481,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3338,7 +3497,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3380,7 +3540,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3397,7 +3558,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3454,7 +3616,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3476,7 +3639,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3500,7 +3664,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3528,7 +3693,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3543,7 +3709,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3558,7 +3725,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3573,7 +3741,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3588,7 +3757,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3603,7 +3773,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3618,7 +3789,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3633,7 +3805,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3648,7 +3821,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -3663,7 +3837,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1638860400000,
-        "runId": "unity-catalog-test"
+        "runId": "unity-catalog-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file

From b778def10aa53cf7852206078834a2cd4499c181 Mon Sep 17 00:00:00 2001
From: Andrew Sikowitz <andrew.sikowitz@acryl.io>
Date: Thu, 16 Nov 2023 17:19:25 -0500
Subject: [PATCH 225/259] build(ingest/hive): Update thrift pin (#8964)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 metadata-ingestion/setup.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 04ae03cd440ac..2b002164a49b9 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -213,11 +213,14 @@
     # - 0.6.12 adds support for Spark Thrift Server
     # - 0.6.13 adds a small fix for Databricks
     # - 0.6.14 uses pure-sasl instead of sasl so it builds on Python 3.11
-    "acryl-pyhive[hive_pure_sasl]==0.6.14",
+    # - 0.6.15 adds support for thrift > 0.14 (cherry-picked from https://github.com/apache/thrift/pull/2491)
+    "acryl-pyhive[hive_pure_sasl]==0.6.15",
     # As per https://github.com/datahub-project/datahub/issues/8405
-    # and https://github.com/dropbox/PyHive/issues/417, new versions
-    # of thrift break PyHive's hive+http transport.
-    "thrift<0.14.0",
+    # and https://github.com/dropbox/PyHive/issues/417, version 0.14.0
+    # of thrift broke PyHive's hive+http transport.
+    # Fixed by https://github.com/apache/thrift/pull/2491 in version 0.17.0
+    # which is unfortunately not on PyPi.
+    # Instead, we put the fix in our PyHive fork, so no thrift pin is needed.
 }
 
 microsoft_common = {"msal==1.22.0"}

From c348f841c6151f85bcc1f299e95cc91a8415c750 Mon Sep 17 00:00:00 2001
From: "Jia (Jason) Teoh" <jiateoh@users.noreply.github.com>
Date: Thu, 16 Nov 2023 16:05:56 -0800
Subject: [PATCH 226/259] docs(airflow): update plugin setup docs to include UI
 setup approach (#9253)

---
 docs/lineage/airflow.md | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md
index 3a13aefa834a4..32da518d6c04c 100644
--- a/docs/lineage/airflow.md
+++ b/docs/lineage/airflow.md
@@ -37,12 +37,24 @@ pip install 'acryl-datahub-airflow-plugin[plugin-v2]'
 
 ### Configuration
 
-Set up a DataHub connection in Airflow.
+Set up a DataHub connection in Airflow, either via command line or the Airflow UI.
+
+#### Command Line
 
 ```shell
 airflow connections add  --conn-type 'datahub-rest' 'datahub_rest_default' --conn-host 'http://datahub-gms:8080' --conn-password '<optional datahub auth token>'
 ```
 
+#### Airflow UI
+
+On the Airflow UI, go to Admin -> Connections and click the "+" symbol to create a new connection. Select "DataHub REST Server" from the dropdown for "Connection Type" and enter the appropriate values.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/airflow/plugin_connection_setup.png"/>
+</p>
+
+#### Optional Configurations
+
 No additional configuration is required to use the plugin. However, there are some optional configuration parameters that can be set in the `airflow.cfg` file.
 
 ```ini title="airflow.cfg"

From b03515fbc7ececfadbbee8f93c19c577298bffe4 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Thu, 16 Nov 2023 18:39:36 -0600
Subject: [PATCH 227/259] feat(usageclient): updates for usageclient (#9255)

---
 build.gradle                                  |   1 +
 .../src/main/resources/application.yml        |   3 +-
 .../gms/factory/usage/UsageClientFactory.java |  14 +-
 metadata-service/restli-client/build.gradle   |   1 +
 .../linkedin/common/client/ClientCache.java   |   8 +-
 .../entity/client/EntityClientCache.java      |   6 +-
 .../java/com/linkedin/usage/UsageClient.java  |  11 +-
 .../com/linkedin/usage/UsageClientCache.java  |   4 +-
 .../resources/usage/UsageStats.java.latest    | 316 ------------------
 .../restli/DefaultRestliClientFactory.java    |  18 +-
 10 files changed, 47 insertions(+), 335 deletions(-)
 delete mode 100644 metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java.latest

diff --git a/build.gradle b/build.gradle
index 9eecb1696bb19..7c5deb4783943 100644
--- a/build.gradle
+++ b/build.gradle
@@ -82,6 +82,7 @@ project.ext.externalDependency = [
     'commonsLang': 'commons-lang:commons-lang:2.6',
     'commonsText': 'org.apache.commons:commons-text:1.10.0',
     'commonsCollections': 'commons-collections:commons-collections:3.2.2',
+    'caffeine': 'com.github.ben-manes.caffeine:caffeine:3.1.8',
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
     'datastaxOssCore': 'com.datastax.oss:java-driver-core:4.14.1',
     'datastaxOssQueryBuilder': 'com.datastax.oss:java-driver-query-builder:4.14.1',
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 40674e13e647f..571cb66c84aa8 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -332,7 +332,8 @@ entityClient:
 
 usageClient:
   retryInterval: ${USAGE_CLIENT_RETRY_INTERVAL:2}
-  numRetries: ${USAGE_CLIENT_NUM_RETRIES:3}
+  numRetries: ${USAGE_CLIENT_NUM_RETRIES:0}
+  timeoutMs: ${USAGE_CLIENT_TIMEOUT_MS:3000}
 
 cache:
   primary:
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java
index e83cbc82d8067..d2bd89de8767a 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java
@@ -5,6 +5,7 @@
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import com.linkedin.metadata.restli.DefaultRestliClientFactory;
 import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
+import com.linkedin.r2.transport.http.client.HttpClientFactory;
 import com.linkedin.restli.client.Client;
 import com.linkedin.usage.UsageClient;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,6 +15,9 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
 
+import java.util.HashMap;
+import java.util.Map;
+
 
 @Configuration
 @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class)
@@ -34,16 +38,22 @@ public class UsageClientFactory {
   @Value("${usageClient.retryInterval:2}")
   private int retryInterval;
 
-  @Value("${usageClient.numRetries:3}")
+  @Value("${usageClient.numRetries:0}")
   private int numRetries;
 
+  @Value("${usageClient.timeoutMs:3000}")
+  private long timeoutMs;
+
   @Autowired
   @Qualifier("configurationProvider")
   private ConfigurationProvider configurationProvider;
 
   @Bean("usageClient")
   public UsageClient getUsageClient(@Qualifier("systemAuthentication") final Authentication systemAuthentication) {
-    Client restClient = DefaultRestliClientFactory.getRestLiClient(gmsHost, gmsPort, gmsUseSSL, gmsSslProtocol);
+    Map<String, String> params = new HashMap<>();
+    params.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT, String.valueOf(timeoutMs));
+
+    Client restClient = DefaultRestliClientFactory.getRestLiClient(gmsHost, gmsPort, gmsUseSSL, gmsSslProtocol, params);
     return new UsageClient(restClient, new ExponentialBackoff(retryInterval), numRetries, systemAuthentication,
             configurationProvider.getCache().getClient().getUsageClient());
   }
diff --git a/metadata-service/restli-client/build.gradle b/metadata-service/restli-client/build.gradle
index b1b778b45c0b5..7cad1981ad911 100644
--- a/metadata-service/restli-client/build.gradle
+++ b/metadata-service/restli-client/build.gradle
@@ -9,6 +9,7 @@ dependencies {
   api project(':metadata-utils')
   implementation project(':metadata-service:configuration')
 
+  implementation externalDependency.caffeine
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/common/client/ClientCache.java b/metadata-service/restli-client/src/main/java/com/linkedin/common/client/ClientCache.java
index 8aa0984be57b9..79d473d1b0090 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/common/client/ClientCache.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/common/client/ClientCache.java
@@ -14,8 +14,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
-import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
@@ -63,15 +63,15 @@ private ClientCache<K, V, C> build() {
 
         public ClientCache<K, V, C> build(Class<?> metricClazz) {
             // loads data from entity client
-            CacheLoader<K, V> loader = new CacheLoader<>() {
+            CacheLoader<K, V> loader = new CacheLoader<K, V>() {
                 @Override
                 public V load(@NonNull K key) {
-                    return loadAll(List.of(key)).get(key);
+                    return loadAll(Set.of(key)).get(key);
                 }
 
                 @Override
                 @NonNull
-                public Map<K, V> loadAll(@NonNull Iterable<? extends K> keys) {
+                public Map<K, V> loadAll(@NonNull Set<? extends K> keys) {
                     return loadFunction.apply(keys);
                 }
             };
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java
index 6006f3a9a87f6..8e103cff283ea 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java
@@ -81,16 +81,14 @@ private EntityClientCacheBuilder cache(LoadingCache<Key, EnvelopedAspect> cache)
         public EntityClientCache build(Class<?> metricClazz) {
             // estimate size
             Weigher<Key, EnvelopedAspect> weighByEstimatedSize = (key, value) ->
-                    value.getValue().data().values().parallelStream()
-                            .mapToInt(o -> o.toString().getBytes().length)
-                            .sum();
+                    value.getValue().data().toString().getBytes().length;
 
             // batch loads data from entity client (restli or java)
             Function<Iterable<? extends Key>, Map<Key, EnvelopedAspect>> loader = (Iterable<? extends Key> keys) -> {
                 Map<String, Set<Key>> keysByEntity = StreamSupport.stream(keys.spliterator(), true)
                         .collect(Collectors.groupingBy(Key::getEntityName, Collectors.toSet()));
 
-                Map<Key, EnvelopedAspect> results = keysByEntity.entrySet().parallelStream()
+                Map<Key, EnvelopedAspect> results = keysByEntity.entrySet().stream()
                         .flatMap(entry -> {
                             Set<Urn> urns = entry.getValue().stream()
                                     .map(Key::getUrn)
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClient.java
index d2b8499615e8d..850847bfd262a 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClient.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClient.java
@@ -9,6 +9,7 @@
 import com.linkedin.parseq.retry.backoff.BackoffPolicy;
 import com.linkedin.r2.RemoteInvocationException;
 import com.linkedin.restli.client.Client;
+
 import java.net.URISyntaxException;
 import javax.annotation.Nonnull;
 
@@ -51,10 +52,12 @@ public UsageQueryResult getUsageStats(@Nonnull String resource, @Nonnull UsageTi
     private UsageQueryResult getUsageStats(@Nonnull String resource, @Nonnull UsageTimeRange range,
                                            @Nonnull Authentication authentication)
             throws RemoteInvocationException, URISyntaxException {
-        final UsageStatsDoQueryRangeRequestBuilder requestBuilder = USAGE_STATS_REQUEST_BUILDERS.actionQueryRange()
-            .resourceParam(resource)
-            .durationParam(WindowDuration.DAY)
-            .rangeFromEndParam(range);
+
+        final UsageStatsDoQueryRangeRequestBuilder requestBuilder = USAGE_STATS_REQUEST_BUILDERS
+                .actionQueryRange()
+                .resourceParam(resource)
+                .durationParam(WindowDuration.DAY)
+                .rangeFromEndParam(range);
         return sendClientRequest(requestBuilder, authentication).getEntity();
     }
 }
diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClientCache.java b/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClientCache.java
index a04c1e90fb4a3..10a1ebb6dcccb 100644
--- a/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClientCache.java
+++ b/metadata-service/restli-client/src/main/java/com/linkedin/usage/UsageClientCache.java
@@ -42,9 +42,7 @@ private UsageClientCacheBuilder cache(LoadingCache<Key, UsageQueryResult> cache)
         public UsageClientCache build() {
             // estimate size
             Weigher<Key, UsageQueryResult> weighByEstimatedSize = (key, value) ->
-                    value.data().values().parallelStream()
-                            .mapToInt(o -> o.toString().getBytes().length)
-                            .sum();
+                    value.data().toString().getBytes().length;
 
             // batch loads data from usage client
             Function<Iterable<? extends Key>, Map<Key, UsageQueryResult>> loader = (Iterable<? extends Key> keys) ->
diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java.latest b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java.latest
deleted file mode 100644
index 91f74c12e6aad..0000000000000
--- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java.latest
+++ /dev/null
@@ -1,316 +0,0 @@
-package com.linkedin.metadata.resources.usage;
-
-import com.linkedin.common.WindowDuration;
-import com.linkedin.common.urn.Urn;
-import com.linkedin.data.template.SetMode;
-import com.linkedin.data.template.StringArray;
-import com.linkedin.data.template.StringArrayArray;
-import com.linkedin.metadata.query.Condition;
-import com.linkedin.metadata.query.Criterion;
-import com.linkedin.metadata.query.CriterionArray;
-import com.linkedin.metadata.query.Filter;
-import com.linkedin.metadata.timeseries.elastic.ElasticSearchTimeseriesAspectService;
-import com.linkedin.metadata.usage.UsageService;
-import com.linkedin.timeseries.AggregationSpec;
-import com.linkedin.timeseries.AggregationType;
-import com.linkedin.timeseries.CalendarInterval;
-import com.linkedin.timeseries.DateGroupingBucket;
-import com.linkedin.timeseries.GenericTable;
-import com.linkedin.metadata.restli.RestliUtils;
-import com.linkedin.parseq.Task;
-import com.linkedin.restli.server.annotations.Action;
-import com.linkedin.restli.server.annotations.ActionParam;
-import com.linkedin.restli.server.annotations.RestLiSimpleResource;
-import com.linkedin.restli.server.resources.SimpleResourceTemplate;
-import com.linkedin.timeseries.GroupingBucket;
-import com.linkedin.timeseries.MetricAggregation;
-import com.linkedin.timeseries.StringGroupingBucket;
-import com.linkedin.usage.FieldUsageCounts;
-import com.linkedin.usage.FieldUsageCountsArray;
-import com.linkedin.usage.UsageAggregation;
-import com.linkedin.usage.UsageAggregationArray;
-import com.linkedin.usage.UsageAggregationMetrics;
-import com.linkedin.usage.UsageQueryResult;
-import com.linkedin.usage.UsageQueryResultAggregations;
-import com.linkedin.usage.UsageTimeRange;
-import com.linkedin.usage.UserUsageCounts;
-import com.linkedin.usage.UserUsageCountsArray;
-import com.linkedin.util.Pair;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nonnull;
-import javax.inject.Inject;
-import javax.inject.Named;
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-
-/**
- * Rest.li entry point: /usageStats
- */
-@RestLiSimpleResource(name = "usageStats", namespace = "com.linkedin.usage")
-public class UsageStats extends SimpleResourceTemplate<UsageAggregation> {
-  private static final String ACTION_BATCH_INGEST = "batchIngest";
-  private static final String PARAM_BUCKETS = "buckets";
-
-  private static final String ACTION_QUERY = "query";
-  private static final String PARAM_RESOURCE = "resource";
-  private static final String PARAM_DURATION = "duration";
-  private static final String PARAM_START_TIME = "startTime";
-  private static final String PARAM_END_TIME = "endTime";
-  private static final String PARAM_MAX_BUCKETS = "maxBuckets";
-
-  private static final String ACTION_QUERY_RANGE = "queryRange";
-  private static final String PARAM_RANGE = "rangeFromEnd";
-  private static final String USAGE_STATS_ENTITY_NAME = "dataset";
-  private static final String USAGE_STATS_ASPECT_NAME = "datasetUsageStatistics";
-  private static final String ES_FIELD_TIMESTAMP = "timestampMillis";
-  private final Logger _logger = LoggerFactory.getLogger(UsageStats.class.getName());
-  @Inject
-  @Named("usageService")
-  private UsageService _usageService;
-  @Inject
-  @Named("elasticSearchTimeseriesAspectService")
-  private ElasticSearchTimeseriesAspectService _elasticSearchTimeseriesAspectService;
-
-  @Action(name = ACTION_BATCH_INGEST)
-  @Nonnull
-  public Task<Void> batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregation[] buckets) {
-    _logger.info("Ingesting {} usage stats aggregations", buckets.length);
-    return RestliUtils.toTask(() -> {
-      for (UsageAggregation agg : buckets) {
-        this.ingest(agg);
-      }
-      return null;
-    });
-  }
-
-  private CalendarInterval windowToInterval(@Nonnull WindowDuration duration) {
-    switch (duration) {
-      case HOUR:
-        return CalendarInterval.HOUR;
-      case DAY:
-        return CalendarInterval.DAY;
-      case WEEK:
-        return CalendarInterval.WEEK;
-      case MONTH:
-        return CalendarInterval.MONTH;
-      case YEAR:
-        return CalendarInterval.YEAR;
-      default:
-        throw new IllegalArgumentException("Unsupported duration value" + duration);
-    }
-  }
-
-  private UsageAggregationArray getBuckets(@Nonnull String resource, @Nonnull WindowDuration duration, Long startTime, Long endTime) {
-    // Populate the filter
-    Filter filter = new Filter();
-    ArrayList<Criterion> criteria = new ArrayList<>();
-    Criterion hasUrnCriterion = new Criterion().setField("urn").setCondition(Condition.EQUAL).setValue(resource);
-    criteria.add(hasUrnCriterion);
-    if (startTime != null) {
-      Criterion startTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP)
-          .setCondition(Condition.GREATER_THAN_OR_EQUAL_TO)
-          .setValue(startTime.toString());
-      criteria.add(startTimeCriterion);
-    }
-    if (endTime != null) {
-      Criterion endTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP)
-          .setCondition(Condition.LESS_THAN_OR_EQUAL_TO)
-          .setValue(endTime.toString());
-      criteria.add(endTimeCriterion);
-    }
-    filter.setCriteria(new CriterionArray(criteria));
-    // Populate the aggregation specs
-    ArrayList<AggregationSpec> aggregationSpecs = new ArrayList<>();
-    aggregationSpecs.add(new AggregationSpec().setAggregationType(AggregationType.LATEST).setMemberName("uniqueUserCount"));
-    aggregationSpecs.add(new AggregationSpec().setAggregationType(AggregationType.LATEST).setMemberName("totalSqlQueries"));
-    aggregationSpecs.add(new AggregationSpec().setAggregationType(AggregationType.LATEST).setMemberName("topSqlQueries"));
-      /*
-      aggregationSpecs.add(new AggregationSpec().setAggregationType(AggregationType.SUM).setMemberName("totalSqlQueries"));
-      aggregationSpecs.add(new AggregationSpec().setAggregationType(AggregationType.SUM).setMemberName("userCounts.count"));
-      aggregationSpecs.add(new AggregationSpec().setAggregationType(AggregationType.SUM).setMemberName("fieldCounts.count"));
-       */
-
-    // Populate the Grouping buckets
-    ArrayList<GroupingBucket> groupingBuckets = new ArrayList<>();
-    // ts bucket
-    GroupingBucket timestampBucket = new GroupingBucket();
-    timestampBucket.setDateGroupingBucket(
-        new DateGroupingBucket().setKey(ES_FIELD_TIMESTAMP).setGranularity(windowToInterval(duration)));
-    groupingBuckets.add(timestampBucket);
-      /*
-      // user counts bucket
-      GroupingBucket userGroupsBucket = new GroupingBucket();
-      userGroupsBucket.setStringGroupingBucket( new StringGroupingBucket().setKey("userCounts.user") );
-      groupingBuckets.add(userGroupsBucket);
-      // field counts bucket
-      GroupingBucket fieldCountGroupBucket = new GroupingBucket();
-      fieldCountGroupBucket.setStringGroupingBucket(new StringGroupingBucket().setKey("fieldCounts.fieldName"));
-      groupingBuckets.add(fieldCountGroupBucket);
-      */
-
-    GenericTable result =
-        _elasticSearchTimeseriesAspectService.getAggregatedStats(USAGE_STATS_ENTITY_NAME, USAGE_STATS_ASPECT_NAME,
-            (AggregationSpec[]) aggregationSpecs.toArray(), filter, (GroupingBucket[]) groupingBuckets.toArray());
-    UsageAggregationArray buckets = new UsageAggregationArray();
-    for(StringArray row: result.getRows()) {
-      UsageAggregation usageAggregation = new UsageAggregation();
-      usageAggregation.setBucket(Long.valueOf(row.get(0)));
-      usageAggregation.setDuration(duration);
-      try {
-        usageAggregation.setResource(new Urn(resource));
-      } catch (URISyntaxException e) {
-        throw new IllegalArgumentException("Invalid resource" + e);
-      }
-      UsageAggregationMetrics usageAggregationMetrics = new UsageAggregationMetrics();
-      usageAggregationMetrics.setUniqueUserCount(Integer.valueOf(row.get(1)));
-      usageAggregationMetrics.setTotalSqlQueries(Integer.valueOf(row.get(2)));
-      //usageAggregationMetrics.setTopSqlQueries(row.get(3));
-      usageAggregation.setMetrics(usageAggregationMetrics);
-    }
-    return buckets;
-  }
-
-  private UsageQueryResultAggregations getAggregations(String resource, WindowDuration duration, Long startTime, Long endTime) {
-    // TODO: make the aggregation computation logic reusable
-    UsageQueryResultAggregations aggregations = new UsageQueryResultAggregations();
-
-    /*
-    // Compute aggregations for users and unique user count.
-    {
-      Map<Pair<Urn, String>, Integer> userAgg = new HashMap<>();
-      buckets.forEach((bucket) -> {
-        Optional.ofNullable(bucket.getMetrics().getUsers()).ifPresent(usersUsageCounts -> {
-          usersUsageCounts.forEach((userCount -> {
-            Pair<Urn, String> key = new Pair<>(userCount.getUser(), userCount.getUserEmail());
-            int count = userAgg.getOrDefault(key, 0);
-            count += userCount.getCount();
-            userAgg.put(key, count);
-          }));
-        });
-      });
-
-      if (!userAgg.isEmpty()) {
-        UserUsageCountsArray users = new UserUsageCountsArray();
-        users.addAll(userAgg.entrySet()
-            .stream()
-            .map((mapping) -> new UserUsageCounts().setUser(mapping.getKey().getFirst(), SetMode.REMOVE_IF_NULL)
-                .setUserEmail(mapping.getKey().getSecond(), SetMode.REMOVE_IF_NULL)
-                .setCount(mapping.getValue()))
-            .collect(Collectors.toList()));
-        aggregations.setUsers(users);
-        aggregations.setUniqueUserCount(userAgg.size());
-      }
-    }
-
-    // Compute aggregation for total query count.
-    {
-      Integer totalQueryCount = null;
-
-      for (UsageAggregation bucket : buckets) {
-        if (bucket.getMetrics().getTotalSqlQueries() != null) {
-          if (totalQueryCount == null) {
-            totalQueryCount = 0;
-          }
-          totalQueryCount += bucket.getMetrics().getTotalSqlQueries();
-        }
-      }
-
-      if (totalQueryCount != null) {
-        aggregations.setTotalSqlQueries(totalQueryCount);
-      }
-    }
-
-    // Compute aggregations for field usage counts.
-    {
-      Map<String, Integer> fieldAgg = new HashMap<>();
-      buckets.forEach((bucket) -> {
-        Optional.ofNullable(bucket.getMetrics().getFields()).ifPresent(fieldUsageCounts -> {
-          fieldUsageCounts.forEach((fieldCount -> {
-            String key = fieldCount.getFieldName();
-            int count = fieldAgg.getOrDefault(key, 0);
-            count += fieldCount.getCount();
-            fieldAgg.put(key, count);
-          }));
-        });
-      });
-
-      if (!fieldAgg.isEmpty()) {
-        FieldUsageCountsArray fields = new FieldUsageCountsArray();
-        fields.addAll(fieldAgg.entrySet()
-            .stream()
-            .map((mapping) -> new FieldUsageCounts().setFieldName(mapping.getKey()).setCount(mapping.getValue()))
-            .collect(Collectors.toList()));
-        aggregations.setFields(fields);
-      }
-    }
-     */
-    return aggregations;
-  }
-
-  @Action(name = ACTION_QUERY)
-  @Nonnull
-  public Task<UsageQueryResult> query(@ActionParam(PARAM_RESOURCE) @Nonnull String resource,
-      @ActionParam(PARAM_DURATION) @Nonnull WindowDuration duration,
-      @ActionParam(PARAM_START_TIME) @com.linkedin.restli.server.annotations.Optional Long startTime,
-      @ActionParam(PARAM_END_TIME) @com.linkedin.restli.server.annotations.Optional Long endTime,
-      @ActionParam(PARAM_MAX_BUCKETS) @com.linkedin.restli.server.annotations.Optional Integer maxBuckets) {
-    _logger.info("Attempting to query usage stats");
-    return RestliUtils.toTask(() -> {
-      UsageAggregationArray buckets =  getBuckets(resource, duration, startTime, endTime);
-      UsageQueryResultAggregations aggregations = getAggregations(resource, duration, startTime, endTime);
-      return new UsageQueryResult().setBuckets(buckets).setAggregations(aggregations);
-    });
-  }
-
-
-  @Action(name = ACTION_QUERY_RANGE)
-  @Nonnull
-  public Task<UsageQueryResult> queryRange(@ActionParam(PARAM_RESOURCE) @Nonnull String resource,
-      @ActionParam(PARAM_DURATION) @Nonnull WindowDuration duration, @ActionParam(PARAM_RANGE) UsageTimeRange range) {
-    final long now = Instant.now().toEpochMilli();
-    return this.query(resource, duration, convertRangeToStartTime(range, now), now, null);
-  }
-
-  private void ingest(@Nonnull UsageAggregation bucket) {
-    // TODO attempt to resolve users into emails
-    _usageService.upsertDocument(bucket);
-  }
-
-  @Nonnull
-  Long convertRangeToStartTime(@Nonnull UsageTimeRange range, long currentEpochMillis) {
-    // TRICKY: since start_time must be before the bucket's start, we actually
-    // need to subtract extra from the current time to ensure that we get precisely
-    // what we're looking for. Note that start_time and end_time are both inclusive,
-    // so we must also do an off-by-one adjustment.
-    final long oneHourMillis = 60 * 60 * 1000;
-    final long oneDayMillis = 24 * oneHourMillis;
-
-    if (range == UsageTimeRange.HOUR) {
-      return currentEpochMillis - (2 * oneHourMillis + 1);
-    } else if (range == UsageTimeRange.DAY) {
-      return currentEpochMillis - (2 * oneDayMillis + 1);
-    } else if (range == UsageTimeRange.WEEK) {
-      return currentEpochMillis - (8 * oneDayMillis + 1);
-    } else if (range == UsageTimeRange.MONTH) {
-      // Assuming month is last 30 days.
-      return currentEpochMillis - (31 * oneDayMillis + 1);
-    } else if (range == UsageTimeRange.QUARTER) {
-      // Assuming a quarter is 91 days.
-      return currentEpochMillis - (92 * oneDayMillis + 1);
-    } else if (range == UsageTimeRange.YEAR) {
-      return currentEpochMillis - (366 * oneDayMillis + 1);
-    } else if (range == UsageTimeRange.ALL) {
-      return 0L;
-    } else {
-      throw new IllegalArgumentException("invalid UsageTimeRange enum state: " + range.name());
-    }
-  }
-}
diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/restli/DefaultRestliClientFactory.java b/metadata-utils/src/main/java/com/linkedin/metadata/restli/DefaultRestliClientFactory.java
index 436c7ae5d77b5..2d4e355a93e53 100644
--- a/metadata-utils/src/main/java/com/linkedin/metadata/restli/DefaultRestliClientFactory.java
+++ b/metadata-utils/src/main/java/com/linkedin/metadata/restli/DefaultRestliClientFactory.java
@@ -44,18 +44,34 @@ public static RestClient getRestLiD2Client(@Nonnull String restLiClientD2ZkHost,
   @Nonnull
   public static RestClient getRestLiClient(@Nonnull String restLiServerHost, int restLiServerPort, boolean useSSL,
                                            @Nullable String sslProtocol) {
+    return getRestLiClient(restLiServerHost, restLiServerPort, useSSL, sslProtocol, null);
+  }
+
+  @Nonnull
+  public static RestClient getRestLiClient(@Nonnull String restLiServerHost, int restLiServerPort, boolean useSSL,
+                                           @Nullable String sslProtocol, @Nullable Map<String, String> params) {
     return getRestLiClient(
             URI.create(String.format("%s://%s:%s", useSSL ? "https" : "http", restLiServerHost, restLiServerPort)),
-            sslProtocol);
+            sslProtocol,
+            params);
   }
 
   @Nonnull
   public static RestClient getRestLiClient(@Nonnull URI gmsUri, @Nullable String sslProtocol) {
+    return getRestLiClient(gmsUri, sslProtocol, null);
+  }
+
+  @Nonnull
+  public static RestClient getRestLiClient(@Nonnull URI gmsUri, @Nullable String sslProtocol,
+                                           @Nullable Map<String, String> inputParams) {
     if (StringUtils.isBlank(gmsUri.getHost()) || gmsUri.getPort() <= 0) {
       throw new InvalidParameterException("Invalid restli server host name or port!");
     }
 
     Map<String, Object> params = new HashMap<>();
+    if (inputParams != null) {
+      params.putAll(inputParams);
+    }
 
     if ("https".equals(gmsUri.getScheme())) {
       try {

From bc89a1c48ef8b60e9b2a3bf07968ebc20a1a5caf Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 17 Nov 2023 10:06:26 -0600
Subject: [PATCH 228/259] fix(graphql): prevent duplicate index queries for
 dataproducts (#9260)

---
 .../resolvers/dataproduct/ListDataProductAssetsResolver.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java
index e727ebe185838..831d449bef9ef 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java
@@ -79,11 +79,11 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
     }
 
     // 2. Get list of entities that we should query based on filters or assets from aspect.
-    List<String> entitiesToQuery = assetUrns.stream().map(Urn::getEntityType).collect(Collectors.toList());
+    List<String> entitiesToQuery = assetUrns.stream().map(Urn::getEntityType).distinct().collect(Collectors.toList());
 
 
     final List<EntityType> inputEntityTypes = (input.getTypes() == null || input.getTypes().isEmpty()) ? ImmutableList.of() : input.getTypes();
-    final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList());
+    final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).distinct().collect(Collectors.toList());
 
     final List<String> finalEntityNames = inputEntityNames.size() > 0 ? inputEntityNames : entitiesToQuery;
 

From 2b0811b9875d7d7ea11fb01d0157a21fdd67f020 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 17 Nov 2023 10:50:13 -0600
Subject: [PATCH 229/259] logging(search): log level highlight value urn
 detection (#9262)

---
 .../com/linkedin/datahub/graphql/types/mappers/MapperUtils.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
index 5ba32b0c2a77c..2a615b24eaac2 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java
@@ -70,7 +70,7 @@ public static List<MatchedField> getMatchedFieldEntry(List<com.linkedin.metadata
                   Urn urn = Urn.createFromString(field.getValue());
                   matchedField.setEntity(UrnToEntityMapper.map(urn));
               } catch (URISyntaxException e) {
-                  log.warn("Failed to create urn from MatchedField value: {}", field.getValue(), e);
+                  log.debug("Failed to create urn from MatchedField value: {}", field.getValue());
               }
           }
           return matchedField;

From d2ca2dd4d422abdc276025f7e69ef1c73bd71efe Mon Sep 17 00:00:00 2001
From: Kevin Chun <kevin1chun@gmail.com>
Date: Fri, 17 Nov 2023 14:57:30 -0600
Subject: [PATCH 230/259] docs(development): Add Python version in Developer
 README (#9268)

---
 .gitignore         | 1 +
 docs/developers.md | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 49ab5c475096c..1fcca8751131f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,6 +32,7 @@ venv.bak/
 dmypy.json
 MANIFEST
 *.pyc
+.python-version
 
 # Generated files
 **/bin
diff --git a/docs/developers.md b/docs/developers.md
index c5aaa9e28ca87..52fd7d356a44c 100644
--- a/docs/developers.md
+++ b/docs/developers.md
@@ -6,6 +6,7 @@ title: "Local Development"
 
 ## Pre-requirements
  - [Java 11 SDK](https://openjdk.org/projects/jdk/11/)
+ - [Python 3.10] (https://www.python.org/downloads/release/python-3100/)
  - [Docker](https://www.docker.com/)
  - [Docker Compose](https://docs.docker.com/compose/)
  - Docker engine with at least 8GB of memory to run tests.

From b7df9e09968b71ff6315184e618d43ecfd251308 Mon Sep 17 00:00:00 2001
From: noggi <anton.kuraev@acryl.io>
Date: Fri, 17 Nov 2023 16:53:19 -0800
Subject: [PATCH 231/259] Sync datahub-head on merge (#9267)

---
 .github/scripts/docker_helpers.sh    |  6 +++---
 .github/workflows/docker-unified.yml | 24 ++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh
index f238c5c409184..334465532db06 100755
--- a/.github/scripts/docker_helpers.sh
+++ b/.github/scripts/docker_helpers.sh
@@ -12,15 +12,15 @@ export SHORT_SHA=$(get_short_sha)
 echo "SHORT_SHA: $SHORT_SHA"
 
 function get_tag {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim\,${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full\,${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index 18cb946b951dd..35cd5363293f8 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -46,6 +46,9 @@ jobs:
       unique_full_tag: ${{ steps.tag.outputs.unique_full_tag }}
       publish: ${{ steps.publish.outputs.publish }}
       python_release_version: ${{ steps.tag.outputs.python_release_version }}
+      short_sha: ${{ steps.tag.outputs.short_sha }}
+      branch_name: ${{ steps.tag.outputs.branch_name }}
+      repository_name: ${{ steps.tag.outputs.repository_name }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -53,6 +56,7 @@ jobs:
         id: tag
         run: |
           source .github/scripts/docker_helpers.sh
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
           echo "tag=$(get_tag)" >> $GITHUB_OUTPUT
           echo "slim_tag=$(get_tag_slim)" >> $GITHUB_OUTPUT
           echo "full_tag=$(get_tag_full)" >> $GITHUB_OUTPUT
@@ -60,6 +64,8 @@ jobs:
           echo "unique_slim_tag=$(get_unique_tag_slim)" >> $GITHUB_OUTPUT
           echo "unique_full_tag=$(get_unique_tag_full)" >> $GITHUB_OUTPUT
           echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
+          echo "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+          echo "repository_name=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
@@ -860,3 +866,21 @@ jobs:
           job-status: ${{ job.status }}
           slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel: github-activities
+  deploy_datahub_head:
+    name: Deploy to Datahub HEAD
+    runs-on: ubuntu-latest
+    needs:
+      [
+        setup,
+        smoke_test
+      ]
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }}
+          aws-region: us-west-2
+      - uses: isbang/sqs-action@v0.2.0
+        with:
+          sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }}
+          message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}'

From a704290be419280618a6acc7c0cc440bc083c667 Mon Sep 17 00:00:00 2001
From: Salman-Apptware <101426513+Salman-Apptware@users.noreply.github.com>
Date: Mon, 20 Nov 2023 10:06:21 +0530
Subject: [PATCH 232/259] PRD-742/fix:Settings tab should have 2 scrollable
 sections (#9218)

---
 .../src/app/entity/ownership/ManageOwnership.tsx      |  9 ++++++++-
 datahub-web-react/src/app/entity/view/ManageViews.tsx |  9 ++++++++-
 .../src/app/identity/ManageIdentities.tsx             |  9 ++++++++-
 .../src/app/identity/group/GroupList.tsx              |  9 ++++++++-
 datahub-web-react/src/app/identity/user/UserList.tsx  |  9 ++++++++-
 .../src/app/permissions/ManagePermissions.tsx         |  7 ++++++-
 .../src/app/permissions/policy/ManagePolicies.tsx     |  9 ++++++++-
 .../src/app/permissions/roles/ManageRoles.tsx         |  9 ++++++++-
 datahub-web-react/src/app/settings/AccessTokens.tsx   |  3 +++
 datahub-web-react/src/app/settings/SettingsPage.tsx   |  9 +++++++--
 .../src/app/settings/posts/ManagePosts.tsx            | 10 ++++++++--
 .../src/app/settings/posts/PostsList.tsx              |  6 +++++-
 datahub-web-react/src/app/shared/RoutedTabs.tsx       | 11 +++++++++--
 13 files changed, 94 insertions(+), 15 deletions(-)

diff --git a/datahub-web-react/src/app/entity/ownership/ManageOwnership.tsx b/datahub-web-react/src/app/entity/ownership/ManageOwnership.tsx
index cff10a3d96b30..a3304ab015faa 100644
--- a/datahub-web-react/src/app/entity/ownership/ManageOwnership.tsx
+++ b/datahub-web-react/src/app/entity/ownership/ManageOwnership.tsx
@@ -6,6 +6,9 @@ import { OwnershipList } from './OwnershipList';
 const PageContainer = styled.div`
     padding-top: 20px;
     width: 100%;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const PageHeaderContainer = styled.div`
@@ -20,7 +23,11 @@ const PageTitle = styled(Typography.Title)`
     }
 `;
 
-const ListContainer = styled.div``;
+const ListContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
 
 /**
  * Component used for displaying the 'Manage Ownership' experience.
diff --git a/datahub-web-react/src/app/entity/view/ManageViews.tsx b/datahub-web-react/src/app/entity/view/ManageViews.tsx
index b31d3869ab158..f32c479e00bea 100644
--- a/datahub-web-react/src/app/entity/view/ManageViews.tsx
+++ b/datahub-web-react/src/app/entity/view/ManageViews.tsx
@@ -6,6 +6,9 @@ import { ViewsList } from './ViewsList';
 const PageContainer = styled.div`
     padding-top: 20px;
     width: 100%;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const PageHeaderContainer = styled.div`
@@ -20,7 +23,11 @@ const PageTitle = styled(Typography.Title)`
     }
 `;
 
-const ListContainer = styled.div``;
+const ListContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
 
 /**
  * Component used for displaying the 'Manage Views' experience.
diff --git a/datahub-web-react/src/app/identity/ManageIdentities.tsx b/datahub-web-react/src/app/identity/ManageIdentities.tsx
index 907687cec36bc..7aafe2676f3f6 100644
--- a/datahub-web-react/src/app/identity/ManageIdentities.tsx
+++ b/datahub-web-react/src/app/identity/ManageIdentities.tsx
@@ -8,6 +8,10 @@ import { UserList } from './user/UserList';
 const PageContainer = styled.div`
     padding-top: 20px;
     width: 100%;
+    overflow: auto;
+    flex: 1;
+    display: flex;
+    flex-direction: column;
 `;
 
 const PageHeaderContainer = styled.div`
@@ -23,11 +27,14 @@ const PageTitle = styled(Typography.Title)`
 `;
 
 const Content = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
     &&& .ant-tabs-nav {
         margin: 0;
     }
     color: #262626;
-    height: calc(100vh - 60px);
+    // height: calc(100vh - 60px);
 
     &&& .ant-tabs > .ant-tabs-nav .ant-tabs-nav-wrap {
         padding-left: 28px;
diff --git a/datahub-web-react/src/app/identity/group/GroupList.tsx b/datahub-web-react/src/app/identity/group/GroupList.tsx
index 5ef77b4dfc8a8..788b9eccafc0a 100644
--- a/datahub-web-react/src/app/identity/group/GroupList.tsx
+++ b/datahub-web-react/src/app/identity/group/GroupList.tsx
@@ -17,9 +17,16 @@ import { GROUPS_CREATE_GROUP_ID, GROUPS_INTRO_ID } from '../../onboarding/config
 import { OnboardingTour } from '../../onboarding/OnboardingTour';
 import { addGroupToListGroupsCache, DEFAULT_GROUP_LIST_PAGE_SIZE, removeGroupFromListGroupsCache } from './cacheUtils';
 
-const GroupContainer = styled.div``;
+const GroupContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
 
 const GroupStyledList = styled(List)`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
     &&& {
         width: 100%;
         border-color: ${(props) => props.theme.styles['border-color-base']};
diff --git a/datahub-web-react/src/app/identity/user/UserList.tsx b/datahub-web-react/src/app/identity/user/UserList.tsx
index e50005b08377e..dce3aa2c68a8d 100644
--- a/datahub-web-react/src/app/identity/user/UserList.tsx
+++ b/datahub-web-react/src/app/identity/user/UserList.tsx
@@ -25,9 +25,16 @@ import { useUpdateEducationStepIdsAllowlist } from '../../onboarding/useUpdateEd
 import { DEFAULT_USER_LIST_PAGE_SIZE, removeUserFromListUsersCache } from './cacheUtils';
 import { useUserContext } from '../../context/useUserContext';
 
-const UserContainer = styled.div``;
+const UserContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
 
 const UserStyledList = styled(List)`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
     &&& {
         width: 100%;
         border-color: ${(props) => props.theme.styles['border-color-base']};
diff --git a/datahub-web-react/src/app/permissions/ManagePermissions.tsx b/datahub-web-react/src/app/permissions/ManagePermissions.tsx
index 7bf517e153fa8..737af3fdee3e4 100644
--- a/datahub-web-react/src/app/permissions/ManagePermissions.tsx
+++ b/datahub-web-react/src/app/permissions/ManagePermissions.tsx
@@ -8,6 +8,9 @@ import { ManageRoles } from './roles/ManageRoles';
 const PageContainer = styled.div`
     padding-top: 20px;
     width: 100%;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const PageHeaderContainer = styled.div`
@@ -27,7 +30,9 @@ const Content = styled.div`
         margin: 0;
     }
     color: #262626;
-    height: calc(100vh - 60px);
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 
     &&& .ant-tabs > .ant-tabs-nav .ant-tabs-nav-wrap {
         padding-left: 28px;
diff --git a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx
index 49b0ec922fd57..2f0c284fc4e8f 100644
--- a/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx
+++ b/datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx
@@ -38,7 +38,11 @@ import analytics, { EventType } from '../../analytics';
 import { POLICIES_CREATE_POLICY_ID, POLICIES_INTRO_ID } from '../../onboarding/config/PoliciesOnboardingConfig';
 import { OnboardingTour } from '../../onboarding/OnboardingTour';
 
-const SourceContainer = styled.div``;
+const SourceContainer = styled.div`
+    overflow: auto;
+    display: flex;
+    flex-direction: column;
+`;
 
 const PaginationContainer = styled.div`
     display: flex;
@@ -75,6 +79,9 @@ const EditPolicyButton = styled(Button)`
 
 const PageContainer = styled.span`
     width: 100%;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const DEFAULT_PAGE_SIZE = 10;
diff --git a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
index 011109e2eb915..4982be27fa421 100644
--- a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
+++ b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx
@@ -21,7 +21,11 @@ import { OnboardingTour } from '../../onboarding/OnboardingTour';
 import { ROLES_INTRO_ID } from '../../onboarding/config/RolesOnboardingConfig';
 import { clearUserListCache } from '../../identity/user/cacheUtils';
 
-const SourceContainer = styled.div``;
+const SourceContainer = styled.div`
+    overflow: auto;
+    display: flex;
+    flex-direction: column;
+`;
 
 const PaginationContainer = styled.div`
     display: flex;
@@ -35,6 +39,9 @@ const RoleName = styled.span`
 
 const PageContainer = styled.span`
     width: 100%;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const ActionsContainer = styled.div`
diff --git a/datahub-web-react/src/app/settings/AccessTokens.tsx b/datahub-web-react/src/app/settings/AccessTokens.tsx
index c7a015de392da..2c986f90aba48 100644
--- a/datahub-web-react/src/app/settings/AccessTokens.tsx
+++ b/datahub-web-react/src/app/settings/AccessTokens.tsx
@@ -21,6 +21,9 @@ const SourceContainer = styled.div`
     padding-top: 20px;
     padding-right: 40px;
     padding-left: 40px;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const TokensContainer = styled.div`
diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx
index 06592656ac719..aba054a7fb94f 100644
--- a/datahub-web-react/src/app/settings/SettingsPage.tsx
+++ b/datahub-web-react/src/app/settings/SettingsPage.tsx
@@ -24,12 +24,15 @@ import ManagePosts from './posts/ManagePosts';
 
 const PageContainer = styled.div`
     display: flex;
+    overflow: auto;
 `;
 
 const SettingsBarContainer = styled.div`
     padding-top: 20px;
-    min-height: 100vh;
+    max-height: 100vh;
     border-right: 1px solid ${ANTD_GRAY[5]};
+    display: flex;
+    flex-direction: column;
 `;
 
 const SettingsBarHeader = styled.div`
@@ -54,6 +57,8 @@ const ItemTitle = styled.span`
     margin-left: 8px;
 `;
 
+const menuStyle = { width: 256, 'margin-top': 8, overflow: 'hidden auto' };
+
 /**
  * URL Paths for each settings page.
  */
@@ -108,7 +113,7 @@ export const SettingsPage = () => {
                 <Menu
                     selectable={false}
                     mode="inline"
-                    style={{ width: 256, marginTop: 8 }}
+                    style={menuStyle}
                     selectedKeys={[activePath]}
                     onClick={(newPath) => {
                         history.replace(`${url}/${newPath.key}`);
diff --git a/datahub-web-react/src/app/settings/posts/ManagePosts.tsx b/datahub-web-react/src/app/settings/posts/ManagePosts.tsx
index e0f694c192c62..0c9e9a7a595b5 100644
--- a/datahub-web-react/src/app/settings/posts/ManagePosts.tsx
+++ b/datahub-web-react/src/app/settings/posts/ManagePosts.tsx
@@ -6,7 +6,9 @@ import { PostList } from './PostsList';
 const PageContainer = styled.div`
     padding-top: 20px;
     width: 100%;
-    height: 100%;
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
 `;
 
 const PageHeaderContainer = styled.div`
@@ -21,7 +23,11 @@ const PageTitle = styled(Typography.Title)`
     }
 `;
 
-const ListContainer = styled.div``;
+const ListContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
 
 export default function ManagePosts() {
     return (
diff --git a/datahub-web-react/src/app/settings/posts/PostsList.tsx b/datahub-web-react/src/app/settings/posts/PostsList.tsx
index 5ae2be1547f9b..849a3765a94b0 100644
--- a/datahub-web-react/src/app/settings/posts/PostsList.tsx
+++ b/datahub-web-react/src/app/settings/posts/PostsList.tsx
@@ -17,7 +17,11 @@ import { SearchBar } from '../../search/SearchBar';
 import { StyledTable } from '../../entity/shared/components/styled/StyledTable';
 import { POST_TYPE_TO_DISPLAY_TEXT } from './constants';
 
-const PostsContainer = styled.div``;
+const PostsContainer = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
 
 export const PostsPaginationContainer = styled.div`
     display: flex;
diff --git a/datahub-web-react/src/app/shared/RoutedTabs.tsx b/datahub-web-react/src/app/shared/RoutedTabs.tsx
index b349ef759940c..574f372e53513 100644
--- a/datahub-web-react/src/app/shared/RoutedTabs.tsx
+++ b/datahub-web-react/src/app/shared/RoutedTabs.tsx
@@ -3,6 +3,7 @@ import { Route, Switch, useRouteMatch, useLocation } from 'react-router-dom';
 import { Redirect, useHistory } from 'react-router';
 import { Tabs } from 'antd';
 import { TabsProps } from 'antd/lib/tabs';
+import styled from 'styled-components';
 
 const { TabPane } = Tabs;
 
@@ -19,6 +20,12 @@ interface Props extends TabsProps {
     onTabChange?: (selectedTab: string) => void;
 }
 
+const RoutedTabsStyle = styled.div`
+    display: flex;
+    flex-direction: column;
+    overflow: auto;
+`;
+
 /**
  * A tab view where each tab is associated with a route mounted on top of the current path.
  * This permits direct navigation to a particular tab via URL.
@@ -33,7 +40,7 @@ export const RoutedTabs = ({ defaultPath, tabs, onTabChange, ...props }: Props)
     const providedPath = splitPathName[splitPathName.length - 1];
     const activePath = subRoutes.includes(providedPath) ? providedPath : defaultPath.replace('/', '');
     return (
-        <div>
+        <RoutedTabsStyle>
             <Tabs
                 defaultActiveKey={activePath}
                 activeKey={activePath}
@@ -62,6 +69,6 @@ export const RoutedTabs = ({ defaultPath, tabs, onTabChange, ...props }: Props)
                     />
                 ))}
             </Switch>
-        </div>
+        </RoutedTabsStyle>
     );
 };

From 1ad4f961aa8a61a6c31add573ac486895787978e Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Mon, 20 Nov 2023 18:02:49 +0900
Subject: [PATCH 233/259] feat: add ingestion overview pages  (#9210)

---
 docs-website/sidebars.js              |  34 ++--
 docs/cli.md                           |  61 +++++++
 metadata-ingestion/README.md          | 235 +++-----------------------
 metadata-ingestion/cli-ingestion.md   |  59 +++++++
 metadata-ingestion/recipe_overview.md | 124 ++++++++++++++
 metadata-ingestion/sink_overview.md   |  33 ++++
 metadata-ingestion/source_overview.md |  37 ++++
 7 files changed, 359 insertions(+), 224 deletions(-)
 create mode 100644 metadata-ingestion/cli-ingestion.md
 create mode 100644 metadata-ingestion/recipe_overview.md
 create mode 100644 metadata-ingestion/sink_overview.md
 create mode 100644 metadata-ingestion/source_overview.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 801e0fbd07d36..be12aa3a827f2 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -30,17 +30,16 @@ module.exports = {
       ],
     },
     {
-      Integrations: [
+      type: "category",
+      label: "Integrations",
+      link: { type: "doc", id: "metadata-ingestion/README" },
+      items: [
         // The purpose of this section is to provide a deeper understanding of how ingestion works.
         // Readers should be able to find details for ingesting from all systems, apply transformers, understand sinks,
         // and understand key concepts of the Ingestion Framework (Sources, Sinks, Transformers, and Recipes)
-        {
-          type: "doc",
-          label: "Introduction",
-          id: "metadata-ingestion/README",
-        },
         {
           "Quickstart Guides": [
+            "metadata-ingestion/cli-ingestion",
             {
               BigQuery: [
                 "docs/quick-ingestion-guides/bigquery/overview",
@@ -85,15 +84,18 @@ module.exports = {
             },
           ],
         },
+        "metadata-ingestion/recipe_overview",
         {
-          Sources: [
+          type: "category",
+          label: "Sources",
+          link: { type: "doc", id: "metadata-ingestion/source_overview" },
+          items: [
             // collapse these; add push-based at top
             {
               type: "doc",
               id: "docs/lineage/airflow",
               label: "Airflow",
             },
-
             //"docker/airflow/local_airflow",
             "metadata-integration/java/spark-lineage/README",
             "metadata-ingestion/integration_docs/great-expectations",
@@ -106,7 +108,10 @@ module.exports = {
           ],
         },
         {
-          Sinks: [
+          type: "category",
+          label: "Sinks",
+          link: { type: "doc", id: "metadata-ingestion/sink_overview" },
+          items: [
             {
               type: "autogenerated",
               dirName: "metadata-ingestion/sink_docs",
@@ -114,10 +119,13 @@ module.exports = {
           ],
         },
         {
-          Transformers: [
-            "metadata-ingestion/docs/transformer/intro",
-            "metadata-ingestion/docs/transformer/dataset_transformer",
-          ],
+          type: "category",
+          label: "Transformers",
+          link: {
+            type: "doc",
+            id: "metadata-ingestion/docs/transformer/intro",
+          },
+          items: ["metadata-ingestion/docs/transformer/dataset_transformer"],
         },
         {
           "Advanced Guides": [
diff --git a/docs/cli.md b/docs/cli.md
index 7dfac1e9b2bff..8845ed5a6dac7 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -99,6 +99,36 @@ Command Options:
   --strict-warnings         If enabled, ingestion runs with warnings will yield a non-zero error code
   --test-source-connection  When set, ingestion will only test the source connection details from the recipe
 ```
+#### ingest --dry-run
+
+The `--dry-run` option of the `ingest` command performs all of the ingestion steps, except writing to the sink. This is useful to validate that the
+ingestion recipe is producing the desired metadata events before ingesting them into datahub.
+
+```shell
+# Dry run
+datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml --dry-run
+# Short-form
+datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml -n
+```
+
+#### ingest --preview
+
+The `--preview` option of the `ingest` command performs all of the ingestion steps, but limits the processing to only the first 10 workunits produced by the source.
+This option helps with quick end-to-end smoke testing of the ingestion recipe.
+
+```shell
+# Preview
+datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml --preview
+# Preview with dry-run
+datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml -n --preview
+```
+
+By default `--preview` creates 10 workunits. But if you wish to try producing more workunits you can use another option `--preview-workunits`
+
+```shell
+# Preview 20 workunits without sending anything to sink
+datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml -n --preview --preview-workunits=20
+```
 
 #### ingest deploy
 
@@ -115,6 +145,37 @@ To update an existing recipe please use the `--urn` parameter to specify the id
 **Note:** Updating a recipe will result in a replacement of the existing options with what was specified in the cli command.
 I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated.
 
+#### ingest --no-default-report
+By default, the cli sends an ingestion report to DataHub, which allows you to see the result of all cli-based ingestion in the UI. This can be turned off with the `--no-default-report` flag.
+
+```shell
+# Running ingestion with reporting to DataHub turned off
+datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --no-default-report
+```
+
+The reports include the recipe that was used for ingestion. This can be turned off by adding an additional section to the ingestion recipe.
+
+```yaml
+source:
+  # source configs
+
+sink:
+  # sink configs
+
+# Add configuration for the datahub reporter
+reporting:
+  - type: datahub
+    config:
+      report_recipe: false
+
+# Optional log to put failed JSONs into a file
+# Helpful in case you are trying to debug some issue with specific ingestion failing
+failure_log:
+  enabled: false
+  log_config:
+    filename: ./path/to/failure.json
+```
+
 ### init
 
 The init command is used to tell `datahub` about where your DataHub instance is located. The CLI will point to localhost DataHub by default.
diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md
index a0fef614528cb..54478fddbe2d0 100644
--- a/metadata-ingestion/README.md
+++ b/metadata-ingestion/README.md
@@ -1,228 +1,41 @@
 # Introduction to Metadata Ingestion
 
-<a
-    className='button button--primary button--lg'
-    href="https://datahubproject.io/integrations">
-    Find Integration Source
-</a>
-
-## Integration Options
-
-DataHub supports both **push-based** and **pull-based** metadata integration.
-
-Push-based integrations allow you to emit metadata directly from your data systems when metadata changes, while pull-based integrations allow you to "crawl" or "ingest" metadata from the data systems by connecting to them and extracting metadata in a batch or incremental-batch manner. Supporting both mechanisms means that you can integrate with all your systems in the most flexible way possible.
-
-Examples of push-based integrations include [Airflow](../docs/lineage/airflow.md), [Spark](../metadata-integration/java/spark-lineage/README.md), [Great Expectations](./integration_docs/great-expectations.md) and [Protobuf Schemas](../metadata-integration/java/datahub-protobuf/README.md). This allows you to get low-latency metadata integration from the "active" agents in your data ecosystem. Examples of pull-based integrations include BigQuery, Snowflake, Looker, Tableau and many others.
-
-This document describes the pull-based metadata ingestion system that is built into DataHub for easy integration with a wide variety of sources in your data stack.
-
-## Getting Started
-
-### Prerequisites
-
-Before running any metadata ingestion job, you should make sure that DataHub backend services are all running. You can either run ingestion via the [UI](../docs/ui-ingestion.md) or via the [CLI](../docs/cli.md). You can reference the CLI usage guide given there as you go through this page.
-
-## Core Concepts
-
-### Sources
-
-Please see our [Integrations page](https://datahubproject.io/integrations) to browse our ingestion sources and filter on their features.
-
-Data systems that we are extracting metadata from are referred to as **Sources**. The `Sources` tab on the left in the sidebar shows you all the sources that are available for you to ingest metadata from. For example, we have sources for [BigQuery](https://datahubproject.io/docs/generated/ingestion/sources/bigquery), [Looker](https://datahubproject.io/docs/generated/ingestion/sources/looker), [Tableau](https://datahubproject.io/docs/generated/ingestion/sources/tableau) and many others.
-
-#### Metadata Ingestion Source Status
-
-We apply a Support Status to each Metadata Source to help you understand the integration reliability at a glance.
-
-![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen): Certified Sources are well-tested & widely-adopted by the DataHub Community. We expect the integration to be stable with few user-facing issues.
-
-![Incubating](https://img.shields.io/badge/support%20status-incubating-blue): Incubating Sources are ready for DataHub Community adoption but have not been tested for a wide variety of edge-cases. We eagerly solicit feedback from the Community to streghten the connector; minor version changes may arise in future releases.
-
-![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey): Testing Sources are available for experiementation by DataHub Community members, but may change without notice.
-
-### Sinks
-
-Sinks are destinations for metadata. When configuring ingestion for DataHub, you're likely to be sending the metadata to DataHub over either the [REST (datahub-sink)](./sink_docs/datahub.md#datahub-rest) or the [Kafka (datahub-kafka)](./sink_docs/datahub.md#datahub-kafka) sink. In some cases, the [File](./sink_docs/file.md) sink is also helpful to store a persistent offline copy of the metadata during debugging.
-
-The default sink that most of the ingestion systems and guides assume is the `datahub-rest` sink, but you should be able to adapt all of them for the other sinks as well!
-
-### Recipes
-
-A recipe is the main configuration file that puts it all together. It tells our ingestion scripts where to pull data from (source) and where to put it (sink).
-
-:::tip
-Name your recipe with **.dhub.yaml** extension like _myrecipe.dhub.yaml_ to use vscode or intellij as a recipe editor with autocomplete
-and syntax validation.
-
-Make sure yaml plugin is installed for your editor:
-
-- For vscode install [Redhat's yaml plugin](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml)
-- For intellij install [official yaml plugin](https://plugins.jetbrains.com/plugin/13126-yaml)
-
+:::tip Find Integration Source
+Please see our **[Integrations page](https://datahubproject.io/integrations)** to browse our ingestion sources and filter on their features.
 :::
 
-Since `acryl-datahub` version `>=0.8.33.2`, the default sink is assumed to be a DataHub REST endpoint:
-
-- Hosted at "http://localhost:8080" or the environment variable `${DATAHUB_GMS_URL}` if present
-- With an empty auth token or the environment variable `${DATAHUB_GMS_TOKEN}` if present.
-
-Here's a simple recipe that pulls metadata from MSSQL (source) and puts it into the default sink (datahub rest).
-
-```yaml
-# The simplest recipe that pulls metadata from MSSQL and puts it into DataHub
-# using the Rest API.
-source:
-  type: mssql
-  config:
-    username: sa
-    password: ${MSSQL_PASSWORD}
-    database: DemoData
-# sink section omitted as we want to use the default datahub-rest sink
-```
-
-Running this recipe is as simple as:
+## Integration Methods
 
-```shell
-datahub ingest -c recipe.dhub.yaml
-```
+DataHub offers three methods for data ingestion:
 
-or if you want to override the default endpoints, you can provide the environment variables as part of the command like below:
+- [UI Ingestion](../docs/ui-ingestion.md) : Easily configure and execute a metadata ingestion pipeline through the UI. 
+- [CLI Ingestion guide](cli-ingestion.md) : Configure the ingestion pipeline using YAML and execute by it through CLI. 
+- SDK-based ingestion : Use [Python Emitter](./as-a-library.md) or [Java emitter](../metadata-integration/java/as-a-library.md) to programmatically control the ingestion pipelines. 
 
-```shell
-DATAHUB_GMS_URL="https://my-datahub-server:8080" DATAHUB_GMS_TOKEN="my-datahub-token" datahub ingest -c recipe.dhub.yaml
-```
+## Types of Integration
 
-A number of recipes are included in the [examples/recipes](./examples/recipes) directory. For full info and context on each source and sink, see the pages described in the [table of plugins](../docs/cli.md#installing-plugins).
+Integration can be divided into two concepts based on the method:
 
-> Note that one recipe file can only have 1 source and 1 sink. If you want multiple sources then you will need multiple recipe files.
+### Push-based Integration
 
-### Handling sensitive information in recipes
+Push-based integrations allow you to emit metadata directly from your data systems when metadata changes.
+Examples of push-based integrations include [Airflow](../docs/lineage/airflow.md), [Spark](../metadata-integration/java/spark-lineage/README.md), [Great Expectations](./integration_docs/great-expectations.md) and [Protobuf Schemas](../metadata-integration/java/datahub-protobuf/README.md). This allows you to get low-latency metadata integration from the "active" agents in your data ecosystem.
 
-We automatically expand environment variables in the config (e.g. `${MSSQL_PASSWORD}`),
-similar to variable substitution in GNU bash or in docker-compose files. For details, see
-https://docs.docker.com/compose/compose-file/compose-file-v2/#variable-substitution. This environment variable substitution should be used to mask sensitive information in recipe files. As long as you can get env variables securely to the ingestion process there would not be any need to store sensitive information in recipes.
+### Pull-based Integration
 
-### Basic Usage of CLI for ingestion
+Pull-based integrations allow you to "crawl" or "ingest" metadata from the data systems by connecting to them and extracting metadata in a batch or incremental-batch manner. 
+Examples of pull-based integrations include BigQuery, Snowflake, Looker, Tableau and many others.
 
-```shell
-pip install 'acryl-datahub[datahub-rest]'  # install the required plugin
-datahub ingest -c ./examples/recipes/mssql_to_datahub.dhub.yml
-```
-
-The `--dry-run` option of the `ingest` command performs all of the ingestion steps, except writing to the sink. This is useful to validate that the
-ingestion recipe is producing the desired metadata events before ingesting them into datahub.
-
-```shell
-# Dry run
-datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml --dry-run
-# Short-form
-datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml -n
-```
-
-The `--preview` option of the `ingest` command performs all of the ingestion steps, but limits the processing to only the first 10 workunits produced by the source.
-This option helps with quick end-to-end smoke testing of the ingestion recipe.
-
-```shell
-# Preview
-datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml --preview
-# Preview with dry-run
-datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml -n --preview
-```
-
-By default `--preview` creates 10 workunits. But if you wish to try producing more workunits you can use another option `--preview-workunits`
-
-```shell
-# Preview 20 workunits without sending anything to sink
-datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yml -n --preview --preview-workunits=20
-```
-
-#### Reporting
-
-By default, the cli sends an ingestion report to DataHub, which allows you to see the result of all cli-based ingestion in the UI. This can be turned off with the `--no-default-report` flag.
-
-```shell
-# Running ingestion with reporting to DataHub turned off
-datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --no-default-report
-```
-
-The reports include the recipe that was used for ingestion. This can be turned off by adding an additional section to the ingestion recipe.
-
-```yaml
-source:
-  # source configs
-
-sink:
-  # sink configs
-
-# Add configuration for the datahub reporter
-reporting:
-  - type: datahub
-    config:
-      report_recipe: false
-
-# Optional log to put failed JSONs into a file
-# Helpful in case you are trying to debug some issue with specific ingestion failing
-failure_log:
-  enabled: false
-  log_config:
-    filename: ./path/to/failure.json
-```
-
-#### Deploying and scheduling ingestion to the UI
-
-The `deploy` subcommand of the `ingest` command tree allows users to upload their recipes and schedule them in the server.
-
-```shell
-datahub ingest deploy -n <user friendly name for ingestion> -c recipe.yaml
-```
-
-By default, no schedule is done unless explicitly configured with the `--schedule` parameter. Schedule timezones are UTC by default and can be overriden with `--time-zone` flag.
-```shell
-datahub ingest deploy -n test --schedule "0 * * * *" --time-zone "Europe/London" -c recipe.yaml
-```
-
-## Transformations
-
-If you'd like to modify data before it reaches the ingestion sinks – for instance, adding additional owners or tags – you can use a transformer to write your own module and integrate it with DataHub. Transformers require extending the recipe with a new section to describe the transformers that you want to run.
-
-For example, a pipeline that ingests metadata from MSSQL and applies a default "important" tag to all datasets is described below:
-
-```yaml
-# A recipe to ingest metadata from MSSQL and apply default tags to all tables
-source:
-  type: mssql
-  config:
-    username: sa
-    password: ${MSSQL_PASSWORD}
-    database: DemoData
-
-transformers: # an array of transformers applied sequentially
-  - type: simple_add_dataset_tags
-    config:
-      tag_urns:
-        - "urn:li:tag:Important"
-# default sink, no config needed
-```
-
-Check out the [transformers guide](./docs/transformer/intro.md) to learn more about how you can create really flexible pipelines for processing metadata using Transformers!
-
-## Using as a library (SDK)
-
-In some cases, you might want to construct Metadata events directly and use programmatic ways to emit that metadata to DataHub. In this case, take a look at the [Python emitter](./as-a-library.md) and the [Java emitter](../metadata-integration/java/as-a-library.md) libraries which can be called from your own code.
-
-### Programmatic Pipeline
-
-In some cases, you might want to configure and run a pipeline entirely from within your custom Python script. Here is an example of how to do it.
-
-- [programmatic_pipeline.py](./examples/library/programatic_pipeline.py) - a basic mysql to REST programmatic pipeline.
-
-## Developing
+## Core Concepts
 
-See the guides on [developing](./developing.md), [adding a source](./adding-source.md) and [using transformers](./docs/transformer/intro.md).
+The following are the core concepts related to ingestion:
 
-## Compatibility
+- [Sources](source_overview.md): Data systems from which extract metadata. (e.g. BigQuery, MySQL)
+- [Sinks](sink_overview.md): Destination for metadata (e.g. File, DataHub)
+- [Recipe](recipe_overview.md): The main configuration for ingestion in the form or .yaml file
 
-DataHub server uses a 3 digit versioning scheme, while the CLI uses a 4 digit scheme. For example, if you're using DataHub server version 0.10.0, you should use CLI version 0.10.0.x, where x is a patch version.
-We do this because we do CLI releases at a much higher frequency than server releases, usually every few days vs twice a month.
+For more advanced guides, please refer to the following:
 
-For ingestion sources, any breaking changes will be highlighted in the [release notes](../docs/how/updating-datahub.md). When fields are deprecated or otherwise changed, we will try to maintain backwards compatibility for two server releases, which is about 4-6 weeks. The CLI will also print warnings whenever deprecated options are used.
+- [Developing on Metadata Ingestion](./developing.md)
+- [Adding a Metadata Ingestion Source](./adding-source.md)
+- [Using Transformers](./docs/transformer/intro.md)
diff --git a/metadata-ingestion/cli-ingestion.md b/metadata-ingestion/cli-ingestion.md
new file mode 100644
index 0000000000000..cbdde2cd30167
--- /dev/null
+++ b/metadata-ingestion/cli-ingestion.md
@@ -0,0 +1,59 @@
+# CLI Ingestion
+
+## Installing the CLI
+
+Make sure you have installed DataHub CLI before following this guide. 
+```shell
+# Requires Python 3.7+
+python3 -m pip install --upgrade pip wheel setuptools
+python3 -m pip install --upgrade acryl-datahub
+# validate that the install was successful
+datahub version
+# If you see "command not found", try running this instead: python3 -m datahub version
+```
+Check out the [CLI Installation Guide](../docs/cli.md#installation) for more installation options and troubleshooting tips. 
+
+After that, install the required plugin for the ingestion.
+
+```shell
+pip install 'acryl-datahub[datahub-rest]'  # install the required plugin
+```
+Check out the [alternative installation options](../docs/cli.md#alternate-installation-options) for more reference. 
+
+## Configuring a Recipe
+Create a recipe.yml file that defines the source and sink for metadata, as shown below.
+```yaml
+# my_reipe.yml
+source:
+  type: <source_name>
+  config:
+    option_1: <value>
+    ...
+  
+sink:
+  type: <sink_type_name>
+  config:
+    ...
+```
+
+For more information and examples on configuring recipes, please refer to [Recipes](recipe_overview.md).
+
+## Ingesting Metadata
+You can run ingestion using `datahub ingest` like below.  
+
+```shell
+datahub ingest -c <path_to_recipe_file.yml>
+```
+
+## Reference
+
+Please refer the following pages for advanced guids on CLI ingestion.
+- [Reference for `datahub ingest` command](../docs/cli.md#ingest)
+- [UI Ingestion Guide](../docs/ui-ingestion.md)
+
+:::Tip Compatibility
+DataHub server uses a 3 digit versioning scheme, while the CLI uses a 4 digit scheme. For example, if you're using DataHub server version 0.10.0, you should use CLI version 0.10.0.x, where x is a patch version.
+We do this because we do CLI releases at a much higher frequency than server releases, usually every few days vs twice a month.
+
+For ingestion sources, any breaking changes will be highlighted in the [release notes](../docs/how/updating-datahub.md). When fields are deprecated or otherwise changed, we will try to maintain backwards compatibility for two server releases, which is about 4-6 weeks. The CLI will also print warnings whenever deprecated options are used.
+:::
\ No newline at end of file
diff --git a/metadata-ingestion/recipe_overview.md b/metadata-ingestion/recipe_overview.md
new file mode 100644
index 0000000000000..a748edbf3bb44
--- /dev/null
+++ b/metadata-ingestion/recipe_overview.md
@@ -0,0 +1,124 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Recipes
+
+A recipe is the main configuration file for metadata ingestion. It tells our ingestion scripts where to pull data from (source) and where to put it (sink).
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sources-sinks.png"/>
+</p>
+
+
+## Configuring Recipe
+
+The basic form of the recipe file consists of:
+
+- `source`, which contains the configuration of the data source. (See [Sources](source_overview.md))
+- `sink`, which defines the destination of the metadata (See [Sinks](sink_overview.md))
+
+Here's a simple recipe that pulls metadata from MSSQL (source) and puts it into the default sink (datahub rest).
+
+```yaml
+# The simplest recipe that pulls metadata from MSSQL and puts it into DataHub
+# using the Rest API.
+source:
+  type: mssql
+  config:
+    username: sa
+    password: ${MSSQL_PASSWORD}
+    database: DemoData
+# sink section omitted as we want to use the default datahub-rest sink
+sink:
+  type: "datahub-rest"
+  config:
+    server: "http://localhost:8080"
+```
+
+A number of recipes are included in the [examples/recipes](./examples/recipes) directory. For full info and context on each source and sink, see the pages described in the [table of plugins](../docs/cli.md#installing-plugins).
+
+:::note One Source/Sink for One Recipe!
+Note that one recipe file can only have 1 source and 1 sink. If you want multiple sources then you will need multiple recipe files.
+:::
+
+## Running a Recipe
+
+DataHub supports running recipes via the CLI or UI.
+
+<Tabs>
+<TabItem value="cli" label="CLI" default>
+
+Install CLI and the plugin for the ingestion.
+```shell
+python3 -m pip install --upgrade acryl-datahub
+pip install 'acryl-datahub[datahub-rest]'
+```
+Running this recipe is as simple as:
+
+```shell
+datahub ingest -c recipe.dhub.yaml
+```
+For a detailed guide on running recipes via CLI, please refer to [CLI Ingestion Guide](cli-ingestion.md).
+
+</TabItem>
+
+<TabItem value="ui" label="UI">
+
+You can configure and run the recipe in **Ingestion** tab in DataHub. 
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/ingestion-tab.png"/>
+</p>
+
+* Make sure you have the **Manage Metadata Ingestion & Manage Secret** privileges.
+* Navigate to **Ingestion** tab in DataHub.
+* Create an ingestion source & configure the recipe via UI.
+* Hit **Execute**.
+
+For a detailed guide on running recipes via UI, please refer to [UI Ingestion Guide](../docs/ui-ingestion.md).
+
+</TabItem>
+</Tabs>
+
+
+## Advanced Configuration
+
+### Handling Sensitive Information in Recipes
+
+We automatically expand environment variables in the config (e.g. `${MSSQL_PASSWORD}`),
+similar to variable substitution in GNU bash or in docker-compose files.
+For details, see [variable-substitution](https://docs.docker.com/compose/compose-file/compose-file-v2/#variable-substitution).
+This environment variable substitution should be used to mask sensitive information in recipe files. As long as you can get env variables securely to the ingestion process there would not be any need to store sensitive information in recipes.
+
+### Transformations
+
+If you'd like to modify data before it reaches the ingestion sinks – for instance, adding additional owners or tags – you can use a transformer to write your own module and integrate it with DataHub. Transformers require extending the recipe with a new section to describe the transformers that you want to run.
+
+For example, a pipeline that ingests metadata from MSSQL and applies a default "important" tag to all datasets is described below:
+
+```yaml
+# A recipe to ingest metadata from MSSQL and apply default tags to all tables
+source:
+  type: mssql
+  config:
+    username: sa
+    password: ${MSSQL_PASSWORD}
+    database: DemoData
+
+transformers: # an array of transformers applied sequentially
+  - type: simple_add_dataset_tags
+    config:
+      tag_urns:
+        - "urn:li:tag:Important"
+# default sink, no config needed
+```
+
+Check out the [transformers guide](./docs/transformer/intro.md) to learn more about how you can create really flexible pipelines for processing metadata using Transformers!
+
+### Autocomplete and Syntax Validation
+
+Name your recipe with **.dhub.yaml** extension like `myrecipe.dhub.yaml_` to use vscode or intellij as a recipe editor with autocomplete
+and syntax validation. Make sure yaml plugin is installed for your editor:
+
+- For vscode install [Redhat's yaml plugin](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml)
+- For intellij install [official yaml plugin](https://plugins.jetbrains.com/plugin/13126-yaml)
diff --git a/metadata-ingestion/sink_overview.md b/metadata-ingestion/sink_overview.md
new file mode 100644
index 0000000000000..c71ba1f97932c
--- /dev/null
+++ b/metadata-ingestion/sink_overview.md
@@ -0,0 +1,33 @@
+# Sinks
+
+Sinks are **destinations for metadata**.
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sources-sinks.png"/>
+</p>
+
+In general, the sink will be defined in the [recipe](./recipe_overview.md) after the [source](./source-docs-template.md) like below.
+
+```yaml
+source: ...
+
+sink:
+  type: <sink_type>
+  config: ...
+```
+
+## Types of Sinks
+
+When configuring ingestion for DataHub, you're likely to be sending the metadata to DataHub over either one of the following.
+
+- [REST (datahub-rest)](sink_docs/datahub.md#datahub-rest)
+- [Kafka (datahub-kafka)](sink_docs/datahub.md#datahub-kafka)
+
+For debugging purposes or troubleshooting, the following sinks can be useful:
+
+- [File](sink_docs/file.md)
+- [Console](sink_docs/console.md)
+
+## Default Sink
+
+Since `acryl-datahub` version `>=0.8.33.2`, the default sink is assumed to be a `datahub-rest` endpoint.
diff --git a/metadata-ingestion/source_overview.md b/metadata-ingestion/source_overview.md
new file mode 100644
index 0000000000000..9647fbdde0a0f
--- /dev/null
+++ b/metadata-ingestion/source_overview.md
@@ -0,0 +1,37 @@
+# Sources
+
+
+Sources are **the data systems that we are extracting metadata from.** 
+
+<p align="center">
+  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/sources-sinks.png"/>
+</p>
+
+In general, the source will be defined at the top of the [recipe](./recipe_overview.md) like below.
+
+
+```yaml
+#my_recipe.yml
+source: 
+  type: <source_name>
+  config: 
+    option_1: <value>
+    ...
+```
+
+## Types of Source
+The `Sources` tab on the left in the sidebar shows you all the sources that are available for you to ingest metadata from. For example, we have sources for [BigQuery](https://datahubproject.io/docs/generated/ingestion/sources/bigquery), [Looker](https://datahubproject.io/docs/generated/ingestion/sources/looker), [Tableau](https://datahubproject.io/docs/generated/ingestion/sources/tableau) and many others.
+
+:::tip Find an Integration Source
+See the full **[list of integrations](https://datahubproject.io/integrations)** and filter on their features.
+:::
+
+## Metadata Ingestion Source Status
+
+We apply a Support Status to each Metadata Source to help you understand the integration reliability at a glance.
+
+![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen): Certified Sources are well-tested & widely-adopted by the DataHub Community. We expect the integration to be stable with few user-facing issues.
+
+![Incubating](https://img.shields.io/badge/support%20status-incubating-blue): Incubating Sources are ready for DataHub Community adoption but have not been tested for a wide variety of edge-cases. We eagerly solicit feedback from the Community to streghten the connector; minor version changes may arise in future releases.
+
+![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey): Testing Sources are available for experiementation by DataHub Community members, but may change without notice.

From 01874808afd0c65518963a22966b1e1592d75182 Mon Sep 17 00:00:00 2001
From: Tim <50115603+bossenti@users.noreply.github.com>
Date: Mon, 20 Nov 2023 16:57:48 +0100
Subject: [PATCH 234/259] fix(ingest/athena): detect decimal type correctly
 (#9270)

---
 .../src/datahub/ingestion/source/sql/athena.py         |  4 ++++
 metadata-ingestion/tests/unit/test_athena_source.py    | 10 ++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
index 75e8fe1d6f7a6..ac0e2bd4bb8a9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py
@@ -183,6 +183,10 @@ def _get_column_type(
         # can also be returned, so we need to extend the handling here as well
         elif type_name in ["bigint", "long"]:
             detected_col_type = types.BIGINT
+        elif type_name in ["decimal"]:
+            detected_col_type = types.DECIMAL
+            precision, scale = type_meta_information.split(",")
+            args = [int(precision), int(scale)]
         else:
             return super()._get_column_type(type_name)
         return detected_col_type(*args)
diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py
index 23dd7dd5a6e45..875cf3800daf8 100644
--- a/metadata-ingestion/tests/unit/test_athena_source.py
+++ b/metadata-ingestion/tests/unit/test_athena_source.py
@@ -166,7 +166,6 @@ def test_get_column_type_map():
 
 
 def test_column_type_struct():
-
     result = CustomAthenaRestDialect()._get_column_type(type_="struct<test:string>")
 
     assert isinstance(result, STRUCT)
@@ -175,8 +174,15 @@ def test_column_type_struct():
     assert isinstance(result._STRUCT_fields[0][1], types.String)
 
 
-def test_column_type_complex_combination():
+def test_column_type_decimal():
+    result = CustomAthenaRestDialect()._get_column_type(type_="decimal(10,2)")
+
+    assert isinstance(result, types.DECIMAL)
+    assert 10 == result.precision
+    assert 2 == result.scale
 
+
+def test_column_type_complex_combination():
     result = CustomAthenaRestDialect()._get_column_type(
         type_="struct<id:string,name:string,choices:array<struct<id:string,label:string>>>"
     )

From d9de854d276c118afc55264ecc9e2712b91b4ab2 Mon Sep 17 00:00:00 2001
From: gaurav2733 <77378510+gaurav2733@users.noreply.github.com>
Date: Tue, 21 Nov 2023 07:34:34 +0530
Subject: [PATCH 235/259] fix(ui): Do not show manage in settings when no
 permissions for it Fix/prd 787 (#9261)

---
 .../src/app/settings/SettingsPage.tsx         | 37 ++++++++++---------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx
index aba054a7fb94f..69d4eb2b10b4d 100644
--- a/datahub-web-react/src/app/settings/SettingsPage.tsx
+++ b/datahub-web-react/src/app/settings/SettingsPage.tsx
@@ -141,24 +141,25 @@ export const SettingsPage = () => {
                             )}
                         </Menu.ItemGroup>
                     )}
-
-                    <Menu.ItemGroup title="Manage">
-                        {showViews && (
-                            <Menu.Item key="views">
-                                <FilterOutlined /> <ItemTitle>My Views</ItemTitle>
-                            </Menu.Item>
-                        )}
-                        {showOwnershipTypes && (
-                            <Menu.Item key="ownership">
-                                <TeamOutlined /> <ItemTitle>Ownership Types</ItemTitle>
-                            </Menu.Item>
-                        )}
-                        {showHomePagePosts && (
-                            <Menu.Item key="posts">
-                                <PushpinOutlined /> <ItemTitle>Home Page Posts</ItemTitle>
-                            </Menu.Item>
-                        )}
-                    </Menu.ItemGroup>
+                    {(showViews || showOwnershipTypes || showHomePagePosts) && (
+                        <Menu.ItemGroup title="Manage">
+                            {showViews && (
+                                <Menu.Item key="views">
+                                    <FilterOutlined /> <ItemTitle>My Views</ItemTitle>
+                                </Menu.Item>
+                            )}
+                            {showOwnershipTypes && (
+                                <Menu.Item key="ownership">
+                                    <TeamOutlined /> <ItemTitle>Ownership Types</ItemTitle>
+                                </Menu.Item>
+                            )}
+                            {showHomePagePosts && (
+                                <Menu.Item key="posts">
+                                    <PushpinOutlined /> <ItemTitle>Home Page Posts</ItemTitle>
+                                </Menu.Item>
+                            )}
+                        </Menu.ItemGroup>
+                    )}
 
                     <Menu.ItemGroup title="Preferences">
                         <Menu.Item key="preferences">

From b51cfc3a408b1d923dc7c8023fc54859e34cc5ba Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 21 Nov 2023 08:59:50 -0800
Subject: [PATCH 236/259] build(deps): bump @babel/traverse from 7.22.11 to
 7.23.2 in /docs-website (#9022)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs-website/yarn.lock | 73 +++++++++++++++++++++++++++++++++++-------
 1 file changed, 62 insertions(+), 11 deletions(-)

diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock
index 5698029bff70a..d06dbcbec6154 100644
--- a/docs-website/yarn.lock
+++ b/docs-website/yarn.lock
@@ -220,7 +220,7 @@
   dependencies:
     tslib "~2.0.1"
 
-"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.5", "@babel/code-frame@^7.8.3":
+"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5", "@babel/code-frame@^7.8.3":
   version "7.22.13"
   resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e"
   integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==
@@ -286,6 +286,16 @@
     "@jridgewell/trace-mapping" "^0.3.17"
     jsesc "^2.5.1"
 
+"@babel/generator@^7.23.0":
+  version "7.23.0"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420"
+  integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==
+  dependencies:
+    "@babel/types" "^7.23.0"
+    "@jridgewell/gen-mapping" "^0.3.2"
+    "@jridgewell/trace-mapping" "^0.3.17"
+    jsesc "^2.5.1"
+
 "@babel/helper-annotate-as-pure@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz#e7f06737b197d580a01edf75d97e2c8be99d3882"
@@ -346,6 +356,11 @@
     lodash.debounce "^4.0.8"
     resolve "^1.14.2"
 
+"@babel/helper-environment-visitor@^7.22.20":
+  version "7.22.20"
+  resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167"
+  integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==
+
 "@babel/helper-environment-visitor@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98"
@@ -359,6 +374,14 @@
     "@babel/template" "^7.22.5"
     "@babel/types" "^7.22.5"
 
+"@babel/helper-function-name@^7.23.0":
+  version "7.23.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759"
+  integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==
+  dependencies:
+    "@babel/template" "^7.22.15"
+    "@babel/types" "^7.23.0"
+
 "@babel/helper-hoist-variables@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb"
@@ -452,6 +475,11 @@
   resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f"
   integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==
 
+"@babel/helper-validator-identifier@^7.22.20":
+  version "7.22.20"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0"
+  integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==
+
 "@babel/helper-validator-identifier@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193"
@@ -494,6 +522,11 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.14.tgz#c7de58e8de106e88efca42ce17f0033209dfd245"
   integrity sha512-1KucTHgOvaw/LzCVrEOAyXkr9rQlp0A1HiHRYnSUE9dmb8PvPW7o5sscg+5169r54n3vGlbx6GevTE/Iw/P3AQ==
 
+"@babel/parser@^7.22.15", "@babel/parser@^7.23.0":
+  version "7.23.0"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719"
+  integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==
+
 "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.22.5.tgz#87245a21cd69a73b0b81bcda98d443d6df08f05e"
@@ -1269,19 +1302,28 @@
     "@babel/parser" "^7.22.5"
     "@babel/types" "^7.22.5"
 
-"@babel/traverse@^7.12.9", "@babel/traverse@^7.18.8", "@babel/traverse@^7.22.11":
-  version "7.22.11"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c"
-  integrity sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==
+"@babel/template@^7.22.15":
+  version "7.22.15"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38"
+  integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==
   dependencies:
-    "@babel/code-frame" "^7.22.10"
-    "@babel/generator" "^7.22.10"
-    "@babel/helper-environment-visitor" "^7.22.5"
-    "@babel/helper-function-name" "^7.22.5"
+    "@babel/code-frame" "^7.22.13"
+    "@babel/parser" "^7.22.15"
+    "@babel/types" "^7.22.15"
+
+"@babel/traverse@^7.12.9", "@babel/traverse@^7.18.8", "@babel/traverse@^7.22.11":
+  version "7.23.2"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8"
+  integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==
+  dependencies:
+    "@babel/code-frame" "^7.22.13"
+    "@babel/generator" "^7.23.0"
+    "@babel/helper-environment-visitor" "^7.22.20"
+    "@babel/helper-function-name" "^7.23.0"
     "@babel/helper-hoist-variables" "^7.22.5"
     "@babel/helper-split-export-declaration" "^7.22.6"
-    "@babel/parser" "^7.22.11"
-    "@babel/types" "^7.22.11"
+    "@babel/parser" "^7.23.0"
+    "@babel/types" "^7.23.0"
     debug "^4.1.0"
     globals "^11.1.0"
 
@@ -1294,6 +1336,15 @@
     "@babel/helper-validator-identifier" "^7.22.5"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.22.15", "@babel/types@^7.23.0":
+  version "7.23.0"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb"
+  integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==
+  dependencies:
+    "@babel/helper-string-parser" "^7.22.5"
+    "@babel/helper-validator-identifier" "^7.22.20"
+    to-fast-properties "^2.0.0"
+
 "@colors/colors@1.5.0":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"

From 38adff869356d072f0eef080f6dc7ebaa007dc4c Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Tue, 21 Nov 2023 15:14:52 -0600
Subject: [PATCH 237/259] fix(gha): fix gha for single tag (#9283)

---
 .../actions/docker-custom-build-and-push/action.yml   | 11 ++++++++++-
 .github/workflows/docker-unified.yml                  |  2 ++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml
index bd6bb842b1fb8..ca0796180cd57 100644
--- a/.github/actions/docker-custom-build-and-push/action.yml
+++ b/.github/actions/docker-custom-build-and-push/action.yml
@@ -70,11 +70,20 @@ runs:
         push: false
         cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
         cache-to: type=inline
+    - name: Single Tag
+      if: ${{ inputs.publish != 'true' }}
+      shell: bash
+      run: |
+        TAGS="""
+        ${{ steps.docker_meta.outputs.tags }}
+        """
+        echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
+      id: single_tag
     - name: Upload image locally for testing (if not publishing)
       uses: ishworkh/docker-image-artifact-upload@v1
       if: ${{ inputs.publish != 'true' }}
       with:
-        image: ${{ steps.docker_meta.outputs.tags }}
+        image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index 35cd5363293f8..8bb82a0a0608c 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -876,11 +876,13 @@ jobs:
       ]
     steps:
       - uses: aws-actions/configure-aws-credentials@v1
+        if: ${{ needs.setup.outputs.publish != 'false' }}
         with:
           aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }}
           aws-region: us-west-2
       - uses: isbang/sqs-action@v0.2.0
+        if: ${{ needs.setup.outputs.publish != 'false' }}
         with:
           sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }}
           message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}'

From 15e68bb77199ba54b77df8ffa0df13b7c1fe16ad Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Tue, 21 Nov 2023 16:23:21 -0600
Subject: [PATCH 238/259] fix(node): fix node_options (#9281)

---
 datahub-web-react/build.gradle | 2 +-
 datahub-web-react/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle
index 13eabe90ee509..fd36e5ac4bc2c 100644
--- a/datahub-web-react/build.gradle
+++ b/datahub-web-react/build.gradle
@@ -73,7 +73,7 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnTest, yarnLint]) {
 }
 
 task yarnQuickBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
-  environment = [NODE_OPTIONS: "--max-old-space-size=3072"]
+  environment = [NODE_OPTIONS: "--max-old-space-size=3072 --openssl-legacy-provider"]
   args = ['run', 'build']
 }
 
diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index 5afbc8fa5892d..a72d9c0a898f8 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -95,7 +95,7 @@
         "start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start",
         "start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start",
         "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start",
-        "build": "yarn run generate && NODE_OPTIONS=--openssl-legacy-provider CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/",
+        "build": "yarn run generate && NODE_OPTIONS='--max-old-space-size=3072 --openssl-legacy-provider' CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/yarn/ dist/ && rm -r build/yarn/",
         "test": "craco test",
         "pretest:e2e:ci": "yarn generate",
         "test:e2e": "start-server-and-test start:e2e 3010",

From fd129c7d5d7efd205a0d1ddf3544a9367fb0fe80 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Wed, 22 Nov 2023 19:00:34 +0900
Subject: [PATCH 239/259] fix: Revamp features page (#8839)

---
 .../FeatureCard/featurecard.module.scss       |  38 ++++++
 .../docs/_components/FeatureCard/index.jsx    |  21 +++
 .../featurecardsection.module.scss            |  36 +++++
 .../_components/FeatureCardSection/index.jsx  |  65 +++++++++
 .../docs/_components/QuickstartCard/index.jsx |  22 +++
 .../QuickstartCard/quickstartcard.module.scss |  48 +++++++
 .../_components/QuickstartCards/index.jsx     |  35 +++++
 .../quickstartcards.module.scss               |  34 +++++
 docs/features.md                              | 127 ++++--------------
 9 files changed, 324 insertions(+), 102 deletions(-)
 create mode 100644 docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss
 create mode 100644 docs-website/src/pages/docs/_components/FeatureCard/index.jsx
 create mode 100644 docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss
 create mode 100644 docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx
 create mode 100644 docs-website/src/pages/docs/_components/QuickstartCard/index.jsx
 create mode 100644 docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss
 create mode 100644 docs-website/src/pages/docs/_components/QuickstartCards/index.jsx
 create mode 100644 docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss

diff --git a/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss b/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss
new file mode 100644
index 0000000000000..61739d5b6922c
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss
@@ -0,0 +1,38 @@
+.feature {
+  flex-direction: row;
+  padding: 1.75rem;
+  color: var(--ifm-hero-text-color);
+  margin: 0rem 2rem 1rem 0rem;
+  min-height: 14rem;
+  max-height: 15rem; 
+  overflow: hidden;
+  text-decoration: none !important;
+
+  img {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  svg {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  strong,
+  span {
+    display: block;
+    margin-bottom: 0.75rem;
+  }
+  strong {
+    font-weight: 600;
+    font-size: 1.1rem;
+  }
+
+  span {
+    font-size: 1rem;
+    line-height: 1.25em;
+  }
+  &:hover {
+    border-color: var(--ifm-color-primary);
+  }
+}
diff --git a/docs-website/src/pages/docs/_components/FeatureCard/index.jsx b/docs-website/src/pages/docs/_components/FeatureCard/index.jsx
new file mode 100644
index 0000000000000..407e8eb401987
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/FeatureCard/index.jsx
@@ -0,0 +1,21 @@
+import React from "react";
+import clsx from "clsx";
+import styles from "./featurecard.module.scss";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+import Link from "@docusaurus/Link";
+
+const FeatureCard = ({icon, title, description, to}) => {
+return (
+    <div className="col col--4">
+      <Link to={useBaseUrl(to)} className={clsx("card", styles.feature)}>
+        <div> 
+          {icon}
+          <strong>{title}&nbsp;→</strong>
+          <span>{description}</span>
+        </div>
+      </Link>
+    </div>
+  );
+};
+
+export default FeatureCard;
diff --git a/docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss b/docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss
new file mode 100644
index 0000000000000..9e08c789c9068
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss
@@ -0,0 +1,36 @@
+
+
+.feature {
+  flex-direction: row;
+  padding: 0.675rem;
+  color: var(--ifm-text-color);
+  margin: 0.5rem;
+  min-height: calc(100% - 1rem);
+  text-decoration: none !important;
+  img {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  svg {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  strong,
+  span {
+    display: block;
+    margin-bottom: 0.25rem;
+  }
+  strong {
+    font-weight: 600;
+  }
+
+  span {
+    font-size: 0.875rem;
+    line-height: 1.25em;
+  }
+  &:hover {
+    border-color: var(--ifm-color-primary);
+  }
+}
diff --git a/docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx b/docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx
new file mode 100644
index 0000000000000..bac97a805d2d7
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx
@@ -0,0 +1,65 @@
+import React from "react";
+import FeatureCard from '../FeatureCard'
+import {
+  EyeTwoTone,
+  HeartTwoTone,
+  ApiTwoTone,
+  AlertTwoTone,
+  CompassTwoTone,
+  ProfileTwoTone,
+} from "@ant-design/icons";
+
+const featureCardContent =  [
+{
+    title: "Data Discovery",
+    description: "Search your entire data ecosystem, including dashboards, datasets, ML models, and raw files.",
+    to: "docs/how/search",
+    icon: <EyeTwoTone />
+  },
+{
+    title: "Data Governance",
+    description: "Define ownership and track PII.",
+    to: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2?utm_source=datahub&utm_medium=referral&utm_content=blog",
+    icon: <CompassTwoTone />
+  },
+{
+    title: "Data Quality Control",
+    description: "Improve data quality through metadata tests, assertions, data freshness checks, and data contracts.",
+    to: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog",
+    icon: <AlertTwoTone />
+  },
+{
+  title: "UI-based Ingestion",
+  description: "Easily set up integrations in minutes using DataHub's intuitive UI-based ingestion feature.",
+  to: "docs/ui-ingestion",
+  icon: <ProfileTwoTone />
+},
+{
+  title: "APIs and SDKs",
+  description: "For users who prefer programmatic control, DataHub offers a comprehensive set of APIs and SDKs.",
+  to: "docs/api/datahub-apis",
+  icon: <ApiTwoTone />
+},
+{
+  title: "Vibrant Community",
+  description: "Our community provides support through office hours, workshops, and a Slack channel.",
+  to: "docs/slack",
+  icon: <HeartTwoTone />
+}
+]
+
+const FeatureCards = () => {
+return (
+    <div style={{ padding: "2vh 0" }}>
+      <div className="container" style={{ padding: "0"}}>
+        <div className="row row--no-gutters">
+          {featureCardContent.map((props, idx) => (
+            <FeatureCard key={idx} {...props} />
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default FeatureCards;
diff --git a/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx b/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx
new file mode 100644
index 0000000000000..b4e3895fa40e7
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx
@@ -0,0 +1,22 @@
+import React from "react";
+import clsx from "clsx";
+import styles from "./quickstartcard.module.scss";
+import Link from "@docusaurus/Link";
+import useBaseUrl from "@docusaurus/useBaseUrl";
+
+
+const QuickstartCard = ({ icon, title, to, color, fontColor }) => {
+  return (
+    <div className="col col--6">
+      <Link to={to} className={clsx("card", styles.feature)} style={{ background: color, color: fontColor}}>
+        <img src={useBaseUrl(`/img/${icon}.svg`)} />
+        <div style={{ margin: "auto 0"}}>
+          <strong>{title}&nbsp;→</strong>
+        </div>
+      </Link>
+    </div>
+  );
+};
+
+
+export default QuickstartCard;
diff --git a/docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss b/docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss
new file mode 100644
index 0000000000000..fd35a4b777c99
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss
@@ -0,0 +1,48 @@
+.feature {
+  flex-direction: row;
+  height: 10rem;
+  flex-shrink: 0;
+  padding: 3rem;
+  color: var(--ifm-text-color);
+  margin: 0rem 2rem 1rem 0rem;
+  min-height: calc(100% - 1rem);
+  text-decoration: none !important;
+
+
+  img {
+    width: 3rem;
+    height: 3rem;
+    margin: auto 1rem;
+  }
+  svg {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  strong,
+  span {
+    display: block;
+    margin-bottom: 0.25rem;
+  }
+  strong {
+    font-weight: 600;
+    padding: auto 0;
+  }
+
+  span {
+    font-size: 0.875rem;
+    line-height: 1.25em;
+  }
+  &:hover {
+    border-color: var(--ifm-color-primary);
+  }
+
+  .quickstart-text {
+    margin: auto 0;
+  }
+  
+}
+
+.quickstart-text {
+  margin: auto 0;
+}
\ No newline at end of file
diff --git a/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx b/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx
new file mode 100644
index 0000000000000..bcb77c043f1d0
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx
@@ -0,0 +1,35 @@
+import React from "react";
+import QuickstartCard from '../QuickstartCard'
+
+const quickstartContent =  [
+{
+    title: "Quickstart with DataHub",
+    icon: "datahub-logo-color-mark",
+    to: "quickstart",
+    color: '#FFF',
+    fontColor: '#091013',
+  },
+{
+    title: "Learn about Managed DataHub",
+    icon: "acryl-logo-transparent-mark",
+    to: "managed-datahub/managed-datahub-overview",
+    color: '#091013',
+    fontColor: '#FFF',
+}
+]
+
+const QuickstartCards = () => {
+return (
+    <div style={{ padding: "2vh 0" }}>
+      <div className="container" style={{ padding: "0"}}>
+        <div className="row row--no-gutters">
+          {quickstartContent.map((props, idx) => (
+            <QuickstartCard key={idx} {...props} />
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default QuickstartCards;
diff --git a/docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss b/docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss
new file mode 100644
index 0000000000000..4fbbc4583d662
--- /dev/null
+++ b/docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss
@@ -0,0 +1,34 @@
+.feature {
+  flex-direction: row;
+  padding: 0.675rem;
+  color: var(--ifm-text-color);
+  margin: 0.5rem;
+  min-height: calc(100% - 1rem);
+  text-decoration: none !important;
+  img {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  svg {
+    width: 1.5rem;
+    height: 1.5rem;
+    margin-right: 0.75rem;
+  }
+  strong,
+  span {
+    display: block;
+    margin-bottom: 0.25rem;
+  }
+  strong {
+    font-weight: 600;
+  }
+
+  span {
+    font-size: 0.875rem;
+    line-height: 1.25em;
+  }
+  &:hover {
+    border-color: var(--ifm-color-primary);
+  }
+}
diff --git a/docs/features.md b/docs/features.md
index a06789f28f82b..9ce85d83ee54a 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -1,118 +1,41 @@
----
-title: "Features"
----
+import QuickstartCards from '@site/src/pages/docs/_components/QuickstartCards';
+import FeatureCardSection from '@site/src/pages/docs/_components/FeatureCardSection';
 
-# DataHub Features Overview
+# What is DataHub?
 
-DataHub is a modern data catalog built to enable end-to-end data discovery, data observability, and data governance. This extensible metadata platform is built for developers to tame the complexity of their rapidly evolving data ecosystems and for data practitioners to leverage the total value of data within their organization.
+DataHub is a modern data catalog built to enable end-to-end data discovery, data observability, and data governance.
+This extensible metadata platform is built for developers to tame the complexity of their rapidly evolving data ecosystems and for data practitioners to leverage the total value of data within their organization.
 
-Here’s an overview of DataHub’s current functionality. Check out our [roadmap](https://feature-requests.datahubproject.io/roadmap) to see what's to come.
+## Quickstart
 
----
+<QuickstartCards/>
 
-## Search and Discovery
+## Key Features
 
-### **Search All Corners of Your Data Stack**
+<FeatureCardSection/>
 
-DataHub's unified search experience surfaces results across databases, data lakes, BI platforms, ML feature stores, orchestration tools, and more.
+## Get Started
 
-<p align="center">
-  <img width="70%"  src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-search-all-corners-of-your-datastack.gif"/>
-</p>
+### Deployment
 
-### **Trace End-to-End Lineage**
+To get started with DataHub, you can use a simple CLI command. Alternatively, you can deploy the instance to production using Docker or Helm charts.
 
-Quickly understand the end-to-end journey of data by tracing lineage across platforms, datasets, ETL/ELT pipelines, charts, dashboards, and beyond.
+- [Quickstart](quickstart.md)
+- [Self-hosted DataHub](deploy/kubernetes.md)
+- [Managed DataHub](managed-datahub/managed-datahub-overview.md)
 
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-end-to-end-lineage.png"/>
-</p>
+### Ingestion
 
-### **Understand the Impact of Breaking Changes on Downstream Dependencies**
+DataHub supports ingestion by UI and CLI.
 
-Proactively identify which entities may be impacted by a breaking change using Impact Analysis.
+- [UI-based Ingestion](ui-ingestion.md)
+- [CLI-based Ingestion](../metadata-ingestion/cli-ingestion.md)
 
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-impact-analysis.gif"/>
-</p>
+## Join the Community
 
-### **View Metadata 360 at a Glance**
+For additional information and assistance, feel free to visit one of these channels!
 
-Combine *technical* and *logical* metadata to provide a 360º view of your data entities.
-
-Generate **Dataset Stats** to understand the shape & distribution of the data
-
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-dataset-stats.png"/>
-</p>
-
-Capture historical **Data Validation Outcomes** from tools like Great Expectations
-
-<p align="center">
-<iframe width="560" height="315" src="https://www.youtube.com/embed/44Pr_55Qkik" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-</p>
-
-Leverage DataHub's **Schema Version History** to track changes to the physical structure of data over time
-
-<p align="center">
-<iframe width="560" height="315" src="https://www.youtube.com/embed/IYaV7r5HjZY" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-</p>
-
----
-
-## Modern Data Governance
-
-### **Govern in Real Time**
-
-[The Actions Framework](./actions/README.md) powers the following real-time use cases:
-
-* **Notifications:** Generate organization-specific notifications when a change is made on DataHub. For example, send an email to the governance team when a "PII" tag is added to any data asset.
-* **Workflow Integration:** Integrate DataHub into your organization's internal workflows. For example, create a Jira ticket when specific Tags or Terms are proposed on a Dataset.
-* **Synchronization:** Sync changes made in DataHub into a 3rd party system. For example, reflect Tag additions in DataHub into Snowflake.
-* **Auditing:** Audit who is making what changes on DataHub through time.
-
-<p align="center">
-    <iframe width="560" height="315" src="https://www.youtube.com/embed/yeloymkK5ow" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-</p>
-
-### **Manage Entity Ownership**
-Quickly and easily assign entity ownership to users and user groups.
-
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-entity-owner.png"/>
-</p>
-
-### **Govern with Tags, Glossary Terms, and Domains**
-Empower data owners to govern their data entities with:
-
-1. **Tags:** Informal, loosely controlled labels that serve as a tool for search & discovery. No formal, central management.
-2. **Glossary Terms:** A controlled vocabulary with optional hierarchy, commonly used to describe core business concepts and measurements.
-3. **Domains:** Curated, top-level folders or categories, widely used in Data Mesh to organize entities by department (i.e., Finance, Marketing) or Data Products.
-
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-tags-terms-domains.png"/>
-</p>
-
----
-## DataHub Administration
-
-### **Create Users, Groups, & Access Policies**
-
-DataHub admins can create Policies to define who can perform what action against which resource(s). When you create a new Policy, you will be able to define the following:
-
-* **Policy Type** - Platform (top-level DataHub Platform privileges, i.e., managing users, groups, and policies) or Metadata (ability to manipulate ownership, tags, documentation, and more)
-* **Resource Type** - Specify the type of resources, such as Datasets, Dashboards, Pipelines, and beyond
-* **Privileges** - Choose the set of permissions, such as Edit Owners, Edit Documentation, Edit Links
-* **Users and/or Groups** - Assign relevant Users and Groups; you can also assign the Policy to Resource Owners, regardless of which Group they belong
-
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-manage-policies.png"/>
-</p>
-
-### **Ingest Metadata from the UI**
-
-Create, configure, schedule, & execute batch metadata ingestion using the DataHub user interface. This makes getting metadata into DataHub easier by minimizing the overhead required to operate custom integration pipelines.
-
-<p align="center">
-  <img width="70%" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/feature-managed-ingestion-config.png"/>
-</p>
\ No newline at end of file
+- [Slack](https://datahubspace.slack.com)
+- [Blog](https://blog.datahubproject.io/)
+- [LinkedIn](https://www.linkedin.com/company/acryl-data/)
+- Our champions - [Data Practitioners Guild](https://datahubproject.io/guild/?_gl=1*11cd6n0*_gcl_au*ODQyMTk0NTI5LjE2OTQ2NjI1MjM.)

From cda980bfb279a70ba85786e4dae8eadffbb2cde9 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 22 Nov 2023 21:58:44 +0530
Subject: [PATCH 240/259] docs(acryl cloud): release notes 0.2.13 (#9291)

---
 docs-website/sidebars.js                      |  1 +
 .../managed-datahub/release-notes/v_0_2_13.md | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 docs/managed-datahub/release-notes/v_0_2_13.md

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index be12aa3a827f2..c70a609a4cc4b 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -630,6 +630,7 @@ module.exports = {
         },
         {
           "Managed DataHub Release History": [
+            "docs/managed-datahub/release-notes/v_0_2_13",
             "docs/managed-datahub/release-notes/v_0_2_12",
             "docs/managed-datahub/release-notes/v_0_2_11",
             "docs/managed-datahub/release-notes/v_0_2_10",
diff --git a/docs/managed-datahub/release-notes/v_0_2_13.md b/docs/managed-datahub/release-notes/v_0_2_13.md
new file mode 100644
index 0000000000000..65cea863d9714
--- /dev/null
+++ b/docs/managed-datahub/release-notes/v_0_2_13.md
@@ -0,0 +1,19 @@
+# v0.2.13
+---
+
+Release Availability Date
+---
+22-Nov-2023
+
+Recommended CLI/SDK
+---
+- `v0.12.0.2` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.12.0.2=
+
+If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better.
+
+## Release Changelog
+---
+- Since `v0.2.12` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/75252a3d9f6a576904be5a0790d644b9ae2df6ac...d9de854d276c118afc55264ecc9e2712b91b4ab2 have been pulled in.
+
+## Some notable features in this SaaS release
+- Data Contract support added. This is currently disabled by default. If you wish to use this feature please reach out to your rep.

From 308de491200b01caa7a3bd314bf5b2086af72138 Mon Sep 17 00:00:00 2001
From: gaurav2733 <77378510+gaurav2733@users.noreply.github.com>
Date: Thu, 23 Nov 2023 00:55:32 +0530
Subject: [PATCH 241/259] fix(): stats are spaced out too far (#9292)

---
 .../src/app/entity/dashboard/shared/DashboardStatsSummary.tsx    | 1 -
 .../src/app/entity/dataset/shared/DatasetStatsSummary.tsx        | 1 -
 2 files changed, 2 deletions(-)

diff --git a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx
index fb6364cffac8b..a359d658d27f7 100644
--- a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx
+++ b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx
@@ -12,7 +12,6 @@ import ExpandingStat from '../../dataset/shared/ExpandingStat';
 const StatText = styled.span`
     color: ${ANTD_GRAY[8]};
     @media (min-width: 1024px) {
-        width: 100%;
         white-space: nowrap;
 `;
 
diff --git a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
index 3dcd41a3f8a41..1a5c01df5bde2 100644
--- a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
+++ b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
@@ -13,7 +13,6 @@ import ExpandingStat from './ExpandingStat';
 const StatText = styled.span<{ color: string }>`
     color: ${(props) => props.color};
     @media (min-width: 1160px) {
-        width: 100%;
         white-space: nowrap;
 `;
 

From a62d52e6b19599e4a4e0a0a9c4fbaf578933ff35 Mon Sep 17 00:00:00 2001
From: RyanHolstien <RyanHolstien@users.noreply.github.com>
Date: Wed, 22 Nov 2023 13:54:12 -0600
Subject: [PATCH 242/259] feat(mysql): upgrade to version 8.2 for quickstart
 (#9241)

Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
---
 .../docker-compose-without-neo4j.override.yml |  2 +-
 docker/docker-compose.override.yml            |  2 +-
 docker/mysql/docker-compose.mysql.yml         |  2 +-
 ...ocker-compose-without-neo4j.quickstart.yml |  2 +-
 .../quickstart/docker-compose.quickstart.yml  |  2 +-
 .../quickstart_version_mapping.yaml           |  9 ++++++
 docs/how/extract-container-logs.md            |  2 +-
 docs/how/updating-datahub.md                  |  1 +
 docs/troubleshooting/quickstart.md            |  2 +-
 .../src/datahub/cli/docker_cli.py             |  4 +++
 .../src/datahub/cli/quickstart_versioning.py  | 14 +++++---
 .../cli/test_quickstart_version_mapping.py    | 32 +++++++++++++++----
 12 files changed, 57 insertions(+), 17 deletions(-)

diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml
index 24f83301351ba..36f3c974b93af 100644
--- a/docker/docker-compose-without-neo4j.override.yml
+++ b/docker/docker-compose-without-neo4j.override.yml
@@ -48,7 +48,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
       - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml
index 0907f47d70c3c..ef13b86a3d151 100644
--- a/docker/docker-compose.override.yml
+++ b/docker/docker-compose.override.yml
@@ -32,7 +32,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
diff --git a/docker/mysql/docker-compose.mysql.yml b/docker/mysql/docker-compose.mysql.yml
index 70287433bb541..853d0c425ea61 100644
--- a/docker/mysql/docker-compose.mysql.yml
+++ b/docker/mysql/docker-compose.mysql.yml
@@ -5,7 +5,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     env_file: env/docker.env
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
     ports:
diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
index ab5182bf98ae5..6eac53229e82a 100644
--- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -218,7 +218,7 @@ services:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
       timeout: 5s
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     restart: on-failure
diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
index 29c980532d46f..86d70abd2b815 100644
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@@ -225,7 +225,7 @@ services:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
       timeout: 5s
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     restart: on-failure
diff --git a/docker/quickstart/quickstart_version_mapping.yaml b/docker/quickstart/quickstart_version_mapping.yaml
index 824e1f8237453..9948bd55fdc0b 100644
--- a/docker/quickstart/quickstart_version_mapping.yaml
+++ b/docker/quickstart/quickstart_version_mapping.yaml
@@ -23,6 +23,7 @@ quickstart_version_map:
   default:
     composefile_git_ref: master
     docker_tag: head
+    mysql_tag: 5.7
   # default:  # Use this to pin default to a specific version.
   #   composefile_git_ref: fd1bd51541a132017a648f4a2f037eec8f70ba26 # v0.10.0 + quickstart compose file fixes
   #   docker_tag: v0.10.0
@@ -30,11 +31,19 @@ quickstart_version_map:
   head:
     composefile_git_ref: master
     docker_tag: head
+    mysql_tag: 5.7
+
+  # v0.13.0 we upgraded MySQL image for EOL
+  v0.13.0:
+    composefile_git_ref: master
+    docker_tag: head
+    mysql_tag: 8.2
 
   # v0.9.6 images contain security vulnerabilities
   v0.9.6:
     composefile_git_ref: v0.9.6.1
     docker_tag: v0.9.6.1
+    mysql_tag: 5.7
 
   # If stable is not defined the latest released version will be used.
   # stable:
diff --git a/docs/how/extract-container-logs.md b/docs/how/extract-container-logs.md
index b93c077eb21f5..9251d0665c02c 100644
--- a/docs/how/extract-container-logs.md
+++ b/docs/how/extract-container-logs.md
@@ -21,7 +21,7 @@ CONTAINER ID   IMAGE                                   COMMAND
 3680fcaef3ed   confluentinc/cp-kafka:5.4.0             "/etc/confluent/dock…"   5 days ago   Up 5 days               0.0.0.0:9092->9092/tcp, 0.0.0.0:29092->29092/tcp           broker
 9d6730ddd4c4   neo4j:4.0.6                             "/sbin/tini -g -- /d…"   5 days ago   Up 5 days               0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp   neo4j
 c97edec663af   confluentinc/cp-zookeeper:5.4.0         "/etc/confluent/dock…"   5 days ago   Up 5 days               2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp                 zookeeper
-150ba161cf26   mysql:5.7                               "docker-entrypoint.s…"   5 days ago   Up 5 days               0.0.0.0:3306->3306/tcp, 33060/tcp                          mysql
+150ba161cf26   mysql:8.2                               "docker-entrypoint.s…"   5 days ago   Up 5 days               0.0.0.0:3306->3306/tcp, 33060/tcp                          mysql
 4b72a3eab73f   elasticsearch:7.9.3                     "/tini -- /usr/local…"   5 days ago   Up 5 days (healthy)     0.0.0.0:9200->9200/tcp, 9300/tcp                           elasticsearch
 ```
 
diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
index 54f5775d8331f..21c4cef2e848b 100644
--- a/docs/how/updating-datahub.md
+++ b/docs/how/updating-datahub.md
@@ -6,6 +6,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
+- Updating MySQL version for quickstarts to 8.2, may cause quickstart issues for existing instances.
 - #9244: The `redshift-legacy` and `redshift-legacy-usage` sources, which have been deprecated for >6 months, have been removed. The new `redshift` source is a superset of the functionality provided by those legacy sources.
 
 ### Potential Downtime
diff --git a/docs/troubleshooting/quickstart.md b/docs/troubleshooting/quickstart.md
index 64684c5ffa36c..0392ffc426a6c 100644
--- a/docs/troubleshooting/quickstart.md
+++ b/docs/troubleshooting/quickstart.md
@@ -99,7 +99,7 @@ c267c287a235        landoop/schema-registry-ui:latest                     "/run.
 943e60f9b4d0        neo4j:4.0.6                                           "/sbin/tini -g -- /d…"   10 hours ago        Up 10 hours         0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp   neo4j
 6d79b6f02735        confluentinc/cp-zookeeper:5.2.1                       "/etc/confluent/dock…"   10 hours ago        Up 10 hours         2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp                 zookeeper
 491d9f2b2e9e        docker.elastic.co/elasticsearch/elasticsearch:5.6.8   "/bin/bash bin/es-do…"   10 hours ago        Up 10 hours         0.0.0.0:9200->9200/tcp, 9300/tcp                           elasticsearch
-ce14b9758eb3        mysql:5.7
+ce14b9758eb3        mysql:8.2
 ```
 
 Also you can check individual Docker container logs by running `docker logs <<container_name>>`. For `datahub-gms`, you should see a log similar to this at the end of the initialization:
diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py
index 08f3faae8abb2..0e0bc37c61573 100644
--- a/metadata-ingestion/src/datahub/cli/docker_cli.py
+++ b/metadata-ingestion/src/datahub/cli/docker_cli.py
@@ -158,6 +158,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) ->
 
 def _set_environment_variables(
     version: Optional[str],
+    mysql_version: Optional[str],
     mysql_port: Optional[pydantic.PositiveInt],
     zk_port: Optional[pydantic.PositiveInt],
     kafka_broker_port: Optional[pydantic.PositiveInt],
@@ -172,6 +173,8 @@ def _set_environment_variables(
             )
             version = f"v{version}"
         os.environ["DATAHUB_VERSION"] = version
+    if mysql_version is not None:
+        os.environ["DATAHUB_MYSQL_VERSION"] = mysql_version
     if mysql_port is not None:
         os.environ["DATAHUB_MAPPED_MYSQL_PORT"] = str(mysql_port)
 
@@ -675,6 +678,7 @@ def quickstart(  # noqa: C901
     # set version
     _set_environment_variables(
         version=quickstart_execution_plan.docker_tag,
+        mysql_version=quickstart_execution_plan.mysql_tag,
         mysql_port=mysql_port,
         zk_port=zk_port,
         kafka_broker_port=kafka_broker_port,
diff --git a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py
index dc7bd1ecb4a89..be7439f330dfb 100644
--- a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py
+++ b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py
@@ -21,6 +21,7 @@
 class QuickstartExecutionPlan(BaseModel):
     composefile_git_ref: str
     docker_tag: str
+    mysql_tag: Optional[str]
 
 
 def _is_it_a_version(version: str) -> bool:
@@ -81,7 +82,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
             return QuickstartVersionMappingConfig(
                 quickstart_version_map={
                     "default": QuickstartExecutionPlan(
-                        composefile_git_ref="master", docker_tag="head"
+                        composefile_git_ref="master", docker_tag="head", mysql_tag="5.7"
                     ),
                 }
             )
@@ -93,7 +94,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
             try:
                 release = cls._fetch_latest_version()
                 config.quickstart_version_map["stable"] = QuickstartExecutionPlan(
-                    composefile_git_ref=release, docker_tag=release
+                    composefile_git_ref=release, docker_tag=release, mysql_tag=release
                 )
             except Exception:
                 click.echo(
@@ -103,7 +104,8 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
         return config
 
     def get_quickstart_execution_plan(
-        self, requested_version: Optional[str]
+        self,
+        requested_version: Optional[str],
     ) -> QuickstartExecutionPlan:
         """
         From the requested version and stable flag, returns the execution plan for the quickstart.
@@ -114,10 +116,14 @@ def get_quickstart_execution_plan(
             requested_version = "default"
         composefile_git_ref = requested_version
         docker_tag = requested_version
+        # Default to 5.7 if not specified in version map
+        mysql_tag = "5.7"
         result = self.quickstart_version_map.get(
             requested_version,
             QuickstartExecutionPlan(
-                composefile_git_ref=composefile_git_ref, docker_tag=docker_tag
+                composefile_git_ref=composefile_git_ref,
+                docker_tag=docker_tag,
+                mysql_tag=mysql_tag,
             ),
         )
         # new CLI version is downloading the composefile corresponding to the requested version
diff --git a/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py b/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py
index 2fd56560c933d..3b06e48522955 100644
--- a/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py
+++ b/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py
@@ -6,16 +6,30 @@
 example_version_mapper = QuickstartVersionMappingConfig.parse_obj(
     {
         "quickstart_version_map": {
-            "default": {"composefile_git_ref": "master", "docker_tag": "latest"},
+            "default": {
+                "composefile_git_ref": "master",
+                "docker_tag": "latest",
+                "mysql_tag": "5.7",
+            },
             "v0.9.6": {
                 "composefile_git_ref": "v0.9.6.1",
                 "docker_tag": "v0.9.6.1",
+                "mysql_tag": "5.7",
+            },
+            "v2.0.0": {
+                "composefile_git_ref": "v2.0.1",
+                "docker_tag": "v2.0.0",
+                "mysql_tag": "5.7",
+            },
+            "v1.0.0": {
+                "composefile_git_ref": "v1.0.0",
+                "docker_tag": "v1.0.0",
+                "mysql_tag": "5.7",
             },
-            "v2.0.0": {"composefile_git_ref": "v2.0.1", "docker_tag": "v2.0.0"},
-            "v1.0.0": {"composefile_git_ref": "v1.0.0", "docker_tag": "v1.0.0"},
             "stable": {
                 "composefile_git_ref": "v1.0.1",
                 "docker_tag": "latest",
+                "mysql_tag": "5.7",
             },
         },
     }
@@ -27,6 +41,7 @@ def test_quickstart_version_config():
     expected = QuickstartExecutionPlan(
         docker_tag="v1.0.0",
         composefile_git_ref="v1.0.0",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -36,6 +51,7 @@ def test_quickstart_version_config_default():
     expected = QuickstartExecutionPlan(
         docker_tag="v2.0.0",
         composefile_git_ref="v2.0.1",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -43,20 +59,22 @@ def test_quickstart_version_config_default():
 def test_quickstart_version_config_stable():
     execution_plan = example_version_mapper.get_quickstart_execution_plan("stable")
     expected = QuickstartExecutionPlan(
-        docker_tag="latest",
-        composefile_git_ref="v1.0.1",
+        docker_tag="latest", composefile_git_ref="v1.0.1", mysql_tag="5.7"
     )
     assert execution_plan == expected
 
 
 def test_quickstart_forced_stable():
     example_version_mapper.quickstart_version_map["default"] = QuickstartExecutionPlan(
-        composefile_git_ref="v1.0.1", docker_tag="latest"
+        composefile_git_ref="v1.0.1",
+        docker_tag="latest",
+        mysql_tag="5.7",
     )
     execution_plan = example_version_mapper.get_quickstart_execution_plan(None)
     expected = QuickstartExecutionPlan(
         docker_tag="latest",
         composefile_git_ref="v1.0.1",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -74,6 +92,7 @@ def test_quickstart_forced_not_a_version_tag():
     expected = QuickstartExecutionPlan(
         docker_tag="NOT A VERSION",
         composefile_git_ref="NOT A VERSION",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -83,5 +102,6 @@ def test_quickstart_get_older_version():
     expected = QuickstartExecutionPlan(
         docker_tag="v0.9.6.1",
         composefile_git_ref="v0.9.6.1",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected

From 37ea292aa619bad2b6796c7cca0f78a683176a13 Mon Sep 17 00:00:00 2001
From: Hyejin Yoon <0327jane@gmail.com>
Date: Thu, 23 Nov 2023 10:32:19 +0900
Subject: [PATCH 243/259] feat: add townhall RSVP link on the main page 
 (#9277)

---
 .../src/pages/_components/Hero/index.js       |  2 ++
 .../_components/TownhallButton/index.jsx      | 31 +++++++++++++++++++
 .../TownhallButton/townhallbutton.module.scss | 14 +++++++++
 3 files changed, 47 insertions(+)
 create mode 100644 docs-website/src/pages/_components/TownhallButton/index.jsx
 create mode 100644 docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss

diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js
index ffa298b27a822..97a04eb21fa73 100644
--- a/docs-website/src/pages/_components/Hero/index.js
+++ b/docs-website/src/pages/_components/Hero/index.js
@@ -8,6 +8,7 @@ import { QuestionCircleOutlined } from "@ant-design/icons";
 import styles from "./hero.module.scss";
 import CodeBlock from "@theme/CodeBlock";
 import CardCTAs from "../CardCTAs";
+import TownhallButton from "../TownhallButton";
 
 const HeroAnnouncement = ({ message, linkUrl, linkText }) => (
   <div className={clsx("hero__alert alert alert--primary", styles.hero__alert)}>
@@ -46,6 +47,7 @@ const Hero = ({}) => {
             <Link className="button button--secondary button--md" to="https://slack.datahubproject.io">
               Join our Slack
             </Link>
+            <TownhallButton />
           </div>
         </div>
         <CardCTAs />
diff --git a/docs-website/src/pages/_components/TownhallButton/index.jsx b/docs-website/src/pages/_components/TownhallButton/index.jsx
new file mode 100644
index 0000000000000..11dc2dc5c8476
--- /dev/null
+++ b/docs-website/src/pages/_components/TownhallButton/index.jsx
@@ -0,0 +1,31 @@
+import React from 'react';
+import styles from "./townhallbutton.module.scss";
+import clsx from "clsx";
+import Link from "@docusaurus/Link";
+
+const TownhallButton = () => {
+  const today = new Date();
+  const currentDay = today.getDate();
+  const lastDayOfMonth = new Date(today.getFullYear(), today.getMonth() + 1, 0);
+  const lastThursday = lastDayOfMonth.getDate() - ((lastDayOfMonth.getDay() + 7 - 4) % 7);
+
+  const daysUntilLastThursday = lastThursday - currentDay;
+
+  let showButton = false;
+  let currentMonth = '';
+
+  if (daysUntilLastThursday > 0 && daysUntilLastThursday <= 14) {
+    showButton = true;
+    currentMonth = new Intl.DateTimeFormat('en-US', { month: 'long' }).format(today);
+  }
+
+  return (
+    showButton && (
+      <Link to="http://rsvp.datahubproject.io" className={clsx('button button--primary button--md', styles.feature)}>
+        Join {currentMonth} Townhall!&nbsp;✨
+      </Link>
+    )
+  );
+};
+
+export default TownhallButton;
diff --git a/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss b/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss
new file mode 100644
index 0000000000000..951bc99015302
--- /dev/null
+++ b/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss
@@ -0,0 +1,14 @@
+.feature {
+  color: white;
+  border: 1px solid transparent;
+  background-image: linear-gradient(to right, #1890ff 0%, #9c27b0 100%);
+  background-origin: border-box;
+  opacity: 90%;
+
+  &:hover {
+    opacity: 100%;
+    background: linear-gradient(to right, #1890ff 0%, #9c27b0 100%);
+    background-image: linear-gradient(to right, #1890ff 0%, #9c27b0 100%);
+    background-origin: border-box;
+  }
+}

From f794a905159c8a299fecdfbc00360a23c5c8e44c Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Thu, 23 Nov 2023 09:45:25 +0100
Subject: [PATCH 244/259] fix(ingest/snowflake): Apply email filter on all
 usage metrics (#9269)

---
 .../source/snowflake/snowflake_query.py       | 38 ++++++++++++++++
 .../source/snowflake/snowflake_usage_v2.py    |  2 +
 .../tests/integration/snowflake/common.py     |  3 ++
 .../tests/unit/test_snowflake_source.py       | 43 +++++++++++++++++++
 4 files changed, 86 insertions(+)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
index 0f89324f5efc6..267f7cf074909 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
@@ -1,5 +1,6 @@
 from typing import List, Optional
 
+from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.time_window_config import BucketDuration
 from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain
 from datahub.ingestion.source.snowflake.snowflake_config import DEFAULT_TABLES_DENY_LIST
@@ -551,6 +552,8 @@ def usage_per_object_per_time_bucket_for_time_window(
         use_base_objects: bool,
         top_n_queries: int,
         include_top_n_queries: bool,
+        email_domain: Optional[str],
+        email_filter: AllowDenyPattern,
     ) -> str:
         if not include_top_n_queries:
             top_n_queries = 0
@@ -561,6 +564,9 @@ def usage_per_object_per_time_bucket_for_time_window(
         objects_column = (
             "BASE_OBJECTS_ACCESSED" if use_base_objects else "DIRECT_OBJECTS_ACCESSED"
         )
+        email_filter_query = SnowflakeQuery.gen_email_filter_query(email_filter)
+
+        email_domain = f"@{email_domain}" if email_domain else ""
 
         return f"""
         WITH object_access_history AS
@@ -578,12 +584,16 @@ def usage_per_object_per_time_bucket_for_time_window(
                         query_id,
                         query_start_time,
                         user_name,
+                        NVL(USERS.email, CONCAT(user_name, '{email_domain}')) AS user_email,
                         {objects_column}
                     from
                         snowflake.account_usage.access_history
+                    LEFT JOIN
+                        snowflake.account_usage.users USERS
                     WHERE
                         query_start_time >= to_timestamp_ltz({start_time_millis}, 3)
                         AND query_start_time < to_timestamp_ltz({end_time_millis}, 3)
+                        {email_filter_query}
                 )
                 t,
                 lateral flatten(input => t.{objects_column}) object
@@ -705,6 +715,34 @@ def usage_per_object_per_time_bucket_for_time_window(
             basic_usage_counts.bucket_start_time
         """
 
+    @staticmethod
+    def gen_email_filter_query(email_filter: AllowDenyPattern) -> str:
+        allow_filters = []
+        allow_filter = ""
+        if len(email_filter.allow) == 1 and email_filter.allow[0] == ".*":
+            allow_filter = ""
+        else:
+            for allow_pattern in email_filter.allow:
+                allow_filters.append(
+                    f"rlike(user_name, '{allow_pattern}','{'i' if email_filter.ignoreCase else 'c'}')"
+                )
+            if allow_filters:
+                allow_filter = " OR ".join(allow_filters)
+                allow_filter = f"AND ({allow_filter})"
+        deny_filters = []
+        deny_filter = ""
+        for deny_pattern in email_filter.deny:
+            deny_filters.append(
+                f"rlike(user_name, '{deny_pattern}','{'i' if email_filter.ignoreCase else 'c'}')"
+            )
+        if deny_filters:
+            deny_filter = " OR ".join(deny_filters)
+            deny_filter = f"({deny_filter})"
+        email_filter_query = allow_filter + (
+            " AND" + f" NOT {deny_filter}" if deny_filter else ""
+        )
+        return email_filter_query
+
     @staticmethod
     def table_upstreams_with_column_lineage(
         start_time_millis: int,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
index 8f571313f1888..f75e994303954 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
@@ -214,6 +214,8 @@ def _get_workunits_internal(
                         use_base_objects=self.config.apply_view_usage_to_tables,
                         top_n_queries=self.config.top_n_queries,
                         include_top_n_queries=self.config.include_top_n_queries,
+                        email_domain=self.config.email_domain,
+                        email_filter=self.config.user_email_pattern,
                     ),
                 )
             except Exception as e:
diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py
index 78e5499697311..b21cea5f0988d 100644
--- a/metadata-ingestion/tests/integration/snowflake/common.py
+++ b/metadata-ingestion/tests/integration/snowflake/common.py
@@ -1,6 +1,7 @@
 import json
 from datetime import datetime, timezone
 
+from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.time_window_config import BucketDuration
 from datahub.ingestion.source.snowflake import snowflake_query
 from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery
@@ -263,6 +264,8 @@ def default_query_results(  # noqa: C901
             top_n_queries=10,
             include_top_n_queries=True,
             time_bucket_size=BucketDuration.DAY,
+            email_domain=None,
+            email_filter=AllowDenyPattern.allow_all(),
         )
     ):
         return []
diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py
index aaff878b81eee..343f4466fd6fd 100644
--- a/metadata-ingestion/tests/unit/test_snowflake_source.py
+++ b/metadata-ingestion/tests/unit/test_snowflake_source.py
@@ -3,6 +3,7 @@
 import pytest
 from pydantic import ValidationError
 
+from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.oauth import OAuthConfiguration
 from datahub.configuration.pattern_utils import UUID_REGEX
 from datahub.ingestion.api.source import SourceCapability
@@ -16,6 +17,7 @@
     SnowflakeV2Config,
 )
 from datahub.ingestion.source.snowflake.snowflake_query import (
+    SnowflakeQuery,
     create_deny_regex_sql_filter,
 )
 from datahub.ingestion.source.snowflake.snowflake_usage_v2 import (
@@ -661,3 +663,44 @@ def test_snowflake_temporary_patterns_config_rename():
         }
     )
     assert conf.temporary_tables_pattern == [".*tmp.*"]
+
+
+def test_email_filter_query_generation_with_one_deny():
+    email_filter = AllowDenyPattern(deny=[".*@example.com"])
+    filter_query = SnowflakeQuery.gen_email_filter_query(email_filter)
+    assert filter_query == " AND NOT (rlike(user_name, '.*@example.com','i'))"
+
+
+def test_email_filter_query_generation_without_any_filter():
+    email_filter = AllowDenyPattern()
+    filter_query = SnowflakeQuery.gen_email_filter_query(email_filter)
+    assert filter_query == ""
+
+
+def test_email_filter_query_generation_one_allow():
+    email_filter = AllowDenyPattern(allow=[".*@example.com"])
+    filter_query = SnowflakeQuery.gen_email_filter_query(email_filter)
+    assert filter_query == "AND (rlike(user_name, '.*@example.com','i'))"
+
+
+def test_email_filter_query_generation_one_allow_and_deny():
+    email_filter = AllowDenyPattern(
+        allow=[".*@example.com", ".*@example2.com"],
+        deny=[".*@example2.com", ".*@example4.com"],
+    )
+    filter_query = SnowflakeQuery.gen_email_filter_query(email_filter)
+    assert (
+        filter_query
+        == "AND (rlike(user_name, '.*@example.com','i') OR rlike(user_name, '.*@example2.com','i')) AND NOT (rlike(user_name, '.*@example2.com','i') OR rlike(user_name, '.*@example4.com','i'))"
+    )
+
+
+def test_email_filter_query_generation_with_case_insensitive_filter():
+    email_filter = AllowDenyPattern(
+        allow=[".*@example.com"], deny=[".*@example2.com"], ignoreCase=False
+    )
+    filter_query = SnowflakeQuery.gen_email_filter_query(email_filter)
+    assert (
+        filter_query
+        == "AND (rlike(user_name, '.*@example.com','c')) AND NOT (rlike(user_name, '.*@example2.com','c'))"
+    )

From 1c5871c1699bd3b3c464b7d576572e06f29649ef Mon Sep 17 00:00:00 2001
From: Simon Osipov <osipov.simon@gmail.com>
Date: Fri, 24 Nov 2023 12:52:35 +0200
Subject: [PATCH 245/259] docs(ingestion): Added mention of host without
 protocol (#9301)

---
 .../src/app/ingest/source/builder/RecipeForm/trino.ts           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/trino.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/trino.ts
index ed3c7ee73b819..1af84c0131d4a 100644
--- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/trino.ts
+++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/trino.ts
@@ -6,7 +6,7 @@ export const TRINO_HOST_PORT: RecipeField = {
     name: 'host_port',
     label: 'Host and Port',
     tooltip:
-        "The host and port where Trino is running. For example, 'trino-server:5432'. Note: this host must be accessible on the network where DataHub is running (or allowed via an IP Allow List, AWS PrivateLink, etc).",
+        "The host (without protocol and ://) and port where Trino is running. For example, 'trino-server:5432'. Note: this host must be accessible on the network where DataHub is running (or allowed via an IP Allow List, AWS PrivateLink, etc).",
     type: FieldType.TEXT,
     fieldPath: 'source.config.host_port',
     placeholder: 'trino-server:5432',

From 5ccb30e8104a948d0440e4bda3ea9807c26c91f8 Mon Sep 17 00:00:00 2001
From: Tamas Nemeth <treff7es@gmail.com>
Date: Fri, 24 Nov 2023 16:31:37 +0100
Subject: [PATCH 246/259] fix(ingest/teradata): Teradata speed up changes
 (#9059)

Co-authored-by: Andrew Sikowitz <andrew.sikowitz@acryl.io>
---
 .../datahub/emitter/sql_parsing_builder.py    |   3 +
 .../datahub/ingestion/source/sql/teradata.py  | 715 +++++++++++++++++-
 2 files changed, 688 insertions(+), 30 deletions(-)

diff --git a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
index cedaa4fbbd7f6..ea5ebf705707a 100644
--- a/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/sql_parsing_builder.py
@@ -195,6 +195,9 @@ def _gen_lineage_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
                 upstreams.append(edge.gen_upstream_aspect())
                 fine_upstreams.extend(edge.gen_fine_grained_lineage_aspects())
 
+            if not upstreams:
+                continue
+
             upstream_lineage = UpstreamLineageClass(
                 upstreams=sorted(upstreams, key=lambda x: x.dataset),
                 fineGrainedLineages=sorted(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index 8aeb1e50cd0b3..4ea8dbe236c53 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -1,14 +1,32 @@
 import logging
+from collections import defaultdict
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Iterable, Optional, Union
+from functools import lru_cache
+from itertools import groupby
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    MutableMapping,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 # This import verifies that the dependencies are available.
 import teradatasqlalchemy  # noqa: F401
 import teradatasqlalchemy.types as custom_types
 from pydantic.fields import Field
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect
 from sqlalchemy.engine import Engine
+from sqlalchemy.engine.base import Connection
+from sqlalchemy.engine.reflection import Inspector
+from sqlalchemy.sql.expression import text
+from teradatasqlalchemy.dialect import TeradataDialect
+from teradatasqlalchemy.options import configure
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.time_window_config import BaseTimeWindowConfig
@@ -22,9 +40,11 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.source_helpers import auto_lowercase_urns
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.sql.sql_common import SqlWorkUnit, register_custom_type
+from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
 from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport
 from datahub.ingestion.source.sql.two_tier_sql_source import (
     TwoTierSQLAlchemyConfig,
@@ -33,6 +53,7 @@
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.ingestion.source_report.time_window import BaseTimeWindowReport
+from datahub.metadata._schema_classes import SchemaMetadataClass
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     BytesTypeClass,
     TimeTypeClass,
@@ -62,6 +83,249 @@
 register_custom_type(custom_types.XML, BytesTypeClass)
 
 
+@dataclass
+class TeradataTable:
+    database: str
+    name: str
+    description: Optional[str]
+    object_type: str
+    create_timestamp: datetime
+    last_alter_name: Optional[str]
+    last_alter_timestamp: Optional[datetime]
+    request_text: Optional[str]
+
+
+# lru cache is set to 1 which work only in single threaded environment but it keeps the memory footprint lower
+@lru_cache(maxsize=1)
+def get_schema_columns(
+    self: Any, connection: Connection, dbc_columns: str, schema: str
+) -> Dict[str, List[Any]]:
+    columns: Dict[str, List[Any]] = {}
+    columns_query = f"select * from dbc.{dbc_columns} where DatabaseName (NOT CASESPECIFIC) = '{schema}' (NOT CASESPECIFIC) order by TableName, ColumnId"
+    rows = connection.execute(text(columns_query)).fetchall()
+    for row in rows:
+        row_mapping = row._mapping
+        if row_mapping.TableName not in columns:
+            columns[row_mapping.TableName] = []
+
+        columns[row_mapping.TableName].append(row_mapping)
+
+    return columns
+
+
+# lru cache is set to 1 which work only in single threaded environment but it keeps the memory footprint lower
+@lru_cache(maxsize=1)
+def get_schema_pk_constraints(
+    self: Any, connection: Connection, schema: str
+) -> Dict[str, List[Any]]:
+    dbc_indices = "IndicesV" + "X" if configure.usexviews else "IndicesV"
+    primary_keys: Dict[str, List[Any]] = {}
+    stmt = f"select * from dbc.{dbc_indices} where DatabaseName (NOT CASESPECIFIC) = '{schema}' (NOT CASESPECIFIC) and IndexType = 'K' order by IndexNumber"
+    rows = connection.execute(text(stmt)).fetchall()
+    for row in rows:
+        row_mapping = row._mapping
+        if row_mapping.TableName not in primary_keys:
+            primary_keys[row_mapping.TableName] = []
+
+        primary_keys[row_mapping.TableName].append(row_mapping)
+
+    return primary_keys
+
+
+def optimized_get_pk_constraint(
+    self: Any,
+    connection: Connection,
+    table_name: str,
+    schema: Optional[str] = None,
+    **kw: Dict[str, Any],
+) -> Dict:
+    """
+    Override
+    TODO: Check if we need PRIMARY Indices or PRIMARY KEY Indices
+    TODO: Check for border cases (No PK Indices)
+    """
+
+    if schema is None:
+        schema = self.default_schema_name
+
+    # Default value for 'usexviews' is False so use dbc.IndicesV by default
+    # dbc_indices = self.__get_xviews_obj("IndicesV")
+
+    # table_obj = table(
+    #    dbc_indices, column("ColumnName"), column("IndexName"), schema="dbc"
+    # )
+
+    res = []
+    pk_keys = self.get_schema_pk_constraints(connection, schema)
+    res = pk_keys.get(table_name, [])
+
+    index_columns = list()
+    index_name = None
+
+    for index_column in res:
+        index_columns.append(self.normalize_name(index_column.ColumnName))
+        index_name = self.normalize_name(
+            index_column.IndexName
+        )  # There should be just one IndexName
+
+    return {"constrained_columns": index_columns, "name": index_name}
+
+
+def optimized_get_columns(
+    self: Any,
+    connection: Connection,
+    table_name: str,
+    schema: Optional[str] = None,
+    tables_cache: MutableMapping[str, List[TeradataTable]] = {},
+    use_qvci: bool = False,
+    **kw: Dict[str, Any],
+) -> List[Dict]:
+    if schema is None:
+        schema = self.default_schema_name
+
+    # Using 'help schema.table.*' statements has been considered.
+    # The DBC.ColumnsV provides the default value which is not available
+    # with the 'help column' commands result.
+
+    td_table: Optional[TeradataTable] = None
+    # Check if the object is a view
+    for t in tables_cache[schema]:
+        if t.name == table_name:
+            td_table = t
+            break
+
+    if td_table is None:
+        logger.warning(
+            f"Table {table_name} not found in cache for schema {schema}, not getting columns"
+        )
+        return []
+
+    res = []
+    if td_table.object_type == "View" and not use_qvci:
+        # Volatile table definition is not stored in the dictionary.
+        # We use the 'help schema.table.*' command instead to get information for all columns.
+        # We have to do the same for views since we need the type information
+        # which is not available in dbc.ColumnsV.
+        res = self._get_column_help(connection, schema, table_name, column_name=None)
+
+        # If this is a view, get types for individual columns (dbc.ColumnsV won't have types for view columns).
+        # For a view or a volatile table, we have to set the default values as the 'help' command does not have it.
+        col_info_list = []
+        for r in res:
+            updated_column_info_dict = self._update_column_help_info(r._mapping)
+            col_info_list.append(dict(r._mapping, **(updated_column_info_dict)))
+        res = col_info_list
+    else:
+        # Default value for 'usexviews' is False so use dbc.ColumnsV by default
+        dbc_columns = "columnsQV" if use_qvci else "columnsV"
+        dbc_columns = dbc_columns + "X" if configure.usexviews else dbc_columns
+        res = self.get_schema_columns(connection, dbc_columns, schema).get(
+            table_name, []
+        )
+
+    final_column_info = []
+    # Don't care about ART tables now
+    # Ignore the non-functional column in a PTI table
+    for row in res:
+        col_info = self._get_column_info(row)
+        if "TSColumnType" in col_info and col_info["TSColumnType"] is not None:
+            if (
+                col_info["ColumnName"] == "TD_TIMEBUCKET"
+                and col_info["TSColumnType"].strip() == "TB"
+            ):
+                continue
+        final_column_info.append(col_info)
+
+    return final_column_info
+
+
+# lru cache is set to 1 which work only in single threaded environment but it keeps the memory footprint lower
+@lru_cache(maxsize=1)
+def get_schema_foreign_keys(
+    self: Any, connection: Connection, schema: str
+) -> Dict[str, List[Any]]:
+    dbc_child_parent_table = (
+        "All_RI_ChildrenV" + "X" if configure.usexviews else "All_RI_ChildrenV"
+    )
+    foreign_keys: Dict[str, List[Any]] = {}
+    stmt = f"""
+    SELECT dbc."All_RI_ChildrenV"."ChildDB",  dbc."All_RI_ChildrenV"."ChildTable", dbc."All_RI_ChildrenV"."IndexID", dbc."{dbc_child_parent_table}"."IndexName", dbc."{dbc_child_parent_table}"."ChildKeyColumn", dbc."{dbc_child_parent_table}"."ParentDB", dbc."{dbc_child_parent_table}"."ParentTable", dbc."{dbc_child_parent_table}"."ParentKeyColumn"
+        FROM dbc."{dbc_child_parent_table}"
+    WHERE ChildDB = '{schema}' ORDER BY "IndexID" ASC
+    """
+    rows = connection.execute(text(stmt)).fetchall()
+    for row in rows:
+        row_mapping = row._mapping
+        if row_mapping.ChildTable not in foreign_keys:
+            foreign_keys[row_mapping.ChildTable] = []
+
+        foreign_keys[row_mapping.ChildTable].append(row_mapping)
+
+    return foreign_keys
+
+
+def optimized_get_foreign_keys(self, connection, table_name, schema=None, **kw):
+    """
+    Overrides base class method
+    """
+
+    if schema is None:
+        schema = self.default_schema_name
+    # Default value for 'usexviews' is False so use DBC.All_RI_ChildrenV by default
+    res = self.get_schema_foreign_keys(connection, schema).get(table_name, [])
+
+    def grouper(fk_row):
+        return {
+            "name": fk_row.IndexName or fk_row.IndexID,  # ID if IndexName is None
+            "schema": fk_row.ParentDB,
+            "table": fk_row.ParentTable,
+        }
+
+    # TODO: Check if there's a better way
+    fk_dicts = list()
+    for constraint_info, constraint_cols in groupby(res, grouper):
+        fk_dict = {
+            "name": str(constraint_info["name"]),
+            "constrained_columns": list(),
+            "referred_table": constraint_info["table"],
+            "referred_schema": constraint_info["schema"],
+            "referred_columns": list(),
+        }
+
+        for constraint_col in constraint_cols:
+            fk_dict["constrained_columns"].append(
+                self.normalize_name(constraint_col.ChildKeyColumn)
+            )
+            fk_dict["referred_columns"].append(
+                self.normalize_name(constraint_col.ParentKeyColumn)
+            )
+
+        fk_dicts.append(fk_dict)
+
+    return fk_dicts
+
+
+def optimized_get_view_definition(
+    self: Any,
+    connection: Connection,
+    view_name: str,
+    schema: Optional[str] = None,
+    tables_cache: MutableMapping[str, List[TeradataTable]] = {},
+    **kw: Dict[str, Any],
+) -> Optional[str]:
+    if schema is None:
+        schema = self.default_schema_name
+
+    if schema not in tables_cache:
+        return None
+
+    for table in tables_cache[schema]:
+        if table.name == view_name:
+            return self.normalize_name(table.request_text)
+
+    return None
+
+
 @dataclass
 class TeradataReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowReport):
     num_queries_parsed: int = 0
@@ -74,8 +338,62 @@ class BaseTeradataConfig(TwoTierSQLAlchemyConfig):
 
 
 class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
+    databases: Optional[List[str]] = Field(
+        default=None,
+        description=(
+            "List of databases to ingest. If not specified, all databases will be ingested."
+            " Even if this is specified, databases will still be filtered by `database_pattern`."
+        ),
+    )
+
     database_pattern = Field(
-        default=AllowDenyPattern(deny=["dbc"]),
+        default=AllowDenyPattern(
+            deny=[
+                "All",
+                "Crashdumps",
+                "Default",
+                "DemoNow_Monitor",
+                "EXTUSER",
+                "External_AP",
+                "GLOBAL_FUNCTIONS",
+                "LockLogShredder",
+                "PUBLIC",
+                "SQLJ",
+                "SYSBAR",
+                "SYSJDBC",
+                "SYSLIB",
+                "SYSSPATIAL",
+                "SYSUDTLIB",
+                "SYSUIF",
+                "SysAdmin",
+                "Sys_Calendar",
+                "SystemFe",
+                "TDBCMgmt",
+                "TDMaps",
+                "TDPUSER",
+                "TDQCD",
+                "TDStats",
+                "TD_ANALYTICS_DB",
+                "TD_SERVER_DB",
+                "TD_SYSFNLIB",
+                "TD_SYSGPL",
+                "TD_SYSXML",
+                "TDaaS_BAR",
+                "TDaaS_DB",
+                "TDaaS_Maint",
+                "TDaaS_Monitor",
+                "TDaaS_Support",
+                "TDaaS_TDBCMgmt1",
+                "TDaaS_TDBCMgmt2",
+                "dbcmngr",
+                "mldb",
+                "system",
+                "tapidb",
+                "tdwm",
+                "val",
+                "dbc",
+            ]
+        ),
         description="Regex patterns for databases to filter in ingestion.",
     )
     include_table_lineage = Field(
@@ -84,6 +402,11 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         "This requires to have the table lineage feature enabled.",
     )
 
+    include_view_lineage = Field(
+        default=True,
+        description="Whether to include view lineage in the ingestion. "
+        "This requires to have the view lineage feature enabled.",
+    )
     usage: BaseUsageConfig = Field(
         description="The usage config to use when generating usage statistics",
         default=BaseUsageConfig(),
@@ -99,6 +422,16 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
         description="Generate usage statistic.",
     )
 
+    use_file_backed_cache: bool = Field(
+        default=True,
+        description="Whether to use a file backed cache for the view definitions.",
+    )
+
+    use_qvci: bool = Field(
+        default=False,
+        description="Whether to use QVCI to get column information. This is faster but requires to have QVCI enabled.",
+    )
+
 
 @platform_name("Teradata")
 @config_class(TeradataConfig)
@@ -122,13 +455,116 @@ class TeradataSource(TwoTierSQLAlchemySource):
 
     config: TeradataConfig
 
-    LINEAGE_QUERY: str = """SELECT ProcID, UserName as "user", StartTime AT TIME ZONE 'GMT' as "timestamp", DefaultDatabase as default_database, QueryText as query
-     FROM "DBC".DBQLogTbl
-     where ErrorCode = 0
-     and QueryText like 'create table demo_user.test_lineage%'
-     and "timestamp" >= TIMESTAMP '{start_time}'
-     and "timestamp" < TIMESTAMP '{end_time}'
-     """
+    LINEAGE_QUERY_DATABASE_FILTER: str = """and default_database IN ({databases})"""
+
+    LINEAGE_TIMESTAMP_BOUND_QUERY: str = """
+    SELECT MIN(CollectTimeStamp) as "min_ts", MAX(CollectTimeStamp) as "max_ts" from DBC.DBQLogTbl
+    """.strip()
+
+    QUERY_TEXT_QUERY: str = """
+    SELECT
+        s.QueryID as "query_id",
+        UserName as "user",
+        StartTime AT TIME ZONE 'GMT' as "timestamp",
+        DefaultDatabase as default_database,
+        s.SqlTextInfo as "query_text",
+        s.SqlRowNo as "row_no"
+    FROM "DBC".DBQLogTbl as l
+    JOIN "DBC".DBQLSqlTbl as s on s.QueryID = l.QueryID
+    WHERE
+        l.ErrorCode = 0
+        AND l.statementtype not in (
+        'Unrecognized type',
+        'Create Database/User',
+        'Help',
+        'Modify Database',
+        'Drop Table',
+        'Show',
+        'Not Applicable',
+        'Grant',
+        'Abort',
+        'Database',
+        'Flush Query Logging',
+        'Null',
+        'Begin/End DBQL',
+        'Revoke'
+    )
+        and "timestamp" >= TIMESTAMP '{start_time}'
+        and "timestamp" < TIMESTAMP '{end_time}'
+        and s.CollectTimeStamp >= TIMESTAMP '{start_time}'
+        and default_database not in ('DEMONOW_MONITOR')
+        {databases_filter}
+    ORDER BY "query_id", "row_no"
+    """.strip()
+
+    TABLES_AND_VIEWS_QUERY: str = """
+SELECT
+    t.DatabaseName,
+    t.TableName as name,
+    t.CommentString as description,
+    CASE t.TableKind
+         WHEN 'I' THEN 'Join index'
+         WHEN 'N' THEN 'Hash index'
+         WHEN 'T' THEN 'Table'
+         WHEN 'V' THEN 'View'
+         WHEN 'O' THEN 'NoPI Table'
+         WHEN 'Q' THEN 'Queue table'
+    END AS object_type,
+    t.CreateTimeStamp,
+    t.LastAlterName,
+    t.LastAlterTimeStamp,
+    t.RequestText
+FROM dbc.Tables t
+WHERE DatabaseName NOT IN (
+                'All',
+                'Crashdumps',
+                'Default',
+                'DemoNow_Monitor',
+                'EXTUSER',
+                'External_AP',
+                'GLOBAL_FUNCTIONS',
+                'LockLogShredder',
+                'PUBLIC',
+                'SQLJ',
+                'SYSBAR',
+                'SYSJDBC',
+                'SYSLIB',
+                'SYSSPATIAL',
+                'SYSUDTLIB',
+                'SYSUIF',
+                'SysAdmin',
+                'Sys_Calendar',
+                'SystemFe',
+                'TDBCMgmt',
+                'TDMaps',
+                'TDPUSER',
+                'TDQCD',
+                'TDStats',
+                'TD_ANALYTICS_DB',
+                'TD_SERVER_DB',
+                'TD_SYSFNLIB',
+                'TD_SYSGPL',
+                'TD_SYSXML',
+                'TDaaS_BAR',
+                'TDaaS_DB',
+                'TDaaS_Maint',
+                'TDaaS_Monitor',
+                'TDaaS_Support',
+                'TDaaS_TDBCMgmt1',
+                'TDaaS_TDBCMgmt2',
+                'dbcmngr',
+                'mldb',
+                'system',
+                'tapidb',
+                'tdwm',
+                'val',
+                'dbc'
+)
+AND t.TableKind in ('T', 'V', 'Q', 'O')
+ORDER by DatabaseName, TableName;
+     """.strip()
+
+    _tables_cache: MutableMapping[str, List[TeradataTable]] = defaultdict(list)
 
     def __init__(self, config: TeradataConfig, ctx: PipelineContext):
         super().__init__(config, ctx, "teradata")
@@ -145,36 +581,246 @@ def __init__(self, config: TeradataConfig, ctx: PipelineContext):
             generate_operations=self.config.usage.include_operational_stats,
         )
 
-        self.schema_resolver = SchemaResolver(
-            platform=self.platform,
-            platform_instance=self.config.platform_instance,
-            graph=None,
-            env=self.config.env,
-        )
+        self.schema_resolver = self._init_schema_resolver()
+
+        if self.config.include_tables or self.config.include_views:
+            self.cache_tables_and_views()
+            logger.info(f"Found {len(self._tables_cache)} tables and views")
+            setattr(self, "loop_tables", self.cached_loop_tables)  # noqa: B010
+            setattr(self, "loop_views", self.cached_loop_views)  # noqa: B010
+            setattr(  # noqa: B010
+                self, "get_table_properties", self.cached_get_table_properties
+            )
+
+            tables_cache = self._tables_cache
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_columns",
+                lambda self, connection, table_name, schema=None, use_qvci=self.config.use_qvci, **kw: optimized_get_columns(
+                    self,
+                    connection,
+                    table_name,
+                    schema,
+                    tables_cache=tables_cache,
+                    use_qvci=use_qvci,
+                    **kw,
+                ),
+            )
+
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_pk_constraint",
+                lambda self, connection, table_name, schema=None, **kw: optimized_get_pk_constraint(
+                    self, connection, table_name, schema, **kw
+                ),
+            )
+
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_foreign_keys",
+                lambda self, connection, table_name, schema=None, **kw: optimized_get_foreign_keys(
+                    self, connection, table_name, schema, **kw
+                ),
+            )
+
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_schema_columns",
+                lambda self, connection, dbc_columns, schema: get_schema_columns(
+                    self, connection, dbc_columns, schema
+                ),
+            )
+
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_view_definition",
+                lambda self, connection, view_name, schema=None, **kw: optimized_get_view_definition(
+                    self, connection, view_name, schema, tables_cache=tables_cache, **kw
+                ),
+            )
+
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_schema_pk_constraints",
+                lambda self, connection, schema: get_schema_pk_constraints(
+                    self, connection, schema
+                ),
+            )
+
+            setattr(  # noqa: B010
+                TeradataDialect,
+                "get_schema_foreign_keys",
+                lambda self, connection, schema: get_schema_foreign_keys(
+                    self, connection, schema
+                ),
+            )
+        else:
+            logger.info(
+                "Disabling stale entity removal as tables and views are disabled"
+            )
+            if self.config.stateful_ingestion:
+                self.config.stateful_ingestion.remove_stale_metadata = False
 
     @classmethod
     def create(cls, config_dict, ctx):
         config = TeradataConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
-    def get_audit_log_mcps(self) -> Iterable[MetadataWorkUnit]:
+    def _init_schema_resolver(self) -> SchemaResolver:
+        if not self.config.include_tables or not self.config.include_views:
+            if self.ctx.graph:
+                return self.ctx.graph.initialize_schema_resolver_from_datahub(
+                    platform=self.platform,
+                    platform_instance=self.config.platform_instance,
+                    env=self.config.env,
+                )
+            else:
+                logger.warning(
+                    "Failed to load schema info from DataHub as DataHubGraph is missing.",
+                )
+        return SchemaResolver(
+            platform=self.platform,
+            platform_instance=self.config.platform_instance,
+            env=self.config.env,
+        )
+
+    def get_inspectors(self):
+        # This method can be overridden in the case that you want to dynamically
+        # run on multiple databases.
+        url = self.config.get_sql_alchemy_url()
+        logger.debug(f"sql_alchemy_url={url}")
+        engine = create_engine(url, **self.config.options)
+        with engine.connect() as conn:
+            inspector = inspect(conn)
+            if self.config.database and self.config.database != "":
+                databases = [self.config.database]
+            elif self.config.databases:
+                databases = self.config.databases
+            else:
+                databases = inspector.get_schema_names()
+            for db in databases:
+                if self.config.database_pattern.allowed(db):
+                    # url = self.config.get_sql_alchemy_url(current_db=db)
+                    # with create_engine(url, **self.config.options).connect() as conn:
+                    #    inspector = inspect(conn)
+                    inspector._datahub_database = db
+                    yield inspector
+
+    def get_db_name(self, inspector: Inspector) -> str:
+        if hasattr(inspector, "_datahub_database"):
+            return inspector._datahub_database
+
+        engine = inspector.engine
+
+        if engine and hasattr(engine, "url") and hasattr(engine.url, "database"):
+            return str(engine.url.database).strip('"')
+        else:
+            raise Exception("Unable to get database name from Sqlalchemy inspector")
+
+    def cached_loop_tables(  # noqa: C901
+        self,
+        inspector: Inspector,
+        schema: str,
+        sql_config: SQLCommonConfig,
+    ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]:
+        setattr(  # noqa: B010
+            inspector,
+            "get_table_names",
+            lambda schema: [
+                i.name
+                for i in filter(
+                    lambda t: t.object_type != "View", self._tables_cache[schema]
+                )
+            ],
+        )
+        yield from super().loop_tables(inspector, schema, sql_config)
+
+    def cached_get_table_properties(
+        self, inspector: Inspector, schema: str, table: str
+    ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]:
+        description: Optional[str] = None
+        properties: Dict[str, str] = {}
+
+        # The location cannot be fetched generically, but subclasses may override
+        # this method and provide a location.
+        location: Optional[str] = None
+
+        for entry in self._tables_cache[schema]:
+            if entry.name == table:
+                description = entry.description
+                if entry.object_type == "View" and entry.request_text:
+                    properties["view_definition"] = entry.request_text
+                break
+        return description, properties, location
+
+    def cached_loop_views(  # noqa: C901
+        self,
+        inspector: Inspector,
+        schema: str,
+        sql_config: SQLCommonConfig,
+    ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]:
+        setattr(  # noqa: B010
+            inspector,
+            "get_view_names",
+            lambda schema: [
+                i.name
+                for i in filter(
+                    lambda t: t.object_type == "View", self._tables_cache[schema]
+                )
+            ],
+        )
+        yield from super().loop_views(inspector, schema, sql_config)
+
+    def cache_tables_and_views(self) -> None:
         engine = self.get_metadata_engine()
-        for entry in engine.execute(
-            self.LINEAGE_QUERY.format(
-                start_time=self.config.start_time, end_time=self.config.end_time
+        for entry in engine.execute(self.TABLES_AND_VIEWS_QUERY):
+            table = TeradataTable(
+                database=entry.DatabaseName.strip(),
+                name=entry.name.strip(),
+                description=entry.description.strip() if entry.description else None,
+                object_type=entry.object_type,
+                create_timestamp=entry.CreateTimeStamp,
+                last_alter_name=entry.LastAlterName,
+                last_alter_timestamp=entry.LastAlterTimeStamp,
+                request_text=entry.RequestText.strip()
+                if entry.object_type == "View" and entry.RequestText
+                else None,
             )
-        ):
+            if table.database not in self._tables_cache:
+                self._tables_cache[table.database] = []
+
+            self._tables_cache[table.database].append(table)
+
+    def get_audit_log_mcps(self, urns: Set[str]) -> Iterable[MetadataWorkUnit]:
+        engine = self.get_metadata_engine()
+        for entry in engine.execute(self._make_lineage_query()):
             self.report.num_queries_parsed += 1
             if self.report.num_queries_parsed % 1000 == 0:
                 logger.info(f"Parsed {self.report.num_queries_parsed} queries")
 
             yield from self.gen_lineage_from_query(
-                query=entry.query,
+                query=entry.query_text,
                 default_database=entry.default_database,
                 timestamp=entry.timestamp,
                 user=entry.user,
-                is_view_ddl=False,
+                urns=urns,
+            )
+
+    def _make_lineage_query(self) -> str:
+        databases_filter = (
+            ""
+            if not self.config.databases
+            else "and default_database in ({databases})".format(
+                databases=",".join([f"'{db}'" for db in self.config.databases])
             )
+        )
+
+        query = self.QUERY_TEXT_QUERY.format(
+            start_time=self.config.start_time,
+            end_time=self.config.end_time,
+            databases_filter=databases_filter,
+        )
+        return query
 
     def gen_lineage_from_query(
         self,
@@ -182,10 +828,12 @@ def gen_lineage_from_query(
         default_database: Optional[str] = None,
         timestamp: Optional[datetime] = None,
         user: Optional[str] = None,
-        is_view_ddl: bool = False,
+        view_urn: Optional[str] = None,
+        urns: Optional[Set[str]] = None,
     ) -> Iterable[MetadataWorkUnit]:
         result = sqlglot_lineage(
-            sql=query,
+            # With this clever hack we can make the query parser to not fail on queries with CASESPECIFIC
+            sql=query.replace("(NOT CASESPECIFIC)", ""),
             schema_resolver=self.schema_resolver,
             default_db=None,
             default_schema=default_database
@@ -194,17 +842,17 @@ def gen_lineage_from_query(
         )
         if result.debug_info.table_error:
             logger.debug(
-                f"Error parsing table lineage, {result.debug_info.table_error}"
+                f"Error parsing table lineage ({view_urn}):\n{result.debug_info.table_error}"
             )
             self.report.num_table_parse_failures += 1
         else:
             yield from self.builder.process_sql_parsing_result(
                 result,
                 query=query,
-                is_view_ddl=is_view_ddl,
+                is_view_ddl=view_urn is not None,
                 query_timestamp=timestamp,
                 user=f"urn:li:corpuser:{user}",
-                include_urns=self.schema_resolver.get_urns(),
+                include_urns=urns,
             )
 
     def get_metadata_engine(self) -> Engine:
@@ -214,10 +862,17 @@ def get_metadata_engine(self) -> Engine:
 
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
         # Add all schemas to the schema resolver
-        yield from super().get_workunits_internal()
+        # Sql parser operates on lowercase urns so we need to lowercase the urns
+        for wu in auto_lowercase_urns(super().get_workunits_internal()):
+            urn = wu.get_urn()
+            schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass)
+            if schema_metadata:
+                self.schema_resolver.add_schema_metadata(urn, schema_metadata)
+            yield wu
 
+        urns = self.schema_resolver.get_urns()
         if self.config.include_table_lineage or self.config.include_usage_statistics:
             self.report.report_ingestion_stage_start("audit log extraction")
-            yield from self.get_audit_log_mcps()
+            yield from self.get_audit_log_mcps(urns=urns)
 
         yield from self.builder.gen_workunits()

From 514c2fb157e14608eace62fc6fb632ccfb7a7157 Mon Sep 17 00:00:00 2001
From: david-leifker <114954101+david-leifker@users.noreply.github.com>
Date: Fri, 24 Nov 2023 14:12:11 -0600
Subject: [PATCH 247/259] fix(kafka): fix consumer properties on due consumer
 (#9304)

---
 .../kafka/SimpleKafkaConsumerFactory.java     |  2 +-
 .../kafka/SimpleKafkaConsumerFactoryTest.java | 32 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 metadata-service/factories/src/test/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactoryTest.java

diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java
index e12cbec87fe45..14ffc01d75781 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java
@@ -43,7 +43,7 @@ protected KafkaListenerContainerFactory<?> createInstance(@Qualifier("configurat
       consumerProps.setBootstrapServers(Arrays.asList(kafkaConfiguration.getBootstrapServers().split(",")));
     } // else we rely on KafkaProperties which defaults to localhost:9092
 
-    Map<String, Object> customizedProperties = consumerProps.buildProperties();
+    Map<String, Object> customizedProperties = properties.buildConsumerProperties();
     customizedProperties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG,
             kafkaConfiguration.getConsumer().getMaxPartitionFetchBytes());
 
diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactoryTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactoryTest.java
new file mode 100644
index 0000000000000..408c7b67b25f0
--- /dev/null
+++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactoryTest.java
@@ -0,0 +1,32 @@
+package com.linkedin.gms.factory.kafka;
+
+import com.linkedin.gms.factory.config.ConfigurationProvider;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
+import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+@SpringBootTest(
+        properties = {
+                "spring.kafka.properties.security.protocol=SSL"
+        },
+        classes = {
+                SimpleKafkaConsumerFactory.class,
+                ConfigurationProvider.class
+        })
+@EnableConfigurationProperties(ConfigurationProvider.class)
+public class SimpleKafkaConsumerFactoryTest extends AbstractTestNGSpringContextTests {
+    @Autowired
+    ConcurrentKafkaListenerContainerFactory<?, ?> testFactory;
+
+    @Test
+    void testInitialization() {
+        assertNotNull(testFactory);
+        assertEquals(testFactory.getConsumerFactory().getConfigurationProperties().get("security.protocol"), "SSL");
+    }
+}

From 298b9becb009ad775c31122ae9933bb2b02b92b3 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Sat, 25 Nov 2023 04:19:34 +0530
Subject: [PATCH 248/259] fix(dbt-cloud): do not pass macros to sorting nodes
 (#9302)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
---
 .../src/datahub/ingestion/source/dbt/dbt_common.py               | 1 +
 1 file changed, 1 insertion(+)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index 94df0a4f8a166..919ba5a4b285a 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -895,6 +895,7 @@ def _infer_schemas_and_update_cll(self, all_nodes_map: Dict[str, DBTNode]) -> No
                 (upstream, node.dbt_name)
                 for node in all_nodes_map.values()
                 for upstream in node.upstream_nodes
+                if upstream in all_nodes_map
             ),
         ):
             node = all_nodes_map[dbt_name]

From a34fdfd8b73b5eeefbf8751ed920f2e5110753b6 Mon Sep 17 00:00:00 2001
From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com>
Date: Sat, 25 Nov 2023 04:29:24 +0530
Subject: [PATCH 249/259] fix(ingest/lookml): emit all views with same name and
 different file path (#9279)

---
 .../ingestion/source/looker/lookml_source.py  |  23 +-
 .../data.model.lkml                           |   7 +
 .../data2.model.lkml                          |   6 +
 .../path1/foo.view.lkml                       |  47 ++
 .../path2/foo.view.lkml                       |  41 ++
 ...l_same_name_views_different_file_path.json | 587 ++++++++++++++++++
 .../tests/integration/lookml/test_lookml.py   |  50 ++
 7 files changed, 755 insertions(+), 6 deletions(-)
 create mode 100644 metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data.model.lkml
 create mode 100644 metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data2.model.lkml
 create mode 100644 metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path1/foo.view.lkml
 create mode 100644 metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path2/foo.view.lkml
 create mode 100644 metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
index 9d7c972612777..2bd469b3f9bcd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
@@ -1982,9 +1982,16 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
             self.reporter,
         )
 
-        # some views can be mentioned by multiple 'include' statements and can be included via different connections.
-        # So this set is used to prevent creating duplicate events
+        # Some views can be mentioned by multiple 'include' statements and can be included via different connections.
+
+        # This map is used to keep track of which views files have already been processed
+        # for a connection in order to prevent creating duplicate events.
+        # Key: connection name, Value: view file paths
         processed_view_map: Dict[str, Set[str]] = {}
+
+        # This map is used to keep track of the connection that a view is processed with.
+        # Key: view unique identifier - determined by variables present in config `view_naming_pattern`
+        # Value: Tuple(model file name, connection name)
         view_connection_map: Dict[str, Tuple[str, str]] = {}
 
         # The ** means "this directory and all subdirectories", and hence should
@@ -2148,13 +2155,17 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                             if self.source_config.view_pattern.allowed(
                                 maybe_looker_view.id.view_name
                             ):
+                                view_urn = maybe_looker_view.id.get_urn(
+                                    self.source_config
+                                )
                                 view_connection_mapping = view_connection_map.get(
-                                    maybe_looker_view.id.view_name
+                                    view_urn
                                 )
                                 if not view_connection_mapping:
-                                    view_connection_map[
-                                        maybe_looker_view.id.view_name
-                                    ] = (model_name, model.connection)
+                                    view_connection_map[view_urn] = (
+                                        model_name,
+                                        model.connection,
+                                    )
                                     # first time we are discovering this view
                                     logger.debug(
                                         f"Generating MCP for view {raw_view['name']}"
diff --git a/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data.model.lkml b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data.model.lkml
new file mode 100644
index 0000000000000..183b16b2a3c1d
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data.model.lkml
@@ -0,0 +1,7 @@
+connection: "my_connection"
+
+include: "path1/foo.view.lkml"
+
+explore: aliased_explore {
+  from: my_view
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data2.model.lkml b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data2.model.lkml
new file mode 100644
index 0000000000000..6a4a96e2630fa
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/data2.model.lkml
@@ -0,0 +1,6 @@
+connection: "my_connection"
+include: "path2/foo.view.lkml"
+
+explore: duplicate_explore {
+  from: my_view
+}
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path1/foo.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path1/foo.view.lkml
new file mode 100644
index 0000000000000..40a981ebc7eb0
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path1/foo.view.lkml
@@ -0,0 +1,47 @@
+view: my_view {
+  derived_table: {
+    sql:
+        SELECT
+          is_latest,
+          country,
+          city,
+          timestamp,
+          measurement
+        FROM
+          my_table ;;
+  }
+
+  dimension: country {
+    type: string
+    description: "The country"
+    sql: ${TABLE}.country ;;
+  }
+
+  dimension: city {
+    type: string
+    description: "City"
+    sql: ${TABLE}.city ;;
+  }
+
+  dimension: is_latest {
+    type: yesno
+    description: "Is latest data"
+    sql: ${TABLE}.is_latest ;;
+  }
+
+  dimension_group: timestamp {
+    group_label: "Timestamp"
+    type: time
+    description: "Timestamp of measurement"
+    sql: ${TABLE}.timestamp ;;
+    timeframes: [hour, date, week, day_of_week]
+  }
+
+  measure: average_measurement {
+    group_label: "Measurement"
+    type: average
+    description: "My measurement"
+    sql: ${TABLE}.measurement ;;
+  }
+
+}
diff --git a/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path2/foo.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path2/foo.view.lkml
new file mode 100644
index 0000000000000..8bd8138f97386
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/lkml_same_name_views_different_file_path_samples/path2/foo.view.lkml
@@ -0,0 +1,41 @@
+view: my_view {
+  derived_table: {
+    sql:
+        SELECT
+          is_latest,
+          country,
+          city,
+          timestamp,
+          measurement
+        FROM
+          my_table ;;
+  }
+
+  dimension: city {
+    type: string
+    description: "City"
+    sql: ${TABLE}.city ;;
+  }
+
+  dimension: is_latest {
+    type: yesno
+    description: "Is latest data"
+    sql: ${TABLE}.is_latest ;;
+  }
+
+  dimension_group: timestamp {
+    group_label: "Timestamp"
+    type: time
+    description: "Timestamp of measurement"
+    sql: ${TABLE}.timestamp ;;
+    timeframes: [hour, date, week, day_of_week]
+  }
+
+  measure: average_measurement {
+    group_label: "Measurement"
+    type: average
+    description: "My measurement"
+    sql: ${TABLE}.measurement ;;
+  }
+
+}
diff --git a/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json b/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json
new file mode 100644
index 0000000000000..c212cc33b66d4
--- /dev/null
+++ b/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json
@@ -0,0 +1,587 @@
+[
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/looker/lkml_samples/path1/foo.view.lkml/views"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
+                        "upstreams": [
+                            {
+                                "auditStamp": {
+                                    "time": 1586847600000,
+                                    "actor": "urn:li:corpuser:datahub"
+                                },
+                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)",
+                                "type": "VIEW"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "my_view",
+                        "platform": "urn:li:dataPlatform:looker",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "country",
+                                "nullable": false,
+                                "description": "The country",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "description": "City",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "is_latest",
+                                "nullable": false,
+                                "description": "Is latest data",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "yesno",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "timestamp",
+                                "nullable": false,
+                                "description": "Timestamp of measurement",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "time",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:Temporal"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "average_measurement",
+                                "nullable": false,
+                                "description": "My measurement",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "average",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Measure"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ],
+                        "primaryKeys": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "looker.file.path": "path1/foo.view.lkml"
+                        },
+                        "name": "my_view",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT\n          is_latest,\n          country,\n          city,\n          timestamp,\n          measurement\n        FROM\n          my_table",
+            "viewLanguage": "sql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "looker"
+                },
+                {
+                    "id": "lkml_samples"
+                },
+                {
+                    "id": "path1"
+                },
+                {
+                    "id": "foo.view.lkml"
+                },
+                {
+                    "id": "views"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/prod/looker/lkml_samples/path2/foo.view.lkml/views"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
+                        "upstreams": [
+                            {
+                                "auditStamp": {
+                                    "time": 1586847600000,
+                                    "actor": "urn:li:corpuser:datahub"
+                                },
+                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)",
+                                "type": "VIEW"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "my_view",
+                        "platform": "urn:li:dataPlatform:looker",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "city",
+                                "nullable": false,
+                                "description": "City",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "string",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "is_latest",
+                                "nullable": false,
+                                "description": "Is latest data",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.BooleanType": {}
+                                    }
+                                },
+                                "nativeDataType": "yesno",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "timestamp",
+                                "nullable": false,
+                                "description": "Timestamp of measurement",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
+                                    }
+                                },
+                                "nativeDataType": "time",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        },
+                                        {
+                                            "tag": "urn:li:tag:Temporal"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            },
+                            {
+                                "fieldPath": "average_measurement",
+                                "nullable": false,
+                                "description": "My measurement",
+                                "label": "",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "average",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Measure"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": false
+                            }
+                        ],
+                        "primaryKeys": []
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "looker.file.path": "path2/foo.view.lkml"
+                        },
+                        "name": "my_view",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "SELECT\n          is_latest,\n          country,\n          city,\n          timestamp,\n          measurement\n        FROM\n          my_table",
+            "viewLanguage": "sql"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "looker"
+                },
+                {
+                    "id": "lkml_samples"
+                },
+                {
+                    "id": "path2"
+                },
+                {
+                    "id": "foo.view.lkml"
+                },
+                {
+                    "id": "views"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
+            "urn": "urn:li:tag:Dimension",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.tag.TagProperties": {
+                        "name": "Dimension",
+                        "description": "A tag that is applied to all dimension fields."
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
+            "urn": "urn:li:tag:Temporal",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.tag.TagProperties": {
+                        "name": "Temporal",
+                        "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations."
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": {
+            "urn": "urn:li:tag:Measure",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.tag.TagProperties": {
+                        "name": "Measure",
+                        "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on"
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Dimension",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Measure",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:Temporal",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py
index b1853cfa2b3c0..a71b597863148 100644
--- a/metadata-ingestion/tests/integration/lookml/test_lookml.py
+++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py
@@ -802,3 +802,53 @@ def test_lookml_base_folder():
         pydantic.ValidationError, match=r"base_folder.+not provided.+deploy_key"
     ):
         LookMLSourceConfig.parse_obj({"api": fake_api})
+
+
+@freeze_time(FROZEN_TIME)
+def test_same_name_views_different_file_path(pytestconfig, tmp_path, mock_time):
+    """Test for reachable views"""
+    test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml"
+    mce_out = "lookml_same_name_views_different_file_path.json"
+    pipeline = Pipeline.create(
+        {
+            "run_id": "lookml-test",
+            "source": {
+                "type": "lookml",
+                "config": {
+                    "base_folder": str(
+                        test_resources_dir
+                        / "lkml_same_name_views_different_file_path_samples"
+                    ),
+                    "connection_to_platform_map": {
+                        "my_connection": {
+                            "platform": "snowflake",
+                            "platform_instance": "warehouse",
+                            "platform_env": "dev",
+                            "default_db": "default_db",
+                            "default_schema": "default_schema",
+                        },
+                    },
+                    "parse_table_names_from_sql": True,
+                    "project_name": "lkml_samples",
+                    "process_refinements": False,
+                    "view_naming_pattern": "{project}.{file_path}.view.{name}",
+                    "view_browse_pattern": "/{env}/{platform}/{project}/{file_path}/views",
+                },
+            },
+            "sink": {
+                "type": "file",
+                "config": {
+                    "filename": f"{tmp_path}/{mce_out}",
+                },
+            },
+        }
+    )
+    pipeline.run()
+    pipeline.pretty_print_summary()
+    pipeline.raise_from_status(raise_warnings=True)
+
+    mce_helpers.check_golden_file(
+        pytestconfig,
+        output_path=tmp_path / mce_out,
+        golden_path=test_resources_dir / mce_out,
+    )

From cacddf7e983198beb7ab54b3f74daf10017692b8 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Mon, 27 Nov 2023 14:26:27 +0530
Subject: [PATCH 250/259] fix(deprecation): bring frontend in-sync with model
 (#9303)

---
 .../EntityDropdown/UpdateDeprecationModal.tsx   |  2 +-
 .../components/styled/DeprecationPill.tsx       | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx
index 512735e60b2c3..6ae893e12575f 100644
--- a/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx
@@ -27,7 +27,7 @@ export const UpdateDeprecationModal = ({ urns, onClose, refetch }: Props) => {
                         resources: [...urns.map((urn) => ({ resourceUrn: urn }))],
                         deprecated: true,
                         note: formData.note,
-                        decommissionTime: formData.decommissionTime && formData.decommissionTime.unix(),
+                        decommissionTime: formData.decommissionTime && formData.decommissionTime.unix() * 1000,
                     },
                 },
             });
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx
index ffc32c1538259..f60a74247ebcc 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx
@@ -83,15 +83,24 @@ export const DeprecationPill = ({ deprecation, urn, refetch, showUndeprecate }:
      * Deprecation Decommission Timestamp
      */
     const localeTimezone = getLocaleTimezone();
+
+    let decommissionTimeSeconds;
+    if (deprecation.decommissionTime) {
+        if (deprecation.decommissionTime < 943920000000) {
+            // Time is set in way past if it was milli-second so considering this as set in seconds
+            decommissionTimeSeconds = deprecation.decommissionTime;
+        } else {
+            decommissionTimeSeconds = deprecation.decommissionTime / 1000;
+        }
+    }
     const decommissionTimeLocal =
-        (deprecation.decommissionTime &&
+        (decommissionTimeSeconds &&
             `Scheduled to be decommissioned on ${moment
-                .unix(deprecation.decommissionTime)
+                .unix(decommissionTimeSeconds)
                 .format('DD/MMM/YYYY')} (${localeTimezone})`) ||
         undefined;
     const decommissionTimeGMT =
-        deprecation.decommissionTime &&
-        moment.unix(deprecation.decommissionTime).utc().format('dddd, DD/MMM/YYYY HH:mm:ss z');
+        decommissionTimeSeconds && moment.unix(decommissionTimeSeconds).utc().format('dddd, DD/MMM/YYYY HH:mm:ss z');
 
     const hasDetails = deprecation.note !== '' || deprecation.decommissionTime !== null;
     const isDividerNeeded = deprecation.note !== '' && deprecation.decommissionTime !== null;

From ae5969fcfb22e25e8b08987285ab2087d1373580 Mon Sep 17 00:00:00 2001
From: Salman-Apptware <101426513+Salman-Apptware@users.noreply.github.com>
Date: Mon, 27 Nov 2023 14:27:16 +0530
Subject: [PATCH 251/259] fix(settings): fix the settings height when there are
 not many items (#9294)

---
 datahub-web-react/src/app/settings/SettingsPage.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx
index 69d4eb2b10b4d..e0a15c73a626d 100644
--- a/datahub-web-react/src/app/settings/SettingsPage.tsx
+++ b/datahub-web-react/src/app/settings/SettingsPage.tsx
@@ -25,11 +25,11 @@ import ManagePosts from './posts/ManagePosts';
 const PageContainer = styled.div`
     display: flex;
     overflow: auto;
+    flex: 1;
 `;
 
 const SettingsBarContainer = styled.div`
     padding-top: 20px;
-    max-height: 100vh;
     border-right: 1px solid ${ANTD_GRAY[5]};
     display: flex;
     flex-direction: column;

From 3806dc184ec372c2b48a2bc5f37dfd09e3e5809e Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Mon, 27 Nov 2023 16:12:44 +0530
Subject: [PATCH 252/259] docs: update recommended CLI (#9307)

---
 docs/managed-datahub/release-notes/v_0_2_13.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/managed-datahub/release-notes/v_0_2_13.md b/docs/managed-datahub/release-notes/v_0_2_13.md
index 65cea863d9714..c4fb1f7dcc252 100644
--- a/docs/managed-datahub/release-notes/v_0_2_13.md
+++ b/docs/managed-datahub/release-notes/v_0_2_13.md
@@ -3,11 +3,11 @@
 
 Release Availability Date
 ---
-22-Nov-2023
+27-Nov-2023
 
 Recommended CLI/SDK
 ---
-- `v0.12.0.2` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.12.0.2=
+- `v0.12.0.3` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.12.0.3
 
 If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better.
 

From a9650b650c59c67fffb1df1292874bb5c8e884d8 Mon Sep 17 00:00:00 2001
From: Amanda Ng <10681923+ngamanda@users.noreply.github.com>
Date: Tue, 28 Nov 2023 00:45:21 +0800
Subject: [PATCH 253/259] feat(ui): bump frontend dependencies (#8353)

Co-authored-by: Chris Collins <chriscollins3456@gmail.com>
---
 datahub-web-react/.eslintrc.js                |    38 +-
 datahub-web-react/craco.config.js             |    40 +-
 datahub-web-react/package.json                |    55 +-
 datahub-web-react/src/__mocks__/styleMock.js  |     1 +
 .../components/BarChart.tsx                   |    10 +-
 .../analyticsDashboard/components/Legend.tsx  |     2 +-
 .../components/TimeSeriesChart.tsx            |    96 +-
 .../components/legacy/MarkdownViewer.tsx      |     1 +
 .../Stats/historical/charts/StatChart.tsx     |     4 +-
 .../Dataset/Validations/BooleanTimeline.tsx   |     8 +-
 .../src/app/lineage/ColumnNode.tsx            |     2 +-
 .../src/app/lineage/LineageEntityColumns.tsx  |     2 +-
 .../src/app/lineage/LineageEntityEdge.tsx     |     6 +-
 .../src/app/lineage/LineageEntityNode.tsx     |     4 +-
 .../src/app/lineage/LineageTree.tsx           |     2 +-
 .../LineageTreeNodeAndEdgeRenderer.tsx        |     4 +-
 .../src/app/lineage/LineageViz.tsx            |     4 +-
 .../src/app/lineage/LineageVizInsideZoom.tsx  |     4 +-
 .../src/app/lineage/LineageVizRootSvg.tsx     |     4 +-
 .../src/app/lineage/NodeColumnsHeader.tsx     |     2 +-
 .../lineage/__tests__/LineageTree.test.tsx    |     2 +-
 .../__tests__/adjustVXTreeLayout.test.tsx     |     2 +-
 .../app/lineage/utils/adjustVXTreeLayout.ts   |     2 +-
 .../app/shared/share/items/EmailMenuItem.tsx  |    31 +-
 .../src/conf/theme/styled-components.d.ts     |     2 +-
 datahub-web-react/src/graphql-mock/server.ts  |     1 +
 datahub-web-react/tsconfig.json               |     4 +-
 datahub-web-react/yarn.lock                   | 12838 ++++++----------
 28 files changed, 5296 insertions(+), 7875 deletions(-)
 create mode 100644 datahub-web-react/src/__mocks__/styleMock.js

diff --git a/datahub-web-react/.eslintrc.js b/datahub-web-react/.eslintrc.js
index 3cdc747d100b5..2806942dd1053 100644
--- a/datahub-web-react/.eslintrc.js
+++ b/datahub-web-react/.eslintrc.js
@@ -1,15 +1,14 @@
 module.exports = {
     parser: '@typescript-eslint/parser', // Specifies the ESLint parser
     extends: [
-        'react-app',
-        'plugin:react/recommended', // Uses the recommended rules from @eslint-plugin-react
-        'plugin:@typescript-eslint/recommended', // Uses the recommended rules from @typescript-eslint/eslint-plugin
-        'plugin:jest/recommended',
+        'airbnb',
         'airbnb-typescript',
         'airbnb/hooks',
+        'plugin:@typescript-eslint/recommended',
+        'plugin:jest/recommended',
         'prettier',
-        'plugin:prettier/recommended',
     ],
+    plugins: ['@typescript-eslint'],
     parserOptions: {
         ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features
         sourceType: 'module', // Allows for the use of imports
@@ -19,18 +18,27 @@ module.exports = {
         project: './tsconfig.json',
     },
     rules: {
-        eqeqeq: ['error', 'always'],
-        'react/destructuring-assignment': 'off',
-        'no-console': 'off',
-        'no-debugger': 'warn',
-        'require-await': 'warn',
+        '@typescript-eslint/no-explicit-any': 'off',
+        'arrow-body-style': 'off',
+        'class-methods-use-this': 'off',
+        'import/no-extraneous-dependencies': 'off',
         'import/prefer-default-export': 'off', // TODO: remove this lint rule
-        'import/extensions': 'off',
-        'react/jsx-props-no-spreading': 'off',
+        'no-console': 'off',
         'no-plusplus': 'off',
         'no-prototype-builtins': 'off',
-        'react/require-default-props': 'off',
+        'no-restricted-exports': ['off', { restrictedNamedExports: ['default', 'then'] }],
         'no-underscore-dangle': 'off',
+        'no-unsafe-optional-chaining': 'off',
+        'prefer-exponentiation-operator': 'off',
+        'prefer-regex-literals': 'off',
+        'react/destructuring-assignment': 'off',
+        'react/function-component-definition': 'off',
+        'react/jsx-no-bind': 'off',
+        'react/jsx-no-constructed-context-values': 'off',
+        'react/jsx-no-useless-fragment': 'off',
+        'react/jsx-props-no-spreading': 'off',
+        'react/no-unstable-nested-components': 'off',
+        'react/require-default-props': 'off',
         '@typescript-eslint/no-unused-vars': [
             'error',
             {
@@ -38,10 +46,6 @@ module.exports = {
                 argsIgnorePattern: '^_',
             },
         ],
-        '@typescript-eslint/no-empty-interface': 'off',
-        "@typescript-eslint/explicit-module-boundary-types": "off",
-        "@typescript-eslint/no-explicit-any": 'off',
-        "import/no-extraneous-dependencies": 'off'
     },
     settings: {
         react: {
diff --git a/datahub-web-react/craco.config.js b/datahub-web-react/craco.config.js
index d3ed895cf840e..6ede45902128f 100644
--- a/datahub-web-react/craco.config.js
+++ b/datahub-web-react/craco.config.js
@@ -1,8 +1,11 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
 require('dotenv').config();
+const { whenProd } = require('@craco/craco');
 const CracoAntDesignPlugin = require('craco-antd');
 const path = require('path');
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 
+// eslint-disable-next-line import/no-dynamic-require
 const themeConfig = require(`./src/conf/theme/${process.env.REACT_APP_THEME_CONFIG}`);
 
 function addLessPrefixToKeys(styles) {
@@ -15,6 +18,23 @@ function addLessPrefixToKeys(styles) {
 
 module.exports = {
     webpack: {
+        configure: {
+            optimization: whenProd(() => ({
+                splitChunks: {
+                    cacheGroups: {
+                        vendor: {
+                            test: /[\\/]node_modules[\\/]/,
+                            name: 'vendors',
+                            chunks: 'all',
+                        },
+                    },
+                },
+            })),
+            // Webpack 5 no longer automatically pollyfill core Node.js modules
+            resolve: { fallback: { fs: false } },
+            // Ignore Webpack 5's missing source map warnings from node_modules
+            ignoreWarnings: [{ module: /node_modules/, message: /source-map-loader/ }],
+        },
         plugins: {
             add: [
                 // Self host images by copying them to the build directory
@@ -24,8 +44,8 @@ module.exports = {
                 // Copy monaco-editor files to the build directory
                 new CopyWebpackPlugin({
                     patterns: [
-                        { from: "node_modules/monaco-editor/min/vs/", to: "monaco-editor/vs" },
-                        { from: "node_modules/monaco-editor/min-maps/vs/", to: "monaco-editor/min-maps/vs" },
+                        { from: 'node_modules/monaco-editor/min/vs/', to: 'monaco-editor/vs' },
+                        { from: 'node_modules/monaco-editor/min-maps/vs/', to: 'monaco-editor/min-maps/vs' },
                     ],
                 }),
             ],
@@ -41,13 +61,15 @@ module.exports = {
         },
     ],
     jest: {
-        configure: (jestConfig) => {
-            jestConfig.transformIgnorePatterns = [
-                // Ensures that lib0 and y-protocol libraries are transformed through babel as well
-                'node_modules/(?!(lib0|y-protocols)).+\\.(js|jsx|mjs|cjs|ts|tsx)$',
-                '^.+\\.module\\.(css|sass|scss)$',
-            ];
-            return jestConfig;
+        configure: {
+            // Use dist files instead of source files
+            moduleNameMapper: {
+                '^d3-interpolate-path': `d3-interpolate-path/build/d3-interpolate-path`,
+                '^d3-(.*)$': `d3-$1/dist/d3-$1`,
+                '^lib0/((?!dist).*)$': 'lib0/dist/$1.cjs',
+                '^y-protocols/(.*)$': 'y-protocols/dist/$1.cjs',
+                '\\.(css|less)$': '<rootDir>/src/__mocks__/styleMock.js',
+            },
         },
     },
 };
diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index a72d9c0a898f8..f55588e46c9c7 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -9,12 +9,12 @@
         "@ant-design/colors": "^5.0.0",
         "@ant-design/icons": "^4.3.0",
         "@apollo/client": "^3.3.19",
-        "@craco/craco": "^6.1.1",
-        "@data-ui/xy-chart": "^0.0.84",
+        "@craco/craco": "^7.1.0",
         "@graphql-codegen/fragment-matcher": "^5.0.0",
         "@miragejs/graphql": "^0.1.11",
         "@monaco-editor/react": "^4.3.1",
         "@react-hook/window-size": "^3.0.7",
+        "@react-spring/web": "^9.7.3",
         "@remirror/pm": "^2.0.3",
         "@remirror/react": "^2.0.24",
         "@remirror/styles": "^2.0.3",
@@ -30,21 +30,22 @@
         "@types/react-router": "^5.1.8",
         "@types/react-router-dom": "^5.1.6",
         "@uiw/react-md-editor": "^3.3.4",
-        "@vx/axis": "^0.0.199",
-        "@vx/glyph": "^0.0.199",
-        "@vx/group": "^0.0.199",
-        "@vx/hierarchy": "^0.0.199",
-        "@vx/legend": "^0.0.199",
-        "@vx/scale": "^0.0.199",
-        "@vx/shape": "^0.0.199",
-        "@vx/tooltip": "^0.0.199",
-        "@vx/zoom": "^0.0.199",
+        "@visx/axis": "^3.1.0",
+        "@visx/curve": "^3.0.0",
+        "@visx/group": "^3.0.0",
+        "@visx/hierarchy": "^3.0.0",
+        "@visx/legend": "^3.2.0",
+        "@visx/scale": "^3.2.0",
+        "@visx/shape": "^3.2.0",
+        "@visx/xychart": "^3.2.0",
+        "@visx/zoom": "^3.1.1",
         "analytics": "^0.8.9",
         "antd": "4.24.7",
         "color-hash": "^2.0.1",
-        "craco-antd": "^1.19.0",
+        "craco-antd": "^2.0.0",
+        "craco-babel-loader": "^1.0.4",
         "cronstrue": "^1.122.0",
-        "d3-scale": "^3.3.0",
+        "d3-scale": "^4.0.2",
         "dayjs": "^1.11.7",
         "deepmerge": "^4.2.2",
         "diff": "^5.0.0",
@@ -67,14 +68,13 @@
         "react": "^17.0.0",
         "react-color": "^2.19.3",
         "react-dom": "^17.0.0",
-        "react-email-share-link": "^1.0.3",
         "react-helmet-async": "^1.3.0",
         "react-highlighter": "^0.4.3",
         "react-icons": "4.3.1",
         "react-js-cron": "^2.1.0",
         "react-router": "^5.2.0",
         "react-router-dom": "^5.1.6",
-        "react-scripts": "4.0.3",
+        "react-scripts": "^5.0.1",
         "react-syntax-highlighter": "^15.4.4",
         "react-visibility-sensor": "^5.1.1",
         "reactour": "1.18.7",
@@ -83,7 +83,7 @@
         "start-server-and-test": "1.12.2",
         "styled-components": "^5.2.1",
         "turndown-plugin-gfm": "^1.0.2",
-        "typescript": "^4.1.3",
+        "typescript": "^4.8.4",
         "uuid": "^8.3.2",
         "virtualizedtableforantd4": "^1.2.1",
         "web-vitals": "^0.2.4",
@@ -123,6 +123,7 @@
         ]
     },
     "devDependencies": {
+        "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
         "@graphql-codegen/cli": "1.20.0",
         "@graphql-codegen/near-operation-file-preset": "^1.17.13",
         "@graphql-codegen/typescript-operations": "1.17.13",
@@ -130,17 +131,19 @@
         "@types/graphql": "^14.5.0",
         "@types/query-string": "^6.3.0",
         "@types/styled-components": "^5.1.7",
-        "@typescript-eslint/eslint-plugin": "^4.25.0",
-        "@typescript-eslint/parser": "^4.25.0",
-        "babel-loader": "8.2.2",
-        "copy-webpack-plugin": "6.4.1",
-        "eslint": "^7.27.0",
-        "eslint-config-airbnb-typescript": "^12.3.1",
-        "eslint-config-prettier": "^8.3.0",
-        "eslint-plugin-prettier": "^3.4.0",
-        "eslint-plugin-react": "^7.23.2",
+        "@typescript-eslint/eslint-plugin": "^5.38.1",
+        "@typescript-eslint/parser": "^5.38.1",
+        "copy-webpack-plugin": "^11.0.0",
+        "eslint": "^8.2.0",
+        "eslint-config-airbnb": "19.0.4",
+        "eslint-config-airbnb-typescript": "^17.0.0",
+        "eslint-config-prettier": "^8.8.0",
+        "eslint-plugin-import": "^2.25.3",
+        "eslint-plugin-jsx-a11y": "^6.5.1",
+        "eslint-plugin-react": "^7.28.0",
+        "eslint-plugin-react-hooks": "^4.3.0",
         "http-proxy-middleware": "2.0.0",
-        "prettier": "^2.3.0",
+        "prettier": "^2.8.8",
         "source-map-explorer": "^2.5.2"
     },
     "resolutions": {
diff --git a/datahub-web-react/src/__mocks__/styleMock.js b/datahub-web-react/src/__mocks__/styleMock.js
new file mode 100644
index 0000000000000..f053ebf7976e3
--- /dev/null
+++ b/datahub-web-react/src/__mocks__/styleMock.js
@@ -0,0 +1 @@
+module.exports = {};
diff --git a/datahub-web-react/src/app/analyticsDashboard/components/BarChart.tsx b/datahub-web-react/src/app/analyticsDashboard/components/BarChart.tsx
index f178b09afcccc..0c9909313e27e 100644
--- a/datahub-web-react/src/app/analyticsDashboard/components/BarChart.tsx
+++ b/datahub-web-react/src/app/analyticsDashboard/components/BarChart.tsx
@@ -1,8 +1,8 @@
 import React, { useMemo } from 'react';
-import { BarStack } from '@vx/shape';
-import { scaleOrdinal, scaleLinear, scaleBand } from '@vx/scale';
-import { Group } from '@vx/group';
-import { AxisBottom, AxisRight } from '@vx/axis';
+import { BarStack } from '@visx/shape';
+import { scaleOrdinal, scaleLinear, scaleBand } from '@visx/scale';
+import { Group } from '@visx/group';
+import { AxisBottom, AxisRight } from '@visx/axis';
 
 import { BarChart as BarChartType } from '../../../types.generated';
 import { lineColors } from './lineColors';
@@ -85,7 +85,7 @@ export const BarChart = ({ chartData, width, height }: Props) => {
             <svg width={width + WIDTH_MARGIN_SIZE} height={height}>
                 <rect x={0} y={0} width={width} height={height} fill="white" rx={14} />
                 <Group top={HEIGHT_MARGIN_SIZE} left={WIDTH_MARGIN_SIZE}>
-                    <BarStack<typeof transformedChartData[0], typeof keys[number]>
+                    <BarStack<(typeof transformedChartData)[0], (typeof keys)[number]>
                         data={transformedChartData}
                         keys={keys}
                         x={(data) => data.displayName}
diff --git a/datahub-web-react/src/app/analyticsDashboard/components/Legend.tsx b/datahub-web-react/src/app/analyticsDashboard/components/Legend.tsx
index c4ef51fb3938b..7796713961ca3 100644
--- a/datahub-web-react/src/app/analyticsDashboard/components/Legend.tsx
+++ b/datahub-web-react/src/app/analyticsDashboard/components/Legend.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { Col, Row } from 'antd';
-import { LegendOrdinal, LegendItem, LegendLabel } from '@vx/legend';
+import { LegendOrdinal, LegendItem, LegendLabel } from '@visx/legend';
 import { ScaleOrdinal } from 'd3-scale/src/ordinal';
 import styled from 'styled-components';
 
diff --git a/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx b/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx
index 6b9b808abfd0f..68851a950bcc5 100644
--- a/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx
+++ b/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx
@@ -1,17 +1,15 @@
 import React, { useMemo } from 'react';
-import { XYChart, LineSeries, CrossHair, XAxis, YAxis } from '@data-ui/xy-chart';
-import { scaleOrdinal } from '@vx/scale';
+import styled from 'styled-components';
+import { AxisScaleOutput } from '@visx/axis';
+import { Axis, LineSeries, XYChart, Tooltip, GlyphSeries } from '@visx/xychart';
+import { curveMonotoneX } from '@visx/curve';
+import { ScaleConfig, scaleOrdinal } from '@visx/scale';
 import { TimeSeriesChart as TimeSeriesChartType, NumericDataPoint, NamedLine } from '../../../types.generated';
 import { lineColors } from './lineColors';
 import Legend from './Legend';
 import { addInterval } from '../../shared/time/timeUtils';
 import { formatNumber } from '../../shared/formatNumber';
 
-type ScaleConfig = {
-    type: 'time' | 'timeUtc' | 'linear' | 'band' | 'ordinal';
-    includeZero?: boolean;
-};
-
 type AxisConfig = {
     formatter: (tick: number) => string;
 };
@@ -29,10 +27,15 @@ type Props = {
         crossHairLineColor?: string;
     };
     insertBlankPoints?: boolean;
-    yScale?: ScaleConfig;
+    yScale?: ScaleConfig<AxisScaleOutput, any, any>;
     yAxis?: AxisConfig;
 };
 
+const StyledTooltip = styled(Tooltip)`
+    font-family: inherit !important;
+    font-weight: 400 !important;
+`;
+
 const MARGIN = {
     TOP: 40,
     RIGHT: 45,
@@ -40,6 +43,11 @@ const MARGIN = {
     LEFT: 40,
 };
 
+const accessors = {
+    xAccessor: (d) => d.x,
+    yAccessor: (d) => d.y,
+};
+
 function insertBlankAt(ts: number, newLine: Array<NumericDataPoint>) {
     const dateString = new Date(ts).toISOString();
     for (let i = 0; i < newLine.length; i++) {
@@ -96,41 +104,61 @@ export const TimeSeriesChart = ({
     return (
         <>
             <XYChart
-                eventTrigger="container"
-                ariaLabel={chartData.title}
+                accessibilityLabel={chartData.title}
                 width={width}
                 height={height}
                 margin={{ top: MARGIN.TOP, right: MARGIN.RIGHT, bottom: MARGIN.BOTTOM, left: MARGIN.LEFT }}
                 xScale={{ type: 'time' }}
-                yScale={
-                    yScale ?? {
-                        type: 'linear',
-                    }
-                }
-                renderTooltip={({ datum }) => (
-                    <div>
-                        <div>{new Date(Number(datum.x)).toDateString()}</div>
-                        <div>{datum.y}</div>
-                    </div>
-                )}
-                snapTooltipToDataX={false}
+                yScale={yScale ?? { type: 'linear' }}
             >
-                <XAxis axisStyles={{ stroke: style && style.axisColor, strokeWidth: style && style.axisWidth }} />
-                <YAxis
-                    axisStyles={{ stroke: style && style.axisColor, strokeWidth: style && style.axisWidth }}
+                <Axis
+                    orientation="bottom"
+                    stroke={style?.axisColor}
+                    strokeWidth={style?.axisWidth}
+                    tickLabelProps={{ fill: 'black', fontFamily: 'inherit', fontSize: 10 }}
+                    numTicks={3}
+                />
+                <Axis
+                    orientation="right"
+                    stroke={style?.axisColor}
+                    strokeWidth={style?.axisWidth}
                     tickFormat={(tick) => (yAxis?.formatter ? yAxis.formatter(tick) : formatNumber(tick))}
+                    tickLabelProps={{ fill: 'black', fontFamily: 'inherit', fontSize: 10 }}
+                    numTicks={3}
                 />
                 {lines.map((line, i) => (
-                    <LineSeries
-                        showPoints
-                        data={line.data.map((point) => ({ x: new Date(point.x).getTime().toString(), y: point.y }))}
-                        stroke={(style && style.lineColor) || lineColors[i]}
-                    />
+                    <>
+                        <LineSeries
+                            dataKey={line.name}
+                            data={line.data.map((point) => ({ x: new Date(point.x), y: point.y }))}
+                            stroke={(style && style.lineColor) || lineColors[i]}
+                            curve={curveMonotoneX}
+                            {...accessors}
+                        />
+                        <GlyphSeries
+                            dataKey={line.name}
+                            data={line.data.map((point) => ({ x: new Date(point.x), y: point.y }))}
+                            {...accessors}
+                        />
+                    </>
                 ))}
-                <CrossHair
-                    showHorizontalLine={false}
-                    fullHeight
-                    stroke={(style && style.crossHairLineColor) || '#D8D8D8'}
+                <StyledTooltip
+                    snapTooltipToDatumX
+                    showVerticalCrosshair
+                    showDatumGlyph
+                    verticalCrosshairStyle={{ stroke: '#D8D8D8', strokeDasharray: '5,2', strokeWidth: 1 }}
+                    renderTooltip={({ tooltipData }) =>
+                        tooltipData?.nearestDatum && (
+                            <div>
+                                <div>
+                                    {new Date(
+                                        Number(accessors.xAccessor(tooltipData.nearestDatum.datum)),
+                                    ).toDateString()}
+                                </div>
+                                <div>{accessors.yAccessor(tooltipData.nearestDatum.datum)}</div>
+                            </div>
+                        )
+                    }
                 />
             </XYChart>
             {!hideLegend && <Legend ordinalScale={ordinalColorScale} />}
diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/MarkdownViewer.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/MarkdownViewer.tsx
index 669e32520b038..579c58214aaa1 100644
--- a/datahub-web-react/src/app/entity/shared/components/legacy/MarkdownViewer.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/legacy/MarkdownViewer.tsx
@@ -76,6 +76,7 @@ export const MarkdownView = styled(MDEditor.Markdown)`
 export type Props = {
     source: string;
     limit?: number;
+    // eslint-disable-next-line react/no-unused-prop-types
     isCompact?: boolean;
     editable?: boolean;
     onEditClicked?: () => void;
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx
index db5b1a59759b1..e3955bc27b9d7 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx
@@ -16,7 +16,7 @@ const ChartTitle = styled(Typography.Text)`
 
 const ChartCard = styled(Card)<{ visible: boolean }>`
     box-shadow: ${(props) => props.theme.styles['box-shadow']};
-    visibility: ${(props) => (props.visible ? 'visible' : 'hidden')}; ;
+    visibility: ${(props) => (props.visible ? 'visible' : 'hidden')};
 `;
 
 type Point = {
@@ -87,7 +87,7 @@ export default function StatChart({ title, values, tickInterval: interval, dateR
                     }}
                     width={360}
                     height={300}
-                    yScale={{ type: 'linear', includeZero: false }}
+                    yScale={{ type: 'linear', zero: false }}
                     yAxis={yAxis}
                 />
             </ChartContainer>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/BooleanTimeline.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/BooleanTimeline.tsx
index cd6f08fc133cf..27db7b0e96612 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/BooleanTimeline.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/BooleanTimeline.tsx
@@ -1,9 +1,9 @@
 import React, { useMemo } from 'react';
 import { Popover } from 'antd';
-import { Bar } from '@vx/shape';
-import { Group } from '@vx/group';
-import { AxisBottom } from '@vx/axis';
-import { scaleUtc } from '@vx/scale';
+import { Bar } from '@visx/shape';
+import { Group } from '@visx/group';
+import { AxisBottom } from '@visx/axis';
+import { scaleUtc } from '@visx/scale';
 import { ANTD_GRAY } from '../../../constants';
 
 export type BooleanResult = {
diff --git a/datahub-web-react/src/app/lineage/ColumnNode.tsx b/datahub-web-react/src/app/lineage/ColumnNode.tsx
index bee79badf1ed6..bfec978c60106 100644
--- a/datahub-web-react/src/app/lineage/ColumnNode.tsx
+++ b/datahub-web-react/src/app/lineage/ColumnNode.tsx
@@ -1,6 +1,6 @@
 import React, { useContext, useState } from 'react';
 import styled from 'styled-components/macro';
-import { Group } from '@vx/group';
+import { Group } from '@visx/group';
 import { SchemaField } from '../../types.generated';
 import { downgradeV2FieldPath } from '../entity/dataset/profile/schema/utils/utils';
 import { NodeData } from './types';
diff --git a/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx b/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx
index 5f288c6e993ca..281d6381741f8 100644
--- a/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx
+++ b/datahub-web-react/src/app/lineage/LineageEntityColumns.tsx
@@ -1,5 +1,5 @@
 import React, { useContext, useEffect, useState } from 'react';
-import { Group } from '@vx/group';
+import { Group } from '@visx/group';
 import { Pagination } from 'antd';
 import styled from 'styled-components';
 import { NodeData, EntitySelectParams } from './types';
diff --git a/datahub-web-react/src/app/lineage/LineageEntityEdge.tsx b/datahub-web-react/src/app/lineage/LineageEntityEdge.tsx
index 1505ce71f7027..a0a7db63381df 100644
--- a/datahub-web-react/src/app/lineage/LineageEntityEdge.tsx
+++ b/datahub-web-react/src/app/lineage/LineageEntityEdge.tsx
@@ -4,9 +4,9 @@ import { ClockCircleOutlined, EyeOutlined } from '@ant-design/icons';
 import dayjs from 'dayjs';
 import LocalizedFormat from 'dayjs/plugin/localizedFormat';
 import styled from 'styled-components';
-import { Group } from '@vx/group';
-import { curveBasis } from '@vx/curve';
-import { LinePath } from '@vx/shape';
+import { Group } from '@visx/group';
+import { curveBasis } from '@visx/curve';
+import { LinePath } from '@visx/shape';
 import { VizEdge } from './types';
 import { ANTD_GRAY } from '../entity/shared/constants';
 
diff --git a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx
index f5be1d57db070..9b6475b648ca1 100644
--- a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx
+++ b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx
@@ -1,6 +1,6 @@
 import React, { useContext, useEffect, useMemo, useState } from 'react';
-import { Group } from '@vx/group';
-import { LinkHorizontal } from '@vx/shape';
+import { Group } from '@visx/group';
+import { LinkHorizontal } from '@visx/shape';
 import styled from 'styled-components';
 
 import { useEntityRegistry } from '../useEntityRegistry';
diff --git a/datahub-web-react/src/app/lineage/LineageTree.tsx b/datahub-web-react/src/app/lineage/LineageTree.tsx
index 5e74c7f072947..8b5de4e78ff17 100644
--- a/datahub-web-react/src/app/lineage/LineageTree.tsx
+++ b/datahub-web-react/src/app/lineage/LineageTree.tsx
@@ -1,5 +1,5 @@
 import React, { useContext, useEffect, useMemo, useState } from 'react';
-import { TransformMatrix } from '@vx/zoom/lib/types';
+import { TransformMatrix } from '@visx/zoom/lib/types';
 
 import { NodeData, EntitySelectParams, TreeProps, EntityAndType, FetchedEntity, UpdatedLineages } from './types';
 import LineageTreeNodeAndEdgeRenderer from './LineageTreeNodeAndEdgeRenderer';
diff --git a/datahub-web-react/src/app/lineage/LineageTreeNodeAndEdgeRenderer.tsx b/datahub-web-react/src/app/lineage/LineageTreeNodeAndEdgeRenderer.tsx
index 638a207999e2b..bec83c80107b3 100644
--- a/datahub-web-react/src/app/lineage/LineageTreeNodeAndEdgeRenderer.tsx
+++ b/datahub-web-react/src/app/lineage/LineageTreeNodeAndEdgeRenderer.tsx
@@ -1,6 +1,6 @@
 import React, { useContext } from 'react';
-import { Group } from '@vx/group';
-import { TransformMatrix } from '@vx/zoom/lib/types';
+import { Group } from '@visx/group';
+import { TransformMatrix } from '@visx/zoom/lib/types';
 
 import { NodeData, EntitySelectParams, TreeProps, VizNode, VizEdge, EntityAndType, UpdatedLineages } from './types';
 import LineageEntityNode from './LineageEntityNode';
diff --git a/datahub-web-react/src/app/lineage/LineageViz.tsx b/datahub-web-react/src/app/lineage/LineageViz.tsx
index 05b269ccabb47..d70bf72e4cbaf 100644
--- a/datahub-web-react/src/app/lineage/LineageViz.tsx
+++ b/datahub-web-react/src/app/lineage/LineageViz.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { useWindowSize } from '@react-hook/window-size';
-import { Zoom } from '@vx/zoom';
+import { Zoom } from '@visx/zoom';
 
 import { TreeProps } from './types';
 import LineageVizInsideZoom from './LineageVizInsideZoom';
@@ -42,7 +42,7 @@ export default function LineageViz({
             scaleXMax={2}
             scaleYMin={1 / 8}
             scaleYMax={2}
-            transformMatrix={initialTransform}
+            initialTransformMatrix={initialTransform}
         >
             {(zoom) => (
                 <LineageVizInsideZoom
diff --git a/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx b/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx
index 315036a4f392d..ab86dfcb335f0 100644
--- a/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx
+++ b/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx
@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react';
 import { PlusOutlined, MinusOutlined } from '@ant-design/icons';
 import styled from 'styled-components/macro';
 import { Button } from 'antd';
-import { ProvidedZoom, TransformMatrix } from '@vx/zoom/lib/types';
+import { ProvidedZoom, TransformMatrix } from '@visx/zoom/lib/types';
 
 import { ColumnEdge, EntityAndType, EntitySelectParams, FetchedEntity } from './types';
 import { LineageExplorerContext } from './utils/LineageExplorerContext';
@@ -41,7 +41,7 @@ type Props = {
     onEntityCenter: (EntitySelectParams) => void;
     onLineageExpand: (data: EntityAndType) => void;
     selectedEntity?: EntitySelectParams;
-    zoom: ProvidedZoom & {
+    zoom: ProvidedZoom<any> & {
         transformMatrix: TransformMatrix;
         isDragging: boolean;
     };
diff --git a/datahub-web-react/src/app/lineage/LineageVizRootSvg.tsx b/datahub-web-react/src/app/lineage/LineageVizRootSvg.tsx
index 0d4bd483358aa..434fb1562bc2a 100644
--- a/datahub-web-react/src/app/lineage/LineageVizRootSvg.tsx
+++ b/datahub-web-react/src/app/lineage/LineageVizRootSvg.tsx
@@ -1,4 +1,4 @@
-import { ProvidedZoom, TransformMatrix } from '@vx/zoom/lib/types';
+import { ProvidedZoom, TransformMatrix } from '@visx/zoom/lib/types';
 import React, { SVGProps, useEffect, useMemo, useState } from 'react';
 import styled from 'styled-components/macro';
 
@@ -15,7 +15,7 @@ type Props = {
     onEntityCenter: (EntitySelectParams) => void;
     onLineageExpand: (data: EntityAndType) => void;
     selectedEntity?: EntitySelectParams;
-    zoom: ProvidedZoom & {
+    zoom: ProvidedZoom<any> & {
         transformMatrix: TransformMatrix;
         isDragging: boolean;
     };
diff --git a/datahub-web-react/src/app/lineage/NodeColumnsHeader.tsx b/datahub-web-react/src/app/lineage/NodeColumnsHeader.tsx
index 4b8c4303cec5a..ea2ae4a1a6ee6 100644
--- a/datahub-web-react/src/app/lineage/NodeColumnsHeader.tsx
+++ b/datahub-web-react/src/app/lineage/NodeColumnsHeader.tsx
@@ -1,6 +1,6 @@
 import React, { useContext, useState } from 'react';
 import { Button, Input } from 'antd';
-import { Group } from '@vx/group';
+import { Group } from '@visx/group';
 import styled from 'styled-components';
 import { DownOutlined, SearchOutlined, UpOutlined } from '@ant-design/icons';
 import { blue } from '@ant-design/colors';
diff --git a/datahub-web-react/src/app/lineage/__tests__/LineageTree.test.tsx b/datahub-web-react/src/app/lineage/__tests__/LineageTree.test.tsx
index 64b6e4f8df520..594a7ade24264 100644
--- a/datahub-web-react/src/app/lineage/__tests__/LineageTree.test.tsx
+++ b/datahub-web-react/src/app/lineage/__tests__/LineageTree.test.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { render } from '@testing-library/react';
-import { Zoom } from '@vx/zoom';
+import { Zoom } from '@visx/zoom';
 import { MockedProvider } from '@apollo/client/testing';
 import {
     dataset3WithLineage,
diff --git a/datahub-web-react/src/app/lineage/__tests__/adjustVXTreeLayout.test.tsx b/datahub-web-react/src/app/lineage/__tests__/adjustVXTreeLayout.test.tsx
index cd97ba11bc50e..27d8d72d2375b 100644
--- a/datahub-web-react/src/app/lineage/__tests__/adjustVXTreeLayout.test.tsx
+++ b/datahub-web-react/src/app/lineage/__tests__/adjustVXTreeLayout.test.tsx
@@ -1,5 +1,5 @@
 import React from 'react';
-import { Tree, hierarchy } from '@vx/hierarchy';
+import { Tree, hierarchy } from '@visx/hierarchy';
 import { render } from '@testing-library/react';
 
 import {
diff --git a/datahub-web-react/src/app/lineage/utils/adjustVXTreeLayout.ts b/datahub-web-react/src/app/lineage/utils/adjustVXTreeLayout.ts
index 307f98b688102..3fde422494555 100644
--- a/datahub-web-react/src/app/lineage/utils/adjustVXTreeLayout.ts
+++ b/datahub-web-react/src/app/lineage/utils/adjustVXTreeLayout.ts
@@ -1,4 +1,4 @@
-import { HierarchyPointNode } from '@vx/hierarchy/lib/types';
+import { HierarchyPointNode } from '@visx/hierarchy/lib/types';
 import { NodeData, Direction } from '../types';
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 import { width as nodeWidth } from '../constants';
diff --git a/datahub-web-react/src/app/shared/share/items/EmailMenuItem.tsx b/datahub-web-react/src/app/shared/share/items/EmailMenuItem.tsx
index e172e961c4fa0..f65e25b140aeb 100644
--- a/datahub-web-react/src/app/shared/share/items/EmailMenuItem.tsx
+++ b/datahub-web-react/src/app/shared/share/items/EmailMenuItem.tsx
@@ -2,7 +2,7 @@ import React, { useState } from 'react';
 import styled from 'styled-components';
 import { Tooltip } from 'antd';
 import { CheckOutlined, MailOutlined } from '@ant-design/icons';
-import EmailShare from 'react-email-share-link';
+import qs from 'query-string';
 import MenuItem from 'antd/lib/menu/MenuItem';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 
@@ -31,26 +31,29 @@ export default function EmailMenuItem({ urn, name, type, key }: EmailMenuItemPro
     const [isClicked, setIsClicked] = useState(false);
     const linkText = window.location.href;
 
+    const link = qs.stringifyUrl({
+        url: 'mailto:',
+        query: {
+            subject: `${name} | ${type}`,
+            body: `Check out this ${type} on DataHub: ${linkText}. Urn: ${urn}`,
+        },
+    });
+
     return (
         <StyledMenuItem
             key={key}
             onClick={() => {
-                navigator.clipboard.writeText(urn);
                 setIsClicked(true);
             }}
         >
-            <EmailShare subject={`${name} | ${type}`} body={`Check out this ${type} on DataHub: ${linkText}`}>
-                {(link) => (
-                    <Tooltip title={`Share this ${type} via email`}>
-                        {isClicked ? <CheckOutlined /> : <MailOutlined />}
-                        <TextSpan>
-                            <a href={link} target="_blank" rel="noreferrer" style={{ color: 'inherit' }}>
-                                <b>Email</b>
-                            </a>
-                        </TextSpan>
-                    </Tooltip>
-                )}
-            </EmailShare>
+            <Tooltip title={`Share this ${type} via email`}>
+                {isClicked ? <CheckOutlined /> : <MailOutlined />}
+                <TextSpan>
+                    <a href={link} target="_blank" rel="noreferrer" style={{ color: 'inherit' }}>
+                        <b>Email</b>
+                    </a>
+                </TextSpan>
+            </Tooltip>
         </StyledMenuItem>
     );
 }
diff --git a/datahub-web-react/src/conf/theme/styled-components.d.ts b/datahub-web-react/src/conf/theme/styled-components.d.ts
index 220f9853572a9..3f9c7c4a8348d 100644
--- a/datahub-web-react/src/conf/theme/styled-components.d.ts
+++ b/datahub-web-react/src/conf/theme/styled-components.d.ts
@@ -1,6 +1,6 @@
 import { Theme } from './types';
 
 declare module 'styled-components' {
-    // eslint:disable-next-line  @typescript-eslint/no-empty-interface
+    // eslint-disable-next-line @typescript-eslint/no-empty-interface
     export interface DefaultTheme extends Theme {}
 }
diff --git a/datahub-web-react/src/graphql-mock/server.ts b/datahub-web-react/src/graphql-mock/server.ts
index a4896e7290548..678527599238b 100644
--- a/datahub-web-react/src/graphql-mock/server.ts
+++ b/datahub-web-react/src/graphql-mock/server.ts
@@ -69,6 +69,7 @@ export function makeServerForCypress() {
             routes() {
                 for (const domain of ['/*', ...otherDomains]) {
                     for (const method of methods) {
+                        // eslint-disable-next-line @typescript-eslint/no-loop-func
                         this[method](`${domain}`, async (_schema, request) => {
                             // eslint-disable-next-line @typescript-eslint/ban-ts-comment
                             // @ts-ignore
diff --git a/datahub-web-react/tsconfig.json b/datahub-web-react/tsconfig.json
index e10aca8d5c7b2..760c992f2ca3a 100644
--- a/datahub-web-react/tsconfig.json
+++ b/datahub-web-react/tsconfig.json
@@ -1,6 +1,6 @@
 {
     "compilerOptions": {
-        "target": "es5",
+        "target": "es2017",
         "lib": ["dom", "dom.iterable", "esnext"],
         "allowJs": true,
         "skipLibCheck": true,
@@ -17,5 +17,5 @@
         "noEmit": true,
         "jsx": "react-jsx"
     },
-    "include": ["src", "src/conf/theme/styled-components.d.ts"]
+    "include": ["src", "src/conf/theme/styled-components.d.ts", "craco.config.js", ".eslintrc.js"]
 }
diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
index ce0f2f514dad1..fbc800c93c460 100644
--- a/datahub-web-react/yarn.lock
+++ b/datahub-web-react/yarn.lock
@@ -2,12 +2,17 @@
 # yarn lockfile v1
 
 
-"@ampproject/remapping@^2.1.0":
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.0.tgz#56c133824780de3174aed5ab6834f3026790154d"
-  integrity sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==
+"@alloc/quick-lru@^5.2.0":
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/@alloc/quick-lru/-/quick-lru-5.2.0.tgz#7bf68b20c0a350f936915fcae06f58e32007ce30"
+  integrity sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==
+
+"@ampproject/remapping@^2.2.0":
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630"
+  integrity sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==
   dependencies:
-    "@jridgewell/gen-mapping" "^0.1.0"
+    "@jridgewell/gen-mapping" "^0.3.0"
     "@jridgewell/trace-mapping" "^0.3.9"
 
 "@analytics/amplitude@0.0.3":
@@ -94,28 +99,12 @@
   dependencies:
     "@ctrl/tinycolor" "^3.3.1"
 
-"@ant-design/icons-svg@^4.0.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@ant-design/icons-svg/-/icons-svg-4.1.0.tgz#480b025f4b20ef7fe8f47d4a4846e4fee84ea06c"
-  integrity sha512-Fi03PfuUqRs76aI3UWYpP864lkrfPo0hluwGqh7NJdLhvH4iRDc3jbJqZIvRDLHKbXrvAfPPV3+zjUccfFvWOQ==
-
 "@ant-design/icons-svg@^4.2.1":
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/@ant-design/icons-svg/-/icons-svg-4.2.1.tgz#8630da8eb4471a4aabdaed7d1ff6a97dcb2cf05a"
   integrity sha512-EB0iwlKDGpG93hW8f85CTJTs4SvMX7tt5ceupvhALp1IF44SeUFOMhKUOYqpsoYWQKAOuTRDMqn75rEaKDp0Xw==
 
-"@ant-design/icons@^4.3.0":
-  version "4.6.2"
-  resolved "https://registry.yarnpkg.com/@ant-design/icons/-/icons-4.6.2.tgz#290f2e8cde505ab081fda63e511e82d3c48be982"
-  integrity sha512-QsBG2BxBYU/rxr2eb8b2cZ4rPKAPBpzAR+0v6rrZLp/lnyvflLH3tw1vregK+M7aJauGWjIGNdFmUfpAOtw25A==
-  dependencies:
-    "@ant-design/colors" "^6.0.0"
-    "@ant-design/icons-svg" "^4.0.0"
-    "@babel/runtime" "^7.11.2"
-    classnames "^2.2.6"
-    rc-util "^5.9.4"
-
-"@ant-design/icons@^4.7.0":
+"@ant-design/icons@^4.3.0", "@ant-design/icons@^4.7.0":
   version "4.7.0"
   resolved "https://registry.yarnpkg.com/@ant-design/icons/-/icons-4.7.0.tgz#8c3cbe0a556ba92af5dc7d1e70c0b25b5179af0f"
   integrity sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==
@@ -137,6 +126,15 @@
     lodash "^4.17.21"
     resize-observer-polyfill "^1.5.1"
 
+"@apideck/better-ajv-errors@^0.3.1":
+  version "0.3.6"
+  resolved "https://registry.yarnpkg.com/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz#957d4c28e886a64a8141f7522783be65733ff097"
+  integrity sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA==
+  dependencies:
+    json-schema "^0.4.0"
+    jsonpointer "^5.0.0"
+    leven "^3.1.0"
+
 "@apollo/client@^3.3.19":
   version "3.3.19"
   resolved "https://registry.yarnpkg.com/@apollo/client/-/client-3.3.19.tgz#f1172dc9b9d7eae04c8940b047fd3b452ef92d2c"
@@ -186,549 +184,262 @@
     signedsource "^1.0.0"
     yargs "^15.3.1"
 
-"@babel/code-frame@7.10.4":
-  version "7.10.4"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a"
-  integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==
-  dependencies:
-    "@babel/highlight" "^7.10.4"
-
-"@babel/code-frame@7.12.11":
-  version "7.12.11"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f"
-  integrity sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==
-  dependencies:
-    "@babel/highlight" "^7.10.4"
-
-"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.5.5":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.13.tgz#dcfc826beef65e75c50e21d3837d7d95798dd658"
-  integrity sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g==
-  dependencies:
-    "@babel/highlight" "^7.12.13"
-
-"@babel/code-frame@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.14.5.tgz#23b08d740e83f49c5e59945fbf1b43e80bbf4edb"
-  integrity sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM+vOtCS5ndmJicPJhKAwYRI6UfFw==
-  dependencies:
-    "@babel/highlight" "^7.14.5"
-
-"@babel/code-frame@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a"
-  integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==
-  dependencies:
-    "@babel/highlight" "^7.18.6"
-
-"@babel/compat-data@^7.12.1", "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.13.15", "@babel/compat-data@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.0.tgz#a901128bce2ad02565df95e6ecbf195cf9465919"
-  integrity sha512-vu9V3uMM/1o5Hl5OekMUowo3FqXLJSw+s+66nt0fSWVWTtmosdzn45JHOB3cPtZoe6CTBDzvSw0RdOY85Q37+Q==
-
-"@babel/compat-data@^7.20.5":
-  version "7.20.10"
-  resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.20.10.tgz#9d92fa81b87542fff50e848ed585b4212c1d34ec"
-  integrity sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg==
-
-"@babel/core@7.12.3":
-  version "7.12.3"
-  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.3.tgz#1b436884e1e3bff6fb1328dc02b208759de92ad8"
-  integrity sha512-0qXcZYKZp3/6N2jKYVxZv0aNCsxTSVCiK72DTiTYZAu7sjg73W0/aynWjMbiGd87EQL4WyA8reiJVh92AVla9g==
-  dependencies:
-    "@babel/code-frame" "^7.10.4"
-    "@babel/generator" "^7.12.1"
-    "@babel/helper-module-transforms" "^7.12.1"
-    "@babel/helpers" "^7.12.1"
-    "@babel/parser" "^7.12.3"
-    "@babel/template" "^7.10.4"
-    "@babel/traverse" "^7.12.1"
-    "@babel/types" "^7.12.1"
-    convert-source-map "^1.7.0"
-    debug "^4.1.0"
-    gensync "^1.0.0-beta.1"
-    json5 "^2.1.2"
-    lodash "^4.17.19"
-    resolve "^1.3.2"
-    semver "^5.4.1"
-    source-map "^0.5.0"
-
-"@babel/core@^7.1.0", "@babel/core@^7.12.3", "@babel/core@^7.7.5", "@babel/core@^7.8.4":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.14.3.tgz#5395e30405f0776067fbd9cf0884f15bfb770a38"
-  integrity sha512-jB5AmTKOCSJIZ72sd78ECEhuPiDMKlQdDI/4QRI6lzYATx5SSogS1oQA2AoPecRCknm30gHi2l+QVvNUu3wZAg==
-  dependencies:
-    "@babel/code-frame" "^7.12.13"
-    "@babel/generator" "^7.14.3"
-    "@babel/helper-compilation-targets" "^7.13.16"
-    "@babel/helper-module-transforms" "^7.14.2"
-    "@babel/helpers" "^7.14.0"
-    "@babel/parser" "^7.14.3"
-    "@babel/template" "^7.12.13"
-    "@babel/traverse" "^7.14.2"
-    "@babel/types" "^7.14.2"
-    convert-source-map "^1.7.0"
-    debug "^4.1.0"
-    gensync "^1.0.0-beta.2"
-    json5 "^2.1.2"
-    semver "^6.3.0"
-    source-map "^0.5.0"
-
-"@babel/core@^7.14.0":
-  version "7.20.12"
-  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.20.12.tgz#7930db57443c6714ad216953d1356dac0eb8496d"
-  integrity sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==
-  dependencies:
-    "@ampproject/remapping" "^2.1.0"
-    "@babel/code-frame" "^7.18.6"
-    "@babel/generator" "^7.20.7"
-    "@babel/helper-compilation-targets" "^7.20.7"
-    "@babel/helper-module-transforms" "^7.20.11"
-    "@babel/helpers" "^7.20.7"
-    "@babel/parser" "^7.20.7"
-    "@babel/template" "^7.20.7"
-    "@babel/traverse" "^7.20.12"
-    "@babel/types" "^7.20.7"
+"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.22.5", "@babel/code-frame@^7.8.3":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.5.tgz#234d98e1551960604f1246e6475891a570ad5658"
+  integrity sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ==
+  dependencies:
+    "@babel/highlight" "^7.22.5"
+
+"@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.5", "@babel/compat-data@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.5.tgz#b1f6c86a02d85d2dd3368a2b67c09add8cd0c255"
+  integrity sha512-4Jc/YuIaYqKnDDz892kPIledykKg12Aw1PYX5i/TY28anJtacvM1Rrr8wbieB9GfEJwlzqT0hUEao0CxEebiDA==
+
+"@babel/core@^7.1.0", "@babel/core@^7.11.1", "@babel/core@^7.12.3", "@babel/core@^7.14.0", "@babel/core@^7.16.0", "@babel/core@^7.7.2", "@babel/core@^7.8.0":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.5.tgz#d67d9747ecf26ee7ecd3ebae1ee22225fe902a89"
+  integrity sha512-SBuTAjg91A3eKOvD+bPEz3LlhHZRNu1nFOVts9lzDJTXshHTjII0BAtDS3Y2DAkdZdDKWVZGVwkDfc4Clxn1dg==
+  dependencies:
+    "@ampproject/remapping" "^2.2.0"
+    "@babel/code-frame" "^7.22.5"
+    "@babel/generator" "^7.22.5"
+    "@babel/helper-compilation-targets" "^7.22.5"
+    "@babel/helper-module-transforms" "^7.22.5"
+    "@babel/helpers" "^7.22.5"
+    "@babel/parser" "^7.22.5"
+    "@babel/template" "^7.22.5"
+    "@babel/traverse" "^7.22.5"
+    "@babel/types" "^7.22.5"
     convert-source-map "^1.7.0"
     debug "^4.1.0"
     gensync "^1.0.0-beta.2"
     json5 "^2.2.2"
     semver "^6.3.0"
 
-"@babel/generator@^7.12.1", "@babel/generator@^7.12.13", "@babel/generator@^7.14.2", "@babel/generator@^7.14.3":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.14.3.tgz#0c2652d91f7bddab7cccc6ba8157e4f40dcedb91"
-  integrity sha512-bn0S6flG/j0xtQdz3hsjJ624h3W0r3llttBMfyHX3YrZ/KtLYr15bjA0FXkgW7FpvrDuTuElXeVjiKlYRpnOFA==
+"@babel/eslint-parser@^7.16.3":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/eslint-parser/-/eslint-parser-7.22.5.tgz#fa032503b9e2d188e25b1b95d29e8b8431042d78"
+  integrity sha512-C69RWYNYtrgIRE5CmTd77ZiLDXqgBipahJc/jHP3sLcAGj6AJzxNIuKNpVnICqbyK7X3pFUfEvL++rvtbQpZkQ==
   dependencies:
-    "@babel/types" "^7.14.2"
-    jsesc "^2.5.1"
-    source-map "^0.5.0"
+    "@nicolo-ribaudo/eslint-scope-5-internals" "5.1.1-v1"
+    eslint-visitor-keys "^2.1.0"
+    semver "^6.3.0"
 
-"@babel/generator@^7.14.0", "@babel/generator@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.7.tgz#f8ef57c8242665c5929fe2e8d82ba75460187b4a"
-  integrity sha512-7wqMOJq8doJMZmP4ApXTzLxSr7+oO2jroJURrVEp6XShrQUObV8Tq/D0NCcoYg2uHqUrjzO0zwBjoYzelxK+sw==
+"@babel/generator@^7.12.13", "@babel/generator@^7.14.0", "@babel/generator@^7.22.5", "@babel/generator@^7.7.2":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.22.5.tgz#1e7bf768688acfb05cf30b2369ef855e82d984f7"
+  integrity sha512-+lcUbnTRhd0jOewtFSedLyiPsD5tswKkbgcezOqqWFUVNEwoUTlpPOBmvhG7OXWLR4jMdv0czPGH5XbflnD1EA==
   dependencies:
-    "@babel/types" "^7.20.7"
+    "@babel/types" "^7.22.5"
     "@jridgewell/gen-mapping" "^0.3.2"
+    "@jridgewell/trace-mapping" "^0.3.17"
     jsesc "^2.5.1"
 
-"@babel/generator@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.14.5.tgz#848d7b9f031caca9d0cd0af01b063f226f52d785"
-  integrity sha512-y3rlP+/G25OIX3mYKKIOlQRcqj7YgrvHxOLbVmyLJ9bPmi5ttvUmpydVjcFjZphOktWuA7ovbx91ECloWTfjIA==
-  dependencies:
-    "@babel/types" "^7.14.5"
-    jsesc "^2.5.1"
-    source-map "^0.5.0"
-
-"@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.10.4", "@babel/helper-annotate-as-pure@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.13.tgz#0f58e86dfc4bb3b1fcd7db806570e177d439b6ab"
-  integrity sha512-7YXfX5wQ5aYM/BOlbSccHDbuXXFPxeoUmfWtz8le2yTkTZc+BxsiEnENFoi2SlmA8ewDkG2LgIMIVzzn2h8kfw==
+"@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.18.6", "@babel/helper-annotate-as-pure@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz#e7f06737b197d580a01edf75d97e2c8be99d3882"
+  integrity sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==
   dependencies:
-    "@babel/types" "^7.12.13"
+    "@babel/types" "^7.22.5"
 
-"@babel/helper-annotate-as-pure@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb"
-  integrity sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==
+"@babel/helper-builder-binary-assignment-operator-visitor@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.22.5.tgz#a3f4758efdd0190d8927fcffd261755937c71878"
+  integrity sha512-m1EP3lVOPptR+2DwD125gziZNcmoNSHGmJROKoy87loWUQyJaVXDgpmruWqDARZSmtYQ+Dl25okU8+qhVzuykw==
   dependencies:
-    "@babel/types" "^7.18.6"
+    "@babel/types" "^7.22.5"
 
-"@babel/helper-builder-binary-assignment-operator-visitor@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.12.13.tgz#6bc20361c88b0a74d05137a65cac8d3cbf6f61fc"
-  integrity sha512-CZOv9tGphhDRlVjVkAgm8Nhklm9RzSmWpX2my+t7Ua/KT616pEzXsQCjinzvkRvHWJ9itO4f296efroX23XCMA==
-  dependencies:
-    "@babel/helper-explode-assignable-expression" "^7.12.13"
-    "@babel/types" "^7.12.13"
-
-"@babel/helper-compilation-targets@^7.12.1", "@babel/helper-compilation-targets@^7.13.0", "@babel/helper-compilation-targets@^7.13.16":
-  version "7.13.16"
-  resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.13.16.tgz#6e91dccf15e3f43e5556dffe32d860109887563c"
-  integrity sha512-3gmkYIrpqsLlieFwjkGgLaSHmhnvlAYzZLlYVjlW+QwI+1zE17kGxuJGmIqDQdYp56XdmGeD+Bswx0UTyG18xA==
-  dependencies:
-    "@babel/compat-data" "^7.13.15"
-    "@babel/helper-validator-option" "^7.12.17"
-    browserslist "^4.14.5"
-    semver "^6.3.0"
-
-"@babel/helper-compilation-targets@^7.18.9", "@babel/helper-compilation-targets@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz#a6cd33e93629f5eb473b021aac05df62c4cd09bb"
-  integrity sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ==
+"@babel/helper-compilation-targets@^7.17.7", "@babel/helper-compilation-targets@^7.20.7", "@babel/helper-compilation-targets@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.5.tgz#fc7319fc54c5e2fa14b2909cf3c5fd3046813e02"
+  integrity sha512-Ji+ywpHeuqxB8WDxraCiqR0xfhYjiDE/e6k7FuIaANnoOFxAHskHChz4vA1mJC9Lbm01s1PVAGhQY4FUKSkGZw==
   dependencies:
-    "@babel/compat-data" "^7.20.5"
-    "@babel/helper-validator-option" "^7.18.6"
+    "@babel/compat-data" "^7.22.5"
+    "@babel/helper-validator-option" "^7.22.5"
     browserslist "^4.21.3"
     lru-cache "^5.1.1"
     semver "^6.3.0"
 
-"@babel/helper-create-class-features-plugin@^7.12.1", "@babel/helper-create-class-features-plugin@^7.13.0", "@babel/helper-create-class-features-plugin@^7.14.0", "@babel/helper-create-class-features-plugin@^7.14.3":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.3.tgz#832111bcf4f57ca57a4c5b1a000fc125abc6554a"
-  integrity sha512-BnEfi5+6J2Lte9LeiL6TxLWdIlEv9Woacc1qXzXBgbikcOzMRM2Oya5XGg/f/ngotv1ej2A/b+3iJH8wbS1+lQ==
-  dependencies:
-    "@babel/helper-annotate-as-pure" "^7.12.13"
-    "@babel/helper-function-name" "^7.14.2"
-    "@babel/helper-member-expression-to-functions" "^7.13.12"
-    "@babel/helper-optimise-call-expression" "^7.12.13"
-    "@babel/helper-replace-supers" "^7.14.3"
-    "@babel/helper-split-export-declaration" "^7.12.13"
-
-"@babel/helper-create-class-features-plugin@^7.18.6":
-  version "7.20.12"
-  resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.20.12.tgz#4349b928e79be05ed2d1643b20b99bb87c503819"
-  integrity sha512-9OunRkbT0JQcednL0UFvbfXpAsUXiGjUk0a7sN8fUXX7Mue79cUSMjHGDRRi/Vz9vYlpIhLV5fMD5dKoMhhsNQ==
-  dependencies:
-    "@babel/helper-annotate-as-pure" "^7.18.6"
-    "@babel/helper-environment-visitor" "^7.18.9"
-    "@babel/helper-function-name" "^7.19.0"
-    "@babel/helper-member-expression-to-functions" "^7.20.7"
-    "@babel/helper-optimise-call-expression" "^7.18.6"
-    "@babel/helper-replace-supers" "^7.20.7"
-    "@babel/helper-skip-transparent-expression-wrappers" "^7.20.0"
-    "@babel/helper-split-export-declaration" "^7.18.6"
+"@babel/helper-create-class-features-plugin@^7.18.6", "@babel/helper-create-class-features-plugin@^7.21.0", "@babel/helper-create-class-features-plugin@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.22.5.tgz#2192a1970ece4685fbff85b48da2c32fcb130b7c"
+  integrity sha512-xkb58MyOYIslxu3gKmVXmjTtUPvBU4odYzbiIQbWwLKIHCsx6UGZGX6F1IznMFVnDdirseUZopzN+ZRt8Xb33Q==
+  dependencies:
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-function-name" "^7.22.5"
+    "@babel/helper-member-expression-to-functions" "^7.22.5"
+    "@babel/helper-optimise-call-expression" "^7.22.5"
+    "@babel/helper-replace-supers" "^7.22.5"
+    "@babel/helper-skip-transparent-expression-wrappers" "^7.22.5"
+    "@babel/helper-split-export-declaration" "^7.22.5"
+    semver "^6.3.0"
 
-"@babel/helper-create-regexp-features-plugin@^7.12.13":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.14.3.tgz#149aa6d78c016e318c43e2409a0ae9c136a86688"
-  integrity sha512-JIB2+XJrb7v3zceV2XzDhGIB902CmKGSpSl4q2C6agU9SNLG/2V1RtFRGPG1Ajh9STj3+q6zJMOC+N/pp2P9DA==
+"@babel/helper-create-regexp-features-plugin@^7.12.13", "@babel/helper-create-regexp-features-plugin@^7.18.6", "@babel/helper-create-regexp-features-plugin@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.22.5.tgz#bb2bf0debfe39b831986a4efbf4066586819c6e4"
+  integrity sha512-1VpEFOIbMRaXyDeUwUfmTIxExLwQ+zkW+Bh5zXpApA3oQedBx9v/updixWxnx/bZpKw7u8VxWjb/qWpIcmPq8A==
   dependencies:
-    "@babel/helper-annotate-as-pure" "^7.12.13"
-    regexpu-core "^4.7.1"
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    regexpu-core "^5.3.1"
+    semver "^6.3.0"
 
-"@babel/helper-define-polyfill-provider@^0.2.2":
-  version "0.2.3"
-  resolved "https://registry.yarnpkg.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.2.3.tgz#0525edec5094653a282688d34d846e4c75e9c0b6"
-  integrity sha512-RH3QDAfRMzj7+0Nqu5oqgO5q9mFtQEVvCRsi8qCEfzLR9p2BHfn5FzhSB2oj1fF7I2+DcTORkYaQ6aTR9Cofew==
+"@babel/helper-define-polyfill-provider@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.4.0.tgz#487053f103110f25b9755c5980e031e93ced24d8"
+  integrity sha512-RnanLx5ETe6aybRi1cO/edaRH+bNYWaryCEmjDDYyNr4wnSzyOp8T0dWipmqVHKEY3AbVKUom50AKSlj1zmKbg==
   dependencies:
-    "@babel/helper-compilation-targets" "^7.13.0"
-    "@babel/helper-module-imports" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/traverse" "^7.13.0"
+    "@babel/helper-compilation-targets" "^7.17.7"
+    "@babel/helper-plugin-utils" "^7.16.7"
     debug "^4.1.1"
     lodash.debounce "^4.0.8"
     resolve "^1.14.2"
     semver "^6.1.2"
 
-"@babel/helper-environment-visitor@^7.18.9":
-  version "7.18.9"
-  resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz#0c0cee9b35d2ca190478756865bb3528422f51be"
-  integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==
-
-"@babel/helper-explode-assignable-expression@^7.12.13":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.13.0.tgz#17b5c59ff473d9f956f40ef570cf3a76ca12657f"
-  integrity sha512-qS0peLTDP8kOisG1blKbaoBg/o9OSa1qoumMjTK5pM+KDTtpxpsiubnCGP34vK8BXGcb2M9eigwgvoJryrzwWA==
-  dependencies:
-    "@babel/types" "^7.13.0"
-
-"@babel/helper-function-name@^7.12.13", "@babel/helper-function-name@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.14.2.tgz#397688b590760b6ef7725b5f0860c82427ebaac2"
-  integrity sha512-NYZlkZRydxw+YT56IlhIcS8PAhb+FEUiOzuhFTfqDyPmzAhRge6ua0dQYT/Uh0t/EDHq05/i+e5M2d4XvjgarQ==
-  dependencies:
-    "@babel/helper-get-function-arity" "^7.12.13"
-    "@babel/template" "^7.12.13"
-    "@babel/types" "^7.14.2"
-
-"@babel/helper-function-name@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.14.5.tgz#89e2c474972f15d8e233b52ee8c480e2cfcd50c4"
-  integrity sha512-Gjna0AsXWfFvrAuX+VKcN/aNNWonizBj39yGwUzVDVTlMYJMK2Wp6xdpy72mfArFq5uK+NOuexfzZlzI1z9+AQ==
-  dependencies:
-    "@babel/helper-get-function-arity" "^7.14.5"
-    "@babel/template" "^7.14.5"
-    "@babel/types" "^7.14.5"
-
-"@babel/helper-function-name@^7.18.9", "@babel/helper-function-name@^7.19.0":
-  version "7.19.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz#941574ed5390682e872e52d3f38ce9d1bef4648c"
-  integrity sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w==
-  dependencies:
-    "@babel/template" "^7.18.10"
-    "@babel/types" "^7.19.0"
-
-"@babel/helper-get-function-arity@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583"
-  integrity sha512-DjEVzQNz5LICkzN0REdpD5prGoidvbdYk1BVgRUOINaWJP2t6avB27X1guXK1kXNrX0WMfsrm1A/ZBthYuIMQg==
-  dependencies:
-    "@babel/types" "^7.12.13"
-
-"@babel/helper-get-function-arity@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz#25fbfa579b0937eee1f3b805ece4ce398c431815"
-  integrity sha512-I1Db4Shst5lewOM4V+ZKJzQ0JGGaZ6VY1jYvMghRjqs6DWgxLCIyFt30GlnKkfUeFLpJt2vzbMVEXVSXlIFYUg==
-  dependencies:
-    "@babel/types" "^7.14.5"
-
-"@babel/helper-hoist-variables@^7.13.0":
-  version "7.13.16"
-  resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.13.16.tgz#1b1651249e94b51f8f0d33439843e33e39775b30"
-  integrity sha512-1eMtTrXtrwscjcAeO4BVK+vvkxaLJSPFz1w1KLawz6HLNi9bPFGBNwwDyVfiu1Tv/vRRFYfoGaKhmAQPGPn5Wg==
-  dependencies:
-    "@babel/traverse" "^7.13.15"
-    "@babel/types" "^7.13.16"
-
-"@babel/helper-hoist-variables@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.14.5.tgz#e0dd27c33a78e577d7c8884916a3e7ef1f7c7f8d"
-  integrity sha512-R1PXiz31Uc0Vxy4OEOm07x0oSjKAdPPCh3tPivn/Eo8cvz6gveAeuyUUPB21Hoiif0uoPQSSdhIPS3352nvdyQ==
-  dependencies:
-    "@babel/types" "^7.14.5"
-
-"@babel/helper-hoist-variables@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678"
-  integrity sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==
-  dependencies:
-    "@babel/types" "^7.18.6"
-
-"@babel/helper-member-expression-to-functions@^7.13.12":
-  version "7.13.12"
-  resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.13.12.tgz#dfe368f26d426a07299d8d6513821768216e6d72"
-  integrity sha512-48ql1CLL59aKbU94Y88Xgb2VFy7a95ykGRbJJaaVv+LX5U8wFpLfiGXJJGUozsmA1oEh/o5Bp60Voq7ACyA/Sw==
-  dependencies:
-    "@babel/types" "^7.13.12"
-
-"@babel/helper-member-expression-to-functions@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.20.7.tgz#a6f26e919582275a93c3aa6594756d71b0bb7f05"
-  integrity sha512-9J0CxJLq315fEdi4s7xK5TQaNYjZw+nDVpVqr1axNGKzdrdwYBD5b4uKv3n75aABG0rCCTK8Im8Ww7eYfMrZgw==
-  dependencies:
-    "@babel/types" "^7.20.7"
-
-"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.0.0-beta.49", "@babel/helper-module-imports@^7.12.1", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.13.12":
-  version "7.13.12"
-  resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.13.12.tgz#c6a369a6f3621cb25da014078684da9196b61977"
-  integrity sha512-4cVvR2/1B693IuOvSI20xqqa/+bl7lqAMR59R4iu39R9aOX8/JoYY1sFaNvUMyMBGnHdwvJgUrzNLoUZxXypxA==
-  dependencies:
-    "@babel/types" "^7.13.12"
-
-"@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz#1e3ebdbbd08aad1437b428c50204db13c5a3ca6e"
-  integrity sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==
-  dependencies:
-    "@babel/types" "^7.18.6"
-
-"@babel/helper-module-transforms@^7.12.1", "@babel/helper-module-transforms@^7.13.0", "@babel/helper-module-transforms@^7.14.0", "@babel/helper-module-transforms@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.14.2.tgz#ac1cc30ee47b945e3e0c4db12fa0c5389509dfe5"
-  integrity sha512-OznJUda/soKXv0XhpvzGWDnml4Qnwp16GN+D/kZIdLsWoHj05kyu8Rm5kXmMef+rVJZ0+4pSGLkeixdqNUATDA==
-  dependencies:
-    "@babel/helper-module-imports" "^7.13.12"
-    "@babel/helper-replace-supers" "^7.13.12"
-    "@babel/helper-simple-access" "^7.13.12"
-    "@babel/helper-split-export-declaration" "^7.12.13"
-    "@babel/helper-validator-identifier" "^7.14.0"
-    "@babel/template" "^7.12.13"
-    "@babel/traverse" "^7.14.2"
-    "@babel/types" "^7.14.2"
-
-"@babel/helper-module-transforms@^7.20.11":
-  version "7.20.11"
-  resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.20.11.tgz#df4c7af713c557938c50ea3ad0117a7944b2f1b0"
-  integrity sha512-uRy78kN4psmji1s2QtbtcCSaj/LILFDp0f/ymhpQH5QY3nljUZCaNWz9X1dEj/8MBdBEFECs7yRhKn8i7NjZgg==
-  dependencies:
-    "@babel/helper-environment-visitor" "^7.18.9"
-    "@babel/helper-module-imports" "^7.18.6"
-    "@babel/helper-simple-access" "^7.20.2"
-    "@babel/helper-split-export-declaration" "^7.18.6"
-    "@babel/helper-validator-identifier" "^7.19.1"
-    "@babel/template" "^7.20.7"
-    "@babel/traverse" "^7.20.10"
-    "@babel/types" "^7.20.7"
-
-"@babel/helper-optimise-call-expression@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.13.tgz#5c02d171b4c8615b1e7163f888c1c81c30a2aaea"
-  integrity sha512-BdWQhoVJkp6nVjB7nkFWcn43dkprYauqtk++Py2eaf/GRDFm5BxRqEIZCiHlZUGAVmtwKcsVL1dC68WmzeFmiA==
-  dependencies:
-    "@babel/types" "^7.12.13"
-
-"@babel/helper-optimise-call-expression@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz#9369aa943ee7da47edab2cb4e838acf09d290ffe"
-  integrity sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==
-  dependencies:
-    "@babel/types" "^7.18.6"
-
-"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.13.0", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.13.0.tgz#806526ce125aed03373bc416a828321e3a6a33af"
-  integrity sha512-ZPafIPSwzUlAoWT8DKs1W2VyF2gOWthGd5NGFMsBcMMol+ZhK+EQY/e6V96poa6PA/Bh+C9plWN0hXO1uB8AfQ==
-
-"@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.18.9", "@babel/helper-plugin-utils@^7.19.0", "@babel/helper-plugin-utils@^7.20.2":
-  version "7.20.2"
-  resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz#d1b9000752b18d0877cff85a5c376ce5c3121629"
-  integrity sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ==
-
-"@babel/helper-remap-async-to-generator@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.13.0.tgz#376a760d9f7b4b2077a9dd05aa9c3927cadb2209"
-  integrity sha512-pUQpFBE9JvC9lrQbpX0TmeNIy5s7GnZjna2lhhcHC7DzgBs6fWn722Y5cfwgrtrqc7NAJwMvOa0mKhq6XaE4jg==
-  dependencies:
-    "@babel/helper-annotate-as-pure" "^7.12.13"
-    "@babel/helper-wrap-function" "^7.13.0"
-    "@babel/types" "^7.13.0"
-
-"@babel/helper-replace-supers@^7.12.13", "@babel/helper-replace-supers@^7.13.12", "@babel/helper-replace-supers@^7.14.3":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.14.3.tgz#ca17b318b859d107f0e9b722d58cf12d94436600"
-  integrity sha512-Rlh8qEWZSTfdz+tgNV/N4gz1a0TMNwCUcENhMjHTHKp3LseYH5Jha0NSlyTQWMnjbYcwFt+bqAMqSLHVXkQ6UA==
-  dependencies:
-    "@babel/helper-member-expression-to-functions" "^7.13.12"
-    "@babel/helper-optimise-call-expression" "^7.12.13"
-    "@babel/traverse" "^7.14.2"
-    "@babel/types" "^7.14.2"
-
-"@babel/helper-replace-supers@^7.18.6", "@babel/helper-replace-supers@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.20.7.tgz#243ecd2724d2071532b2c8ad2f0f9f083bcae331"
-  integrity sha512-vujDMtB6LVfNW13jhlCrp48QNslK6JXi7lQG736HVbHz/mbf4Dc7tIRh1Xf5C0rF7BP8iiSxGMCmY6Ci1ven3A==
-  dependencies:
-    "@babel/helper-environment-visitor" "^7.18.9"
-    "@babel/helper-member-expression-to-functions" "^7.20.7"
-    "@babel/helper-optimise-call-expression" "^7.18.6"
-    "@babel/template" "^7.20.7"
-    "@babel/traverse" "^7.20.7"
-    "@babel/types" "^7.20.7"
-
-"@babel/helper-simple-access@^7.13.12":
-  version "7.13.12"
-  resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.13.12.tgz#dd6c538afb61819d205a012c31792a39c7a5eaf6"
-  integrity sha512-7FEjbrx5SL9cWvXioDbnlYTppcZGuCY6ow3/D5vMggb2Ywgu4dMrpTJX0JdQAIcRRUElOIxF3yEooa9gUb9ZbA==
-  dependencies:
-    "@babel/types" "^7.13.12"
-
-"@babel/helper-simple-access@^7.20.2":
-  version "7.20.2"
-  resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz#0ab452687fe0c2cfb1e2b9e0015de07fc2d62dd9"
-  integrity sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==
-  dependencies:
-    "@babel/types" "^7.20.2"
-
-"@babel/helper-skip-transparent-expression-wrappers@^7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.12.1.tgz#462dc63a7e435ade8468385c63d2b84cce4b3cbf"
-  integrity sha512-Mf5AUuhG1/OCChOJ/HcADmvcHM42WJockombn8ATJG3OnyiSxBK/Mm5x78BQWvmtXZKHgbjdGL2kin/HOLlZGA==
-  dependencies:
-    "@babel/types" "^7.12.1"
-
-"@babel/helper-skip-transparent-expression-wrappers@^7.20.0":
-  version "7.20.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.20.0.tgz#fbe4c52f60518cab8140d77101f0e63a8a230684"
-  integrity sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg==
-  dependencies:
-    "@babel/types" "^7.20.0"
-
-"@babel/helper-split-export-declaration@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05"
-  integrity sha512-tCJDltF83htUtXx5NLcaDqRmknv652ZWCHyoTETf1CXYJdPC7nohZohjUgieXhv0hTJdRf2FjDueFehdNucpzg==
-  dependencies:
-    "@babel/types" "^7.12.13"
-
-"@babel/helper-split-export-declaration@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.14.5.tgz#22b23a54ef51c2b7605d851930c1976dd0bc693a"
-  integrity sha512-hprxVPu6e5Kdp2puZUmvOGjaLv9TCe58E/Fl6hRq4YiVQxIcNvuq6uTM2r1mT/oPskuS9CgR+I94sqAYv0NGKA==
-  dependencies:
-    "@babel/types" "^7.14.5"
-
-"@babel/helper-split-export-declaration@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz#7367949bc75b20c6d5a5d4a97bba2824ae8ef075"
-  integrity sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==
-  dependencies:
-    "@babel/types" "^7.18.6"
-
-"@babel/helper-string-parser@^7.19.4":
-  version "7.19.4"
-  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63"
-  integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==
-
-"@babel/helper-validator-identifier@^7.12.11", "@babel/helper-validator-identifier@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.0.tgz#d26cad8a47c65286b15df1547319a5d0bcf27288"
-  integrity sha512-V3ts7zMSu5lfiwWDVWzRDGIN+lnCEUdaXgtVHJgLb1rGaA6jMrtB9EmE7L18foXJIE8Un/A/h6NJfGQp/e1J4A==
-
-"@babel/helper-validator-identifier@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8"
-  integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg==
-
-"@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1":
-  version "7.19.1"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2"
-  integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==
-
-"@babel/helper-validator-option@^7.12.1", "@babel/helper-validator-option@^7.12.17":
-  version "7.12.17"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.12.17.tgz#d1fbf012e1a79b7eebbfdc6d270baaf8d9eb9831"
-  integrity sha512-TopkMDmLzq8ngChwRlyjR6raKD6gMSae4JdYDB8bByKreQgG0RBTuKe9LRxW3wFtUnjxOPRKBDwEH6Mg5KeDfw==
-
-"@babel/helper-validator-option@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz#bf0d2b5a509b1f336099e4ff36e1a63aa5db4db8"
-  integrity sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==
-
-"@babel/helper-wrap-function@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.13.0.tgz#bdb5c66fda8526ec235ab894ad53a1235c79fcc4"
-  integrity sha512-1UX9F7K3BS42fI6qd2A4BjKzgGjToscyZTdp1DjknHLCIvpgne6918io+aL5LXFcER/8QWiwpoY902pVEqgTXA==
-  dependencies:
-    "@babel/helper-function-name" "^7.12.13"
-    "@babel/template" "^7.12.13"
-    "@babel/traverse" "^7.13.0"
-    "@babel/types" "^7.13.0"
-
-"@babel/helpers@^7.12.1", "@babel/helpers@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.14.0.tgz#ea9b6be9478a13d6f961dbb5f36bf75e2f3b8f62"
-  integrity sha512-+ufuXprtQ1D1iZTO/K9+EBRn+qPWMJjZSw/S0KlFrxCw4tkrzv9grgpDHkY9MeQTjTY8i2sp7Jep8DfU6tN9Mg==
-  dependencies:
-    "@babel/template" "^7.12.13"
-    "@babel/traverse" "^7.14.0"
-    "@babel/types" "^7.14.0"
-
-"@babel/helpers@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.20.7.tgz#04502ff0feecc9f20ecfaad120a18f011a8e6dce"
-  integrity sha512-PBPjs5BppzsGaxHQCDKnZ6Gd9s6xl8bBCluz3vEInLGRJmnZan4F6BYCeqtyXqkk4W5IlPmjK4JlOuZkpJ3xZA==
-  dependencies:
-    "@babel/template" "^7.20.7"
-    "@babel/traverse" "^7.20.7"
-    "@babel/types" "^7.20.7"
-
-"@babel/highlight@^7.10.4", "@babel/highlight@^7.12.13":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.14.0.tgz#3197e375711ef6bf834e67d0daec88e4f46113cf"
-  integrity sha512-YSCOwxvTYEIMSGaBQb5kDDsCopDdiUGsqpatp3fOlI4+2HQSkTmEVWnVuySdAC5EWCqSWWTv0ib63RjR7dTBdg==
-  dependencies:
-    "@babel/helper-validator-identifier" "^7.14.0"
-    chalk "^2.0.0"
-    js-tokens "^4.0.0"
-
-"@babel/highlight@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz#6861a52f03966405001f6aa534a01a24d99e8cd9"
-  integrity sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==
-  dependencies:
-    "@babel/helper-validator-identifier" "^7.14.5"
-    chalk "^2.0.0"
-    js-tokens "^4.0.0"
-
-"@babel/highlight@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.18.6.tgz#81158601e93e2563795adcbfbdf5d64be3f2ecdf"
-  integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==
-  dependencies:
-    "@babel/helper-validator-identifier" "^7.18.6"
+"@babel/helper-environment-visitor@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98"
+  integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==
+
+"@babel/helper-function-name@^7.12.13", "@babel/helper-function-name@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz#ede300828905bb15e582c037162f99d5183af1be"
+  integrity sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==
+  dependencies:
+    "@babel/template" "^7.22.5"
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-hoist-variables@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb"
+  integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-member-expression-to-functions@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.22.5.tgz#0a7c56117cad3372fbf8d2fb4bf8f8d64a1e76b2"
+  integrity sha512-aBiH1NKMG0H2cGZqspNvsaBe6wNGjbJjuLy29aU+eDZjSbbN53BaxlpB02xm9v34pLTZ1nIQPFYn2qMZoa5BQQ==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz#1a8f4c9f4027d23f520bd76b364d44434a72660c"
+  integrity sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-module-transforms@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.5.tgz#0f65daa0716961b6e96b164034e737f60a80d2ef"
+  integrity sha512-+hGKDt/Ze8GFExiVHno/2dvG5IdstpzCq0y4Qc9OJ25D4q3pKfiIP/4Vp3/JvhDkLKsDK2api3q3fpIgiIF5bw==
+  dependencies:
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-module-imports" "^7.22.5"
+    "@babel/helper-simple-access" "^7.22.5"
+    "@babel/helper-split-export-declaration" "^7.22.5"
+    "@babel/helper-validator-identifier" "^7.22.5"
+    "@babel/template" "^7.22.5"
+    "@babel/traverse" "^7.22.5"
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-optimise-call-expression@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.22.5.tgz#f21531a9ccbff644fdd156b4077c16ff0c3f609e"
+  integrity sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.13.0", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.16.7", "@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.20.2", "@babel/helper-plugin-utils@^7.22.5", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz#dd7ee3735e8a313b9f7b05a773d892e88e6d7295"
+  integrity sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==
+
+"@babel/helper-remap-async-to-generator@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.22.5.tgz#14a38141a7bf2165ad38da61d61cf27b43015da2"
+  integrity sha512-cU0Sq1Rf4Z55fgz7haOakIyM7+x/uCFwXpLPaeRzfoUtAEAuUZjZvFPjL/rk5rW693dIgn2hng1W7xbT7lWT4g==
+  dependencies:
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-wrap-function" "^7.22.5"
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-replace-supers@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.22.5.tgz#71bc5fb348856dea9fdc4eafd7e2e49f585145dc"
+  integrity sha512-aLdNM5I3kdI/V9xGNyKSF3X/gTyMUBohTZ+/3QdQKAA9vxIiy12E+8E2HoOP1/DjeqU+g6as35QHJNMDDYpuCg==
+  dependencies:
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-member-expression-to-functions" "^7.22.5"
+    "@babel/helper-optimise-call-expression" "^7.22.5"
+    "@babel/template" "^7.22.5"
+    "@babel/traverse" "^7.22.5"
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-simple-access@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz#4938357dc7d782b80ed6dbb03a0fba3d22b1d5de"
+  integrity sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-skip-transparent-expression-wrappers@^7.20.0", "@babel/helper-skip-transparent-expression-wrappers@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.22.5.tgz#007f15240b5751c537c40e77abb4e89eeaaa8847"
+  integrity sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-split-export-declaration@^7.12.13", "@babel/helper-split-export-declaration@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.5.tgz#88cf11050edb95ed08d596f7a044462189127a08"
+  integrity sha512-thqK5QFghPKWLhAV321lxF95yCg2K3Ob5yw+M3VHWfdia0IkPXUtoLH8x/6Fh486QUvzhb8YOWHChTVen2/PoQ==
+  dependencies:
+    "@babel/types" "^7.22.5"
+
+"@babel/helper-string-parser@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f"
+  integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==
+
+"@babel/helper-validator-identifier@^7.12.11", "@babel/helper-validator-identifier@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193"
+  integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==
+
+"@babel/helper-validator-option@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.5.tgz#de52000a15a177413c8234fa3a8af4ee8102d0ac"
+  integrity sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==
+
+"@babel/helper-wrap-function@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.22.5.tgz#44d205af19ed8d872b4eefb0d2fa65f45eb34f06"
+  integrity sha512-bYqLIBSEshYcYQyfks8ewYA8S30yaGSeRslcvKMvoUk6HHPySbxHq9YRi6ghhzEU+yhQv9bP/jXnygkStOcqZw==
+  dependencies:
+    "@babel/helper-function-name" "^7.22.5"
+    "@babel/template" "^7.22.5"
+    "@babel/traverse" "^7.22.5"
+    "@babel/types" "^7.22.5"
+
+"@babel/helpers@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.5.tgz#74bb4373eb390d1ceed74a15ef97767e63120820"
+  integrity sha512-pSXRmfE1vzcUIDFQcSGA5Mr+GxBV9oiRKDuDxXvWQQBCh8HoIjs/2DlDB7H8smac1IVrB9/xdXj2N3Wol9Cr+Q==
+  dependencies:
+    "@babel/template" "^7.22.5"
+    "@babel/traverse" "^7.22.5"
+    "@babel/types" "^7.22.5"
+
+"@babel/highlight@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.5.tgz#aa6c05c5407a67ebce408162b7ede789b4d22031"
+  integrity sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.22.5"
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
@@ -737,48 +448,28 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.16.tgz#cc31257419d2c3189d394081635703f549fc1ed4"
   integrity sha512-c/+u9cqV6F0+4Hpq01jnJO+GLp2DdT63ppz9Xa+6cHaajM9VFzK/iDXiKK65YtpeVwu+ctfS6iqlMqRgQRzeCw==
 
-"@babel/parser@^7.1.0", "@babel/parser@^7.12.13", "@babel/parser@^7.12.3", "@babel/parser@^7.14.2", "@babel/parser@^7.14.3", "@babel/parser@^7.7.0":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.3.tgz#9b530eecb071fd0c93519df25c5ff9f14759f298"
-  integrity sha512-7MpZDIfI7sUC5zWo2+foJ50CSI5lcqDehZ0lVgIhSi4bFEk94fLAKlF3Q0nzSQQ+ca0lm+O6G9ztKVBeu8PMRQ==
-
-"@babel/parser@^7.1.6", "@babel/parser@^7.14.5", "@babel/parser@^7.14.7":
-  version "7.14.7"
-  resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.14.7.tgz#6099720c8839ca865a2637e6c85852ead0bdb595"
-  integrity sha512-X67Z5y+VBJuHB/RjwECp8kSl5uYi0BvRbNeWqkaJCVh+LiTPl19WBUfG627psSgp9rSf6ojuXghQM3ha6qHHdA==
-
-"@babel/parser@^7.14.0", "@babel/parser@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.7.tgz#66fe23b3c8569220817d5feb8b9dcdc95bb4f71b"
-  integrity sha512-T3Z9oHybU+0vZlY9CiDSJQTD5ZapcW18ZctFMi0MOAl/4BjFF4ul7NVSARLdbGO5vDqy9eQiGTV0LtKfvCYvcg==
-
-"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.13.12":
-  version "7.13.12"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.13.12.tgz#a3484d84d0b549f3fc916b99ee4783f26fabad2a"
-  integrity sha512-d0u3zWKcoZf379fOeJdr1a5WPDny4aOFZ6hlfKivgK0LY7ZxNfoaHL2fWwdGtHyVvra38FC+HVYkO+byfSA8AQ==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1"
-    "@babel/plugin-proposal-optional-chaining" "^7.13.12"
+"@babel/parser@^7.1.0", "@babel/parser@^7.1.6", "@babel/parser@^7.12.13", "@babel/parser@^7.14.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.5.tgz#721fd042f3ce1896238cf1b341c77eb7dee7dbea"
+  integrity sha512-DFZMC9LJUG9PLOclRC32G63UXwzqS2koQC8dkx+PLdmt1xSePYpbT/NbsrJy8Q/muXz7o/h/d4A7Fuyixm559Q==
 
-"@babel/plugin-proposal-async-generator-functions@^7.12.1", "@babel/plugin-proposal-async-generator-functions@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.14.2.tgz#3a2085abbf5d5f962d480dbc81347385ed62eb1e"
-  integrity sha512-b1AM4F6fwck4N8ItZ/AtC4FP/cqZqmKRQ4FaTDutwSYyjuhtvsGEMLK4N/ztV/ImP40BjIDyMgBQAeAMsQYVFQ==
+"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.22.5.tgz#87245a21cd69a73b0b81bcda98d443d6df08f05e"
+  integrity sha512-NP1M5Rf+u2Gw9qfSO4ihjcTGW5zXTi36ITLd4/EoAcEhIZ0yjMqmftDNl3QC19CX7olhrjpyU454g/2W7X0jvQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-remap-async-to-generator" "^7.13.0"
-    "@babel/plugin-syntax-async-generators" "^7.8.4"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-proposal-class-properties@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.12.1.tgz#a082ff541f2a29a4821065b8add9346c0c16e5de"
-  integrity sha512-cKp3dlQsFsEs5CWKnN7BnSHOd0EOW8EKpEjkoz1pO2E5KzIDNV9Ros1b0CnmbVgAGXJubOYVBOGCT1OmJwOI7w==
+"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.22.5.tgz#fef09f9499b1f1c930da8a0c419db42167d792ca"
+  integrity sha512-31Bb65aZaUwqCbWMnZPduIZxCBngHFlzyN6Dq6KAJjtx+lx6ohKHubc61OomYi7XwVD4Ol0XCVz4h+pYFR048g==
   dependencies:
-    "@babel/helper-create-class-features-plugin" "^7.12.1"
-    "@babel/helper-plugin-utils" "^7.10.4"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-skip-transparent-expression-wrappers" "^7.22.5"
+    "@babel/plugin-transform-optional-chaining" "^7.22.5"
 
-"@babel/plugin-proposal-class-properties@^7.0.0":
+"@babel/plugin-proposal-class-properties@^7.0.0", "@babel/plugin-proposal-class-properties@^7.16.0":
   version "7.18.6"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz#b110f59741895f7ec21a6fff696ec46265c446a3"
   integrity sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==
@@ -786,94 +477,31 @@
     "@babel/helper-create-class-features-plugin" "^7.18.6"
     "@babel/helper-plugin-utils" "^7.18.6"
 
-"@babel/plugin-proposal-class-properties@^7.12.1", "@babel/plugin-proposal-class-properties@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.13.0.tgz#146376000b94efd001e57a40a88a525afaab9f37"
-  integrity sha512-KnTDjFNC1g+45ka0myZNvSBFLhNCLN+GeGYLDEA8Oq7MZ6yMgfLoIRh86GRT0FjtJhZw8JyUskP9uvj5pHM9Zg==
-  dependencies:
-    "@babel/helper-create-class-features-plugin" "^7.13.0"
-    "@babel/helper-plugin-utils" "^7.13.0"
-
-"@babel/plugin-proposal-class-static-block@^7.13.11":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.14.3.tgz#5a527e2cae4a4753119c3a3e7f64ecae8ccf1360"
-  integrity sha512-HEjzp5q+lWSjAgJtSluFDrGGosmwTgKwCXdDQZvhKsRlwv3YdkUEqxNrrjesJd+B9E9zvr1PVPVBvhYZ9msjvQ==
+"@babel/plugin-proposal-decorators@^7.16.4":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.22.5.tgz#dc8cdda048e5aea947efda920e030199806b868d"
+  integrity sha512-h8hlezQ4dl6ixodgXkH8lUfcD7x+WAuIqPUjwGoItynrXOAv4a4Tci1zA/qjzQjjcl0v3QpLdc2LM6ZACQuY7A==
   dependencies:
-    "@babel/helper-create-class-features-plugin" "^7.14.3"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-class-static-block" "^7.12.13"
-
-"@babel/plugin-proposal-decorators@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.12.1.tgz#59271439fed4145456c41067450543aee332d15f"
-  integrity sha512-knNIuusychgYN8fGJHONL0RbFxLGawhXOJNLBk75TniTsZZeA+wdkDuv6wp4lGwzQEKjZi6/WYtnb3udNPmQmQ==
-  dependencies:
-    "@babel/helper-create-class-features-plugin" "^7.12.1"
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/plugin-syntax-decorators" "^7.12.1"
-
-"@babel/plugin-proposal-dynamic-import@^7.12.1", "@babel/plugin-proposal-dynamic-import@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.14.2.tgz#01ebabd7c381cff231fa43e302939a9de5be9d9f"
-  integrity sha512-oxVQZIWFh91vuNEMKltqNsKLFWkOIyJc95k2Gv9lWVyDfPUQGSSlbDEgWuJUU1afGE9WwlzpucMZ3yDRHIItkA==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-dynamic-import" "^7.8.3"
+    "@babel/helper-create-class-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-replace-supers" "^7.22.5"
+    "@babel/helper-split-export-declaration" "^7.22.5"
+    "@babel/plugin-syntax-decorators" "^7.22.5"
 
-"@babel/plugin-proposal-export-namespace-from@^7.12.1", "@babel/plugin-proposal-export-namespace-from@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.14.2.tgz#62542f94aa9ce8f6dba79eec698af22112253791"
-  integrity sha512-sRxW3z3Zp3pFfLAgVEvzTFutTXax837oOatUIvSG9o5gRj9mKwm3br1Se5f4QalTQs9x4AzlA/HrCWbQIHASUQ==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-export-namespace-from" "^7.8.3"
-
-"@babel/plugin-proposal-json-strings@^7.12.1", "@babel/plugin-proposal-json-strings@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.14.2.tgz#830b4e2426a782e8b2878fbfe2cba85b70cbf98c"
-  integrity sha512-w2DtsfXBBJddJacXMBhElGEYqCZQqN99Se1qeYn8DVLB33owlrlLftIbMzn5nz1OITfDVknXF433tBrLEAOEjA==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-json-strings" "^7.8.3"
-
-"@babel/plugin-proposal-logical-assignment-operators@^7.12.1", "@babel/plugin-proposal-logical-assignment-operators@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.14.2.tgz#222348c080a1678e0e74ea63fe76f275882d1fd7"
-  integrity sha512-1JAZtUrqYyGsS7IDmFeaem+/LJqujfLZ2weLR9ugB0ufUPjzf8cguyVT1g5im7f7RXxuLq1xUxEzvm68uYRtGg==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4"
-
-"@babel/plugin-proposal-nullish-coalescing-operator@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.12.1.tgz#3ed4fff31c015e7f3f1467f190dbe545cd7b046c"
-  integrity sha512-nZY0ESiaQDI1y96+jk6VxMOaL4LPo/QDHBqL+SF3/vl6dHkTwHlOI8L4ZwuRBHgakRBw5zsVylel7QPbbGuYgg==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0"
-
-"@babel/plugin-proposal-nullish-coalescing-operator@^7.12.1", "@babel/plugin-proposal-nullish-coalescing-operator@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.14.2.tgz#425b11dc62fc26939a2ab42cbba680bdf5734546"
-  integrity sha512-ebR0zU9OvI2N4qiAC38KIAK75KItpIPTpAtd2r4OZmMFeKbKJpUFLYP2EuDut82+BmYi8sz42B+TfTptJ9iG5Q==
+"@babel/plugin-proposal-nullish-coalescing-operator@^7.16.0":
+  version "7.18.6"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz#fdd940a99a740e577d6c753ab6fbb43fdb9467e1"
+  integrity sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.18.6"
     "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3"
 
-"@babel/plugin-proposal-numeric-separator@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.1.tgz#0e2c6774c4ce48be412119b4d693ac777f7685a6"
-  integrity sha512-MR7Ok+Af3OhNTCxYVjJZHS0t97ydnJZt/DbR4WISO39iDnhiD8XHrY12xuSJ90FFEGjir0Fzyyn7g/zY6hxbxA==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/plugin-syntax-numeric-separator" "^7.10.4"
-
-"@babel/plugin-proposal-numeric-separator@^7.12.1", "@babel/plugin-proposal-numeric-separator@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.14.2.tgz#82b4cc06571143faf50626104b335dd71baa4f9e"
-  integrity sha512-DcTQY9syxu9BpU3Uo94fjCB3LN9/hgPS8oUL7KrSW3bA2ePrKZZPJcc5y0hoJAM9dft3pGfErtEUvxXQcfLxUg==
+"@babel/plugin-proposal-numeric-separator@^7.16.0":
+  version "7.18.6"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz#899b14fbafe87f053d2c5ff05b36029c62e13c75"
+  integrity sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.18.6"
     "@babel/plugin-syntax-numeric-separator" "^7.10.4"
 
 "@babel/plugin-proposal-object-rest-spread@^7.0.0":
@@ -887,62 +515,39 @@
     "@babel/plugin-syntax-object-rest-spread" "^7.8.3"
     "@babel/plugin-transform-parameters" "^7.20.7"
 
-"@babel/plugin-proposal-object-rest-spread@^7.12.1", "@babel/plugin-proposal-object-rest-spread@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.14.2.tgz#e17d418f81cc103fedd4ce037e181c8056225abc"
-  integrity sha512-hBIQFxwZi8GIp934+nj5uV31mqclC1aYDhctDu5khTi9PCCUOczyy0b34W0oE9U/eJXiqQaKyVsmjeagOaSlbw==
-  dependencies:
-    "@babel/compat-data" "^7.14.0"
-    "@babel/helper-compilation-targets" "^7.13.16"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-object-rest-spread" "^7.8.3"
-    "@babel/plugin-transform-parameters" "^7.14.2"
-
-"@babel/plugin-proposal-optional-catch-binding@^7.12.1", "@babel/plugin-proposal-optional-catch-binding@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.14.2.tgz#150d4e58e525b16a9a1431bd5326c4eed870d717"
-  integrity sha512-XtkJsmJtBaUbOxZsNk0Fvrv8eiqgneug0A6aqLFZ4TSkar2L5dSXWcnUKHgmjJt49pyB/6ZHvkr3dPgl9MOWRQ==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-optional-catch-binding" "^7.8.3"
-
-"@babel/plugin-proposal-optional-chaining@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.1.tgz#cce122203fc8a32794296fc377c6dedaf4363797"
-  integrity sha512-c2uRpY6WzaVDzynVY9liyykS+kVU+WRZPMPYpkelXH8KBt1oXoI89kPbZKKG/jDT5UK92FTW2fZkZaJhdiBabw==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1"
-    "@babel/plugin-syntax-optional-chaining" "^7.8.0"
-
-"@babel/plugin-proposal-optional-chaining@^7.12.1", "@babel/plugin-proposal-optional-chaining@^7.13.12", "@babel/plugin-proposal-optional-chaining@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.14.2.tgz#df8171a8b9c43ebf4c1dabe6311b432d83e1b34e"
-  integrity sha512-qQByMRPwMZJainfig10BoaDldx/+VDtNcrA7qdNaEOAj6VXud+gfrkA8j4CRAU5HjnWREXqIpSpH30qZX1xivA==
+"@babel/plugin-proposal-optional-chaining@^7.16.0":
+  version "7.21.0"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.21.0.tgz#886f5c8978deb7d30f678b2e24346b287234d3ea"
+  integrity sha512-p4zeefM72gpmEe2fkUr/OnOXpWEf8nAgk7ZYVqqfFiyIG7oFfVZcCrU64hWn5xp4tQ9LkV4bTIa5rD0KANpKNA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1"
+    "@babel/helper-plugin-utils" "^7.20.2"
+    "@babel/helper-skip-transparent-expression-wrappers" "^7.20.0"
     "@babel/plugin-syntax-optional-chaining" "^7.8.3"
 
-"@babel/plugin-proposal-private-methods@^7.12.1", "@babel/plugin-proposal-private-methods@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.13.0.tgz#04bd4c6d40f6e6bbfa2f57e2d8094bad900ef787"
-  integrity sha512-MXyyKQd9inhx1kDYPkFRVOBXQ20ES8Pto3T7UZ92xj2mY0EVD8oAVzeyYuVfy/mxAdTSIayOvg+aVzcHV2bn6Q==
+"@babel/plugin-proposal-private-methods@^7.16.0":
+  version "7.18.6"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz#5209de7d213457548a98436fa2882f52f4be6bea"
+  integrity sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==
   dependencies:
-    "@babel/helper-create-class-features-plugin" "^7.13.0"
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-create-class-features-plugin" "^7.18.6"
+    "@babel/helper-plugin-utils" "^7.18.6"
 
-"@babel/plugin-proposal-private-property-in-object@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.14.0.tgz#b1a1f2030586b9d3489cc26179d2eb5883277636"
-  integrity sha512-59ANdmEwwRUkLjB7CRtwJxxwtjESw+X2IePItA+RGQh+oy5RmpCh/EvVVvh5XQc3yxsm5gtv0+i9oBZhaDNVTg==
+"@babel/plugin-proposal-private-property-in-object@7.21.0-placeholder-for-preset-env.2":
+  version "7.21.0-placeholder-for-preset-env.2"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.0-placeholder-for-preset-env.2.tgz#7844f9289546efa9febac2de4cfe358a050bd703"
+  integrity sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==
+
+"@babel/plugin-proposal-private-property-in-object@^7.21.11":
+  version "7.21.11"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.11.tgz#69d597086b6760c4126525cfa154f34631ff272c"
+  integrity sha512-0QZ8qP/3RLDVBwBFoWAwCtgcDZJVwA5LUJRZU8x2YFfKNuFq161wK3cuGrALu5yiPu+vzwTAg/sMWVNeWeNyaw==
   dependencies:
-    "@babel/helper-annotate-as-pure" "^7.12.13"
-    "@babel/helper-create-class-features-plugin" "^7.14.0"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-private-property-in-object" "^7.14.0"
+    "@babel/helper-annotate-as-pure" "^7.18.6"
+    "@babel/helper-create-class-features-plugin" "^7.21.0"
+    "@babel/helper-plugin-utils" "^7.20.2"
+    "@babel/plugin-syntax-private-property-in-object" "^7.14.5"
 
-"@babel/plugin-proposal-unicode-property-regex@^7.12.1", "@babel/plugin-proposal-unicode-property-regex@^7.12.13", "@babel/plugin-proposal-unicode-property-regex@^7.4.4":
+"@babel/plugin-proposal-unicode-property-regex@^7.4.4":
   version "7.12.13"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.12.13.tgz#bebde51339be829c17aaaaced18641deb62b39ba"
   integrity sha512-XyJmZidNfofEkqFV5VC/bLabGmO5QzenPO/YOfGuEbgU+2sSwMmio3YLb4WtBgcmmdwZHyVyv8on77IUjQ5Gvg==
@@ -950,7 +555,7 @@
     "@babel/helper-create-regexp-features-plugin" "^7.12.13"
     "@babel/helper-plugin-utils" "^7.12.13"
 
-"@babel/plugin-syntax-async-generators@^7.8.0", "@babel/plugin-syntax-async-generators@^7.8.4":
+"@babel/plugin-syntax-async-generators@^7.8.4":
   version "7.8.4"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d"
   integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==
@@ -964,28 +569,28 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.8.0"
 
-"@babel/plugin-syntax-class-properties@^7.0.0", "@babel/plugin-syntax-class-properties@^7.12.1", "@babel/plugin-syntax-class-properties@^7.12.13", "@babel/plugin-syntax-class-properties@^7.8.3":
+"@babel/plugin-syntax-class-properties@^7.0.0", "@babel/plugin-syntax-class-properties@^7.12.13", "@babel/plugin-syntax-class-properties@^7.8.3":
   version "7.12.13"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10"
   integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==
   dependencies:
     "@babel/helper-plugin-utils" "^7.12.13"
 
-"@babel/plugin-syntax-class-static-block@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.12.13.tgz#8e3d674b0613e67975ceac2776c97b60cafc5c9c"
-  integrity sha512-ZmKQ0ZXR0nYpHZIIuj9zE7oIqCx2hw9TKi+lIo73NNrMPAZGHfS92/VRV0ZmPj6H2ffBgyFHXvJ5NYsNeEaP2A==
+"@babel/plugin-syntax-class-static-block@^7.14.5":
+  version "7.14.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz#195df89b146b4b78b3bf897fd7a257c84659d406"
+  integrity sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.14.5"
 
-"@babel/plugin-syntax-decorators@^7.12.1":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.12.13.tgz#fac829bf3c7ef4a1bc916257b403e58c6bdaf648"
-  integrity sha512-Rw6aIXGuqDLr6/LoBBYE57nKOzQpz/aDkKlMqEwH+Vp0MXbG6H/TfRjaY343LKxzAKAMXIHsQ8JzaZKuDZ9MwA==
+"@babel/plugin-syntax-decorators@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.22.5.tgz#329fe2907c73de184033775637dbbc507f09116a"
+  integrity sha512-avpUOBS7IU6al8MmF1XpAyj9QYeLPuSDJI5D4pVMSMdL7xQokKqJPYQC67RCT0aCTashUXPiGwMJ0DEXXCEmMA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-syntax-dynamic-import@^7.8.0", "@babel/plugin-syntax-dynamic-import@^7.8.3":
+"@babel/plugin-syntax-dynamic-import@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz#62bf98b2da3cd21d626154fc96ee5b3cb68eacb3"
   integrity sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==
@@ -999,47 +604,47 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.8.3"
 
-"@babel/plugin-syntax-flow@^7.0.0", "@babel/plugin-syntax-flow@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz#774d825256f2379d06139be0c723c4dd444f3ca1"
-  integrity sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==
+"@babel/plugin-syntax-flow@^7.0.0", "@babel/plugin-syntax-flow@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.22.5.tgz#163b820b9e7696ce134df3ee716d9c0c98035859"
+  integrity sha512-9RdCl0i+q0QExayk2nOS7853w08yLucnnPML6EN9S8fgMPVtdLDCdx/cOQ/i44Lb9UeQX9A35yaqBBOMMZxPxQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-syntax-flow@^7.12.1":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.12.13.tgz#5df9962503c0a9c918381c929d51d4d6949e7e86"
-  integrity sha512-J/RYxnlSLXZLVR7wTRsozxKT8qbsx1mNKJzXEEjQ0Kjx1ZACcyHgbanNWNCFtc36IzuWhYWPpvJFFoexoOWFmA==
+"@babel/plugin-syntax-import-assertions@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.22.5.tgz#07d252e2aa0bc6125567f742cd58619cb14dce98"
+  integrity sha512-rdV97N7KqsRzeNGoWUOK6yUsWarLjE5Su/Snk9IYPU9CwkWHs4t+rTGOvffTR8XGkJMTAdLfO0xVnXm8wugIJg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-syntax-import-meta@^7.8.3":
+"@babel/plugin-syntax-import-attributes@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.22.5.tgz#ab840248d834410b829f569f5262b9e517555ecb"
+  integrity sha512-KwvoWDeNKPETmozyFE0P2rOLqh39EoQHNjqizrI5B8Vt0ZNS7M56s7dAiAqbYfiAYOuIzIh96z3iR2ktgu3tEg==
+  dependencies:
+    "@babel/helper-plugin-utils" "^7.22.5"
+
+"@babel/plugin-syntax-import-meta@^7.10.4", "@babel/plugin-syntax-import-meta@^7.8.3":
   version "7.10.4"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz#ee601348c370fa334d2207be158777496521fd51"
   integrity sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==
   dependencies:
     "@babel/helper-plugin-utils" "^7.10.4"
 
-"@babel/plugin-syntax-json-strings@^7.8.0", "@babel/plugin-syntax-json-strings@^7.8.3":
+"@babel/plugin-syntax-json-strings@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a"
   integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==
   dependencies:
     "@babel/helper-plugin-utils" "^7.8.0"
 
-"@babel/plugin-syntax-jsx@^7.0.0", "@babel/plugin-syntax-jsx@^7.17.12", "@babel/plugin-syntax-jsx@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.18.6.tgz#a8feef63b010150abd97f1649ec296e849943ca0"
-  integrity sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q==
+"@babel/plugin-syntax-jsx@^7.0.0", "@babel/plugin-syntax-jsx@^7.17.12", "@babel/plugin-syntax-jsx@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.22.5.tgz#a6b68e84fb76e759fc3b93e901876ffabbe1d918"
+  integrity sha512-gvyP4hZrgrs/wWMaocvxZ44Hw0b3W8Pe+cMxc8V1ULQ07oh8VNbIRaoD1LRZVTvD+0nieDKjfgKg89sD7rrKrg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
-
-"@babel/plugin-syntax-jsx@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.13.tgz#044fb81ebad6698fe62c478875575bcbb9b70f15"
-  integrity sha512-d4HM23Q1K7oq/SLNmG6mRt85l2csmQ0cHRaxRXjKW0YFdEXqlZ5kzFQKH5Uc3rDJECgu+yCRgPkG04Mm98R/1g==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
 "@babel/plugin-syntax-logical-assignment-operators@^7.10.4", "@babel/plugin-syntax-logical-assignment-operators@^7.8.3":
   version "7.10.4"
@@ -1048,7 +653,7 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.10.4"
 
-"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.0", "@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3":
+"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9"
   integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==
@@ -1062,358 +667,369 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.10.4"
 
-"@babel/plugin-syntax-object-rest-spread@^7.0.0", "@babel/plugin-syntax-object-rest-spread@^7.8.0", "@babel/plugin-syntax-object-rest-spread@^7.8.3":
+"@babel/plugin-syntax-object-rest-spread@^7.0.0", "@babel/plugin-syntax-object-rest-spread@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871"
   integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==
   dependencies:
     "@babel/helper-plugin-utils" "^7.8.0"
 
-"@babel/plugin-syntax-optional-catch-binding@^7.8.0", "@babel/plugin-syntax-optional-catch-binding@^7.8.3":
+"@babel/plugin-syntax-optional-catch-binding@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1"
   integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==
   dependencies:
     "@babel/helper-plugin-utils" "^7.8.0"
 
-"@babel/plugin-syntax-optional-chaining@^7.8.0", "@babel/plugin-syntax-optional-chaining@^7.8.3":
+"@babel/plugin-syntax-optional-chaining@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a"
   integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==
   dependencies:
     "@babel/helper-plugin-utils" "^7.8.0"
 
-"@babel/plugin-syntax-private-property-in-object@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.0.tgz#762a4babec61176fec6c88480dec40372b140c0b"
-  integrity sha512-bda3xF8wGl5/5btF794utNOL0Jw+9jE5C1sLZcoK7c4uonE/y3iQiyG+KbkF3WBV/paX58VCpjhxLPkdj5Fe4w==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-
-"@babel/plugin-syntax-top-level-await@^7.12.1", "@babel/plugin-syntax-top-level-await@^7.12.13", "@babel/plugin-syntax-top-level-await@^7.8.3":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.13.tgz#c5f0fa6e249f5b739727f923540cf7a806130178"
-  integrity sha512-A81F9pDwyS7yM//KwbCSDqy3Uj4NMIurtplxphWxoYtNPov7cJsDkAFNNyVlIZ3jwGycVsurZ+LtOA8gZ376iQ==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
-
-"@babel/plugin-syntax-typescript@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.12.13.tgz#9dff111ca64154cef0f4dc52cf843d9f12ce4474"
-  integrity sha512-cHP3u1JiUiG2LFDKbXnwVad81GvfyIOmCD6HIEId6ojrY0Drfy2q1jw7BwN7dE84+kTnBjLkXoL3IEy/3JPu2w==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
-
-"@babel/plugin-transform-arrow-functions@^7.0.0":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.20.7.tgz#bea332b0e8b2dab3dafe55a163d8227531ab0551"
-  integrity sha512-3poA5E7dzDomxj9WXWwuD6A5F3kc7VXwIJO+E+J8qtDtS+pXPAhrgEyh+9GBwBgPq1Z+bB+/JD60lp5jsN7JPQ==
+"@babel/plugin-syntax-private-property-in-object@^7.14.5":
+  version "7.14.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz#0dc6671ec0ea22b6e94a1114f857970cd39de1ad"
+  integrity sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.20.2"
+    "@babel/helper-plugin-utils" "^7.14.5"
 
-"@babel/plugin-transform-arrow-functions@^7.12.1", "@babel/plugin-transform-arrow-functions@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.13.0.tgz#10a59bebad52d637a027afa692e8d5ceff5e3dae"
-  integrity sha512-96lgJagobeVmazXFaDrbmCLQxBysKu7U6Do3mLsx27gf5Dk85ezysrs2BZUpXD703U/Su1xTBDxxar2oa4jAGg==
+"@babel/plugin-syntax-top-level-await@^7.14.5", "@babel/plugin-syntax-top-level-await@^7.8.3":
+  version "7.14.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz#c1cfdadc35a646240001f06138247b741c34d94c"
+  integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.14.5"
 
-"@babel/plugin-transform-async-to-generator@^7.12.1", "@babel/plugin-transform-async-to-generator@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.13.0.tgz#8e112bf6771b82bf1e974e5e26806c5c99aa516f"
-  integrity sha512-3j6E004Dx0K3eGmhxVJxwwI89CTJrce7lg3UrtFuDAVQ/2+SJ/h/aSFOeE6/n0WB1GsOffsJp6MnPQNQ8nmwhg==
+"@babel/plugin-syntax-typescript@^7.22.5", "@babel/plugin-syntax-typescript@^7.7.2":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.22.5.tgz#aac8d383b062c5072c647a31ef990c1d0af90272"
+  integrity sha512-1mS2o03i7t1c6VzH6fdQ3OA8tcEIxwG18zIPRp+UY1Ihv6W+XZzBCVxExF9upussPXJ0xE9XRHwMoNs1ep/nRQ==
   dependencies:
-    "@babel/helper-module-imports" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-remap-async-to-generator" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-block-scoped-functions@^7.0.0":
+"@babel/plugin-syntax-unicode-sets-regex@^7.18.6":
   version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz#9187bf4ba302635b9d70d986ad70f038726216a8"
-  integrity sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==
+  resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-unicode-sets-regex/-/plugin-syntax-unicode-sets-regex-7.18.6.tgz#d49a3b3e6b52e5be6740022317580234a6a47357"
+  integrity sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==
   dependencies:
+    "@babel/helper-create-regexp-features-plugin" "^7.18.6"
     "@babel/helper-plugin-utils" "^7.18.6"
 
-"@babel/plugin-transform-block-scoped-functions@^7.12.1", "@babel/plugin-transform-block-scoped-functions@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.12.13.tgz#a9bf1836f2a39b4eb6cf09967739de29ea4bf4c4"
-  integrity sha512-zNyFqbc3kI/fVpqwfqkg6RvBgFpC4J18aKKMmv7KdQ/1GgREapSJAykLMVNwfRGO3BtHj3YQZl8kxCXPcVMVeg==
+"@babel/plugin-transform-arrow-functions@^7.0.0", "@babel/plugin-transform-arrow-functions@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.22.5.tgz#e5ba566d0c58a5b2ba2a8b795450641950b71958"
+  integrity sha512-26lTNXoVRdAnsaDXPpvCNUq+OVWEVC6bx7Vvz9rC53F2bagUWW4u4ii2+h8Fejfh7RYqPxn+libeFBBck9muEw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-block-scoping@^7.0.0":
-  version "7.20.11"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.20.11.tgz#9f5a3424bd112a3f32fe0cf9364fbb155cff262a"
-  integrity sha512-tA4N427a7fjf1P0/2I4ScsHGc5jcHPbb30xMbaTke2gxDuWpUfXDuX1FEymJwKk4tuGUvGcejAR6HdZVqmmPyw==
+"@babel/plugin-transform-async-generator-functions@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.22.5.tgz#7336356d23380eda9a56314974f053a020dab0c3"
+  integrity sha512-gGOEvFzm3fWoyD5uZq7vVTD57pPJ3PczPUD/xCFGjzBpUosnklmXyKnGQbbbGs1NPNPskFex0j93yKbHt0cHyg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.20.2"
-
-"@babel/plugin-transform-block-scoping@^7.12.1", "@babel/plugin-transform-block-scoping@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.14.2.tgz#761cb12ab5a88d640ad4af4aa81f820e6b5fdf5c"
-  integrity sha512-neZZcP19NugZZqNwMTH+KoBjx5WyvESPSIOQb4JHpfd+zPfqcH65RMu5xJju5+6q/Y2VzYrleQTr+b6METyyxg==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-
-"@babel/plugin-transform-classes@^7.0.0":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.20.7.tgz#f438216f094f6bb31dc266ebfab8ff05aecad073"
-  integrity sha512-LWYbsiXTPKl+oBlXUGlwNlJZetXD5Am+CyBdqhPsDVjM9Jc8jwBJFrKhHf900Kfk2eZG1y9MAG3UNajol7A4VQ==
-  dependencies:
-    "@babel/helper-annotate-as-pure" "^7.18.6"
-    "@babel/helper-compilation-targets" "^7.20.7"
-    "@babel/helper-environment-visitor" "^7.18.9"
-    "@babel/helper-function-name" "^7.19.0"
-    "@babel/helper-optimise-call-expression" "^7.18.6"
-    "@babel/helper-plugin-utils" "^7.20.2"
-    "@babel/helper-replace-supers" "^7.20.7"
-    "@babel/helper-split-export-declaration" "^7.18.6"
-    globals "^11.1.0"
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-remap-async-to-generator" "^7.22.5"
+    "@babel/plugin-syntax-async-generators" "^7.8.4"
 
-"@babel/plugin-transform-classes@^7.12.1", "@babel/plugin-transform-classes@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.14.2.tgz#3f1196c5709f064c252ad056207d87b7aeb2d03d"
-  integrity sha512-7oafAVcucHquA/VZCsXv/gmuiHeYd64UJyyTYU+MPfNu0KeNlxw06IeENBO8bJjXVbolu+j1MM5aKQtH1OMCNg==
-  dependencies:
-    "@babel/helper-annotate-as-pure" "^7.12.13"
-    "@babel/helper-function-name" "^7.14.2"
-    "@babel/helper-optimise-call-expression" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-replace-supers" "^7.13.12"
-    "@babel/helper-split-export-declaration" "^7.12.13"
+"@babel/plugin-transform-async-to-generator@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.22.5.tgz#c7a85f44e46f8952f6d27fe57c2ed3cc084c3775"
+  integrity sha512-b1A8D8ZzE/VhNDoV1MSJTnpKkCG5bJo+19R4o4oy03zM7ws8yEMK755j61Dc3EyvdysbqH5BOOTquJ7ZX9C6vQ==
+  dependencies:
+    "@babel/helper-module-imports" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-remap-async-to-generator" "^7.22.5"
+
+"@babel/plugin-transform-block-scoped-functions@^7.0.0", "@babel/plugin-transform-block-scoped-functions@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.22.5.tgz#27978075bfaeb9fa586d3cb63a3d30c1de580024"
+  integrity sha512-tdXZ2UdknEKQWKJP1KMNmuF5Lx3MymtMN/pvA+p/VEkhK8jVcQ1fzSy8KM9qRYhAf2/lV33hoMPKI/xaI9sADA==
+  dependencies:
+    "@babel/helper-plugin-utils" "^7.22.5"
+
+"@babel/plugin-transform-block-scoping@^7.0.0", "@babel/plugin-transform-block-scoping@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.22.5.tgz#8bfc793b3a4b2742c0983fadc1480d843ecea31b"
+  integrity sha512-EcACl1i5fSQ6bt+YGuU/XGCeZKStLmyVGytWkpyhCLeQVA0eu6Wtiw92V+I1T/hnezUv7j74dA/Ro69gWcU+hg==
+  dependencies:
+    "@babel/helper-plugin-utils" "^7.22.5"
+
+"@babel/plugin-transform-class-properties@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-class-properties/-/plugin-transform-class-properties-7.22.5.tgz#97a56e31ad8c9dc06a0b3710ce7803d5a48cca77"
+  integrity sha512-nDkQ0NfkOhPTq8YCLiWNxp1+f9fCobEjCb0n8WdbNUBc4IB5V7P1QnX9IjpSoquKrXF5SKojHleVNs2vGeHCHQ==
+  dependencies:
+    "@babel/helper-create-class-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+
+"@babel/plugin-transform-class-static-block@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-class-static-block/-/plugin-transform-class-static-block-7.22.5.tgz#3e40c46f048403472d6f4183116d5e46b1bff5ba"
+  integrity sha512-SPToJ5eYZLxlnp1UzdARpOGeC2GbHvr9d/UV0EukuVx8atktg194oe+C5BqQ8jRTkgLRVOPYeXRSBg1IlMoVRA==
+  dependencies:
+    "@babel/helper-create-class-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-class-static-block" "^7.14.5"
+
+"@babel/plugin-transform-classes@^7.0.0", "@babel/plugin-transform-classes@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.22.5.tgz#635d4e98da741fad814984639f4c0149eb0135e1"
+  integrity sha512-2edQhLfibpWpsVBx2n/GKOz6JdGQvLruZQfGr9l1qes2KQaWswjBzhQF7UDUZMNaMMQeYnQzxwOMPsbYF7wqPQ==
+  dependencies:
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-compilation-targets" "^7.22.5"
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-function-name" "^7.22.5"
+    "@babel/helper-optimise-call-expression" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-replace-supers" "^7.22.5"
+    "@babel/helper-split-export-declaration" "^7.22.5"
     globals "^11.1.0"
 
-"@babel/plugin-transform-computed-properties@^7.0.0":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.20.7.tgz#704cc2fd155d1c996551db8276d55b9d46e4d0aa"
-  integrity sha512-Lz7MvBK6DTjElHAmfu6bfANzKcxpyNPeYBGEafyA6E5HtRpjpZwU+u7Qrgz/2OR0z+5TvKYbPdphfSaAcZBrYQ==
+"@babel/plugin-transform-computed-properties@^7.0.0", "@babel/plugin-transform-computed-properties@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.22.5.tgz#cd1e994bf9f316bd1c2dafcd02063ec261bb3869"
+  integrity sha512-4GHWBgRf0krxPX+AaPtgBAlTgTeZmqDynokHOX7aqqAB4tHs3U2Y02zH6ETFdLZGcg9UQSD1WCmkVrE9ErHeOg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.20.2"
-    "@babel/template" "^7.20.7"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/template" "^7.22.5"
 
-"@babel/plugin-transform-computed-properties@^7.12.1", "@babel/plugin-transform-computed-properties@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.13.0.tgz#845c6e8b9bb55376b1fa0b92ef0bdc8ea06644ed"
-  integrity sha512-RRqTYTeZkZAz8WbieLTvKUEUxZlUTdmL5KGMyZj7FnMfLNKV4+r5549aORG/mgojRmFlQMJDUupwAMiF2Q7OUg==
+"@babel/plugin-transform-destructuring@^7.0.0", "@babel/plugin-transform-destructuring@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.22.5.tgz#d3aca7438f6c26c78cdd0b0ba920a336001b27cc"
+  integrity sha512-GfqcFuGW8vnEqTUBM7UtPd5A4q797LTvvwKxXTgRsFjoqaJiEg9deBG6kWeQYkVEL569NpnmpC0Pkr/8BLKGnQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-destructuring@^7.0.0":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.20.7.tgz#8bda578f71620c7de7c93af590154ba331415454"
-  integrity sha512-Xwg403sRrZb81IVB79ZPqNQME23yhugYVqgTxAhT99h485F4f+GMELFhhOsscDUB7HCswepKeCKLn/GZvUKoBA==
+"@babel/plugin-transform-dotall-regex@^7.22.5", "@babel/plugin-transform-dotall-regex@^7.4.4":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.22.5.tgz#dbb4f0e45766eb544e193fb00e65a1dd3b2a4165"
+  integrity sha512-5/Yk9QxCQCl+sOIB1WelKnVRxTJDSAIxtJLL2/pqL14ZVlbH0fUQUZa/T5/UnQtBNgghR7mfB8ERBKyKPCi7Vw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.20.2"
+    "@babel/helper-create-regexp-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-destructuring@^7.12.1", "@babel/plugin-transform-destructuring@^7.13.17":
-  version "7.13.17"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.13.17.tgz#678d96576638c19d5b36b332504d3fd6e06dea27"
-  integrity sha512-UAUqiLv+uRLO+xuBKKMEpC+t7YRNVRqBsWWq1yKXbBZBje/t3IXCiSinZhjn/DC3qzBfICeYd2EFGEbHsh5RLA==
+"@babel/plugin-transform-duplicate-keys@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.22.5.tgz#b6e6428d9416f5f0bba19c70d1e6e7e0b88ab285"
+  integrity sha512-dEnYD+9BBgld5VBXHnF/DbYGp3fqGMsyxKbtD1mDyIA7AkTSpKXFhCVuj/oQVOoALfBs77DudA0BE4d5mcpmqw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-dotall-regex@^7.12.1", "@babel/plugin-transform-dotall-regex@^7.12.13", "@babel/plugin-transform-dotall-regex@^7.4.4":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.12.13.tgz#3f1601cc29905bfcb67f53910f197aeafebb25ad"
-  integrity sha512-foDrozE65ZFdUC2OfgeOCrEPTxdB3yjqxpXh8CH+ipd9CHd4s/iq81kcUpyH8ACGNEPdFqbtzfgzbT/ZGlbDeQ==
+"@babel/plugin-transform-dynamic-import@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dynamic-import/-/plugin-transform-dynamic-import-7.22.5.tgz#d6908a8916a810468c4edff73b5b75bda6ad393e"
+  integrity sha512-0MC3ppTB1AMxd8fXjSrbPa7LT9hrImt+/fcj+Pg5YMD7UQyWp/02+JWpdnCymmsXwIx5Z+sYn1bwCn4ZJNvhqQ==
   dependencies:
-    "@babel/helper-create-regexp-features-plugin" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-dynamic-import" "^7.8.3"
 
-"@babel/plugin-transform-duplicate-keys@^7.12.1", "@babel/plugin-transform-duplicate-keys@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.12.13.tgz#6f06b87a8b803fd928e54b81c258f0a0033904de"
-  integrity sha512-NfADJiiHdhLBW3pulJlJI2NB0t4cci4WTZ8FtdIuNc2+8pslXdPtRRAEWqUY+m9kNOk2eRYbTAOipAxlrOcwwQ==
+"@babel/plugin-transform-exponentiation-operator@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.22.5.tgz#402432ad544a1f9a480da865fda26be653e48f6a"
+  integrity sha512-vIpJFNM/FjZ4rh1myqIya9jXwrwwgFRHPjT3DkUA9ZLHuzox8jiXkOLvwm1H+PQIP3CqfC++WPKeuDi0Sjdj1g==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-builder-binary-assignment-operator-visitor" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-exponentiation-operator@^7.12.1", "@babel/plugin-transform-exponentiation-operator@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.12.13.tgz#4d52390b9a273e651e4aba6aee49ef40e80cd0a1"
-  integrity sha512-fbUelkM1apvqez/yYx1/oICVnGo2KM5s63mhGylrmXUxK/IAXSIf87QIxVfZldWf4QsOafY6vV3bX8aMHSvNrA==
+"@babel/plugin-transform-export-namespace-from@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-export-namespace-from/-/plugin-transform-export-namespace-from-7.22.5.tgz#57c41cb1d0613d22f548fddd8b288eedb9973a5b"
+  integrity sha512-X4hhm7FRnPgd4nDA4b/5V280xCx6oL7Oob5+9qVS5C13Zq4bh1qq7LU0GgRU6b5dBWBvhGaXYVB4AcN6+ol6vg==
   dependencies:
-    "@babel/helper-builder-binary-assignment-operator-visitor" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-export-namespace-from" "^7.8.3"
 
-"@babel/plugin-transform-flow-strip-types@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.12.1.tgz#8430decfa7eb2aea5414ed4a3fa6e1652b7d77c4"
-  integrity sha512-8hAtkmsQb36yMmEtk2JZ9JnVyDSnDOdlB+0nEGzIDLuK4yR3JcEjfuFPYkdEPSh8Id+rAMeBEn+X0iVEyho6Hg==
+"@babel/plugin-transform-flow-strip-types@^7.0.0", "@babel/plugin-transform-flow-strip-types@^7.16.0":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.22.5.tgz#0bb17110c7bf5b35a60754b2f00c58302381dee2"
+  integrity sha512-tujNbZdxdG0/54g/oua8ISToaXTFBf8EnSb5PgQSciIXWOWKX3S4+JR7ZE9ol8FZwf9kxitzkGQ+QWeov/mCiA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/plugin-syntax-flow" "^7.12.1"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-flow" "^7.22.5"
 
-"@babel/plugin-transform-flow-strip-types@^7.0.0":
-  version "7.19.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.19.0.tgz#e9e8606633287488216028719638cbbb2f2dde8f"
-  integrity sha512-sgeMlNaQVbCSpgLSKP4ZZKfsJVnFnNQlUSk6gPYzR/q7tzCgQF2t8RBKAP6cKJeZdveei7Q7Jm527xepI8lNLg==
+"@babel/plugin-transform-for-of@^7.0.0", "@babel/plugin-transform-for-of@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.22.5.tgz#ab1b8a200a8f990137aff9a084f8de4099ab173f"
+  integrity sha512-3kxQjX1dU9uudwSshyLeEipvrLjBCVthCgeTp6CzE/9JYrlAIaeekVxRpCWsDDfYTfRZRoCeZatCQvwo+wvK8A==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.19.0"
-    "@babel/plugin-syntax-flow" "^7.18.6"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-for-of@^7.0.0":
-  version "7.18.8"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.8.tgz#6ef8a50b244eb6a0bdbad0c7c61877e4e30097c1"
-  integrity sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ==
+"@babel/plugin-transform-function-name@^7.0.0", "@babel/plugin-transform-function-name@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.22.5.tgz#935189af68b01898e0d6d99658db6b164205c143"
+  integrity sha512-UIzQNMS0p0HHiQm3oelztj+ECwFnj+ZRV4KnguvlsD2of1whUeM6o7wGNj6oLwcDoAXQ8gEqfgC24D+VdIcevg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
+    "@babel/helper-compilation-targets" "^7.22.5"
+    "@babel/helper-function-name" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-for-of@^7.12.1", "@babel/plugin-transform-for-of@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.13.0.tgz#c799f881a8091ac26b54867a845c3e97d2696062"
-  integrity sha512-IHKT00mwUVYE0zzbkDgNRP6SRzvfGCYsOxIRz8KsiaaHCcT9BWIkO+H9QRJseHBLOGBZkHUdHiqj6r0POsdytg==
+"@babel/plugin-transform-json-strings@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-json-strings/-/plugin-transform-json-strings-7.22.5.tgz#14b64352fdf7e1f737eed68de1a1468bd2a77ec0"
+  integrity sha512-DuCRB7fu8MyTLbEQd1ew3R85nx/88yMoqo2uPSjevMj3yoN7CDM8jkgrY0wmVxfJZyJ/B9fE1iq7EQppWQmR5A==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-json-strings" "^7.8.3"
 
-"@babel/plugin-transform-function-name@^7.0.0":
-  version "7.18.9"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.9.tgz#cc354f8234e62968946c61a46d6365440fc764e0"
-  integrity sha512-WvIBoRPaJQ5yVHzcnJFor7oS5Ls0PYixlTYE63lCj2RtdQEl15M68FXQlxnG6wdraJIXRdR7KI+hQ7q/9QjrCQ==
+"@babel/plugin-transform-literals@^7.0.0", "@babel/plugin-transform-literals@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.22.5.tgz#e9341f4b5a167952576e23db8d435849b1dd7920"
+  integrity sha512-fTLj4D79M+mepcw3dgFBTIDYpbcB9Sm0bpm4ppXPaO+U+PKFFyV9MGRvS0gvGw62sd10kT5lRMKXAADb9pWy8g==
   dependencies:
-    "@babel/helper-compilation-targets" "^7.18.9"
-    "@babel/helper-function-name" "^7.18.9"
-    "@babel/helper-plugin-utils" "^7.18.9"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-function-name@^7.12.1", "@babel/plugin-transform-function-name@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.12.13.tgz#bb024452f9aaed861d374c8e7a24252ce3a50051"
-  integrity sha512-6K7gZycG0cmIwwF7uMK/ZqeCikCGVBdyP2J5SKNCXO5EOHcqi+z7Jwf8AmyDNcBgxET8DrEtCt/mPKPyAzXyqQ==
+"@babel/plugin-transform-logical-assignment-operators@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-logical-assignment-operators/-/plugin-transform-logical-assignment-operators-7.22.5.tgz#66ae5f068fd5a9a5dc570df16f56c2a8462a9d6c"
+  integrity sha512-MQQOUW1KL8X0cDWfbwYP+TbVbZm16QmQXJQ+vndPtH/BoO0lOKpVoEDMI7+PskYxH+IiE0tS8xZye0qr1lGzSA==
   dependencies:
-    "@babel/helper-function-name" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4"
 
-"@babel/plugin-transform-literals@^7.0.0":
-  version "7.18.9"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.9.tgz#72796fdbef80e56fba3c6a699d54f0de557444bc"
-  integrity sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg==
+"@babel/plugin-transform-member-expression-literals@^7.0.0", "@babel/plugin-transform-member-expression-literals@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.22.5.tgz#4fcc9050eded981a468347dd374539ed3e058def"
+  integrity sha512-RZEdkNtzzYCFl9SE9ATaUMTj2hqMb4StarOJLrZRbqqU4HSBE7UlBw9WBWQiDzrJZJdUWiMTVDI6Gv/8DPvfew==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.9"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-literals@^7.12.1", "@babel/plugin-transform-literals@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.12.13.tgz#2ca45bafe4a820197cf315794a4d26560fe4bdb9"
-  integrity sha512-FW+WPjSR7hiUxMcKqyNjP05tQ2kmBCdpEpZHY1ARm96tGQCCBvXKnpjILtDplUnJ/eHZ0lALLM+d2lMFSpYJrQ==
+"@babel/plugin-transform-modules-amd@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.22.5.tgz#4e045f55dcf98afd00f85691a68fc0780704f526"
+  integrity sha512-R+PTfLTcYEmb1+kK7FNkhQ1gP4KgjpSO6HfH9+f8/yfp2Nt3ggBjiVpRwmwTlfqZLafYKJACy36yDXlEmI9HjQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-module-transforms" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-member-expression-literals@^7.0.0":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz#ac9fdc1a118620ac49b7e7a5d2dc177a1bfee88e"
-  integrity sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==
+"@babel/plugin-transform-modules-commonjs@^7.0.0", "@babel/plugin-transform-modules-commonjs@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.22.5.tgz#7d9875908d19b8c0536085af7b053fd5bd651bfa"
+  integrity sha512-B4pzOXj+ONRmuaQTg05b3y/4DuFz3WcCNAXPLb2Q0GT0TrGKGxNKV4jwsXts+StaM0LQczZbOpj8o1DLPDJIiA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
+    "@babel/helper-module-transforms" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-simple-access" "^7.22.5"
 
-"@babel/plugin-transform-member-expression-literals@^7.12.1", "@babel/plugin-transform-member-expression-literals@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.12.13.tgz#5ffa66cd59b9e191314c9f1f803b938e8c081e40"
-  integrity sha512-kxLkOsg8yir4YeEPHLuO2tXP9R/gTjpuTOjshqSpELUN3ZAg2jfDnKUvzzJxObun38sw3wm4Uu69sX/zA7iRvg==
+"@babel/plugin-transform-modules-systemjs@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.22.5.tgz#18c31410b5e579a0092638f95c896c2a98a5d496"
+  integrity sha512-emtEpoaTMsOs6Tzz+nbmcePl6AKVtS1yC4YNAeMun9U8YCsgadPNxnOPQ8GhHFB2qdx+LZu9LgoC0Lthuu05DQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-hoist-variables" "^7.22.5"
+    "@babel/helper-module-transforms" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-validator-identifier" "^7.22.5"
 
-"@babel/plugin-transform-modules-amd@^7.12.1", "@babel/plugin-transform-modules-amd@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.14.2.tgz#6622806fe1a7c07a1388444222ef9535f2ca17b0"
-  integrity sha512-hPC6XBswt8P3G2D1tSV2HzdKvkqOpmbyoy+g73JG0qlF/qx2y3KaMmXb1fLrpmWGLZYA0ojCvaHdzFWjlmV+Pw==
+"@babel/plugin-transform-modules-umd@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.22.5.tgz#4694ae40a87b1745e3775b6a7fe96400315d4f98"
+  integrity sha512-+S6kzefN/E1vkSsKx8kmQuqeQsvCKCd1fraCM7zXm4SFoggI099Tr4G8U81+5gtMdUeMQ4ipdQffbKLX0/7dBQ==
   dependencies:
-    "@babel/helper-module-transforms" "^7.14.2"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    babel-plugin-dynamic-import-node "^2.3.3"
+    "@babel/helper-module-transforms" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-modules-commonjs@^7.0.0":
-  version "7.20.11"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.20.11.tgz#8cb23010869bf7669fd4b3098598b6b2be6dc607"
-  integrity sha512-S8e1f7WQ7cimJQ51JkAaDrEtohVEitXjgCGAS2N8S31Y42E+kWwfSz83LYz57QdBm7q9diARVqanIaH2oVgQnw==
+"@babel/plugin-transform-named-capturing-groups-regex@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.22.5.tgz#67fe18ee8ce02d57c855185e27e3dc959b2e991f"
+  integrity sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==
   dependencies:
-    "@babel/helper-module-transforms" "^7.20.11"
-    "@babel/helper-plugin-utils" "^7.20.2"
-    "@babel/helper-simple-access" "^7.20.2"
+    "@babel/helper-create-regexp-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-modules-commonjs@^7.12.1", "@babel/plugin-transform-modules-commonjs@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.14.0.tgz#52bc199cb581e0992edba0f0f80356467587f161"
-  integrity sha512-EX4QePlsTaRZQmw9BsoPeyh5OCtRGIhwfLquhxGp5e32w+dyL8htOcDwamlitmNFK6xBZYlygjdye9dbd9rUlQ==
+"@babel/plugin-transform-new-target@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.22.5.tgz#1b248acea54ce44ea06dfd37247ba089fcf9758d"
+  integrity sha512-AsF7K0Fx/cNKVyk3a+DW0JLo+Ua598/NxMRvxDnkpCIGFh43+h/v2xyhRUYf6oD8gE4QtL83C7zZVghMjHd+iw==
   dependencies:
-    "@babel/helper-module-transforms" "^7.14.0"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-simple-access" "^7.13.12"
-    babel-plugin-dynamic-import-node "^2.3.3"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-modules-systemjs@^7.12.1", "@babel/plugin-transform-modules-systemjs@^7.13.8":
-  version "7.13.8"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.13.8.tgz#6d066ee2bff3c7b3d60bf28dec169ad993831ae3"
-  integrity sha512-hwqctPYjhM6cWvVIlOIe27jCIBgHCsdH2xCJVAYQm7V5yTMoilbVMi9f6wKg0rpQAOn6ZG4AOyvCqFF/hUh6+A==
+"@babel/plugin-transform-nullish-coalescing-operator@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.22.5.tgz#f8872c65776e0b552e0849d7596cddd416c3e381"
+  integrity sha512-6CF8g6z1dNYZ/VXok5uYkkBBICHZPiGEl7oDnAx2Mt1hlHVHOSIKWJaXHjQJA5VB43KZnXZDIexMchY4y2PGdA==
   dependencies:
-    "@babel/helper-hoist-variables" "^7.13.0"
-    "@babel/helper-module-transforms" "^7.13.0"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-validator-identifier" "^7.12.11"
-    babel-plugin-dynamic-import-node "^2.3.3"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3"
 
-"@babel/plugin-transform-modules-umd@^7.12.1", "@babel/plugin-transform-modules-umd@^7.14.0":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.14.0.tgz#2f8179d1bbc9263665ce4a65f305526b2ea8ac34"
-  integrity sha512-nPZdnWtXXeY7I87UZr9VlsWme3Y0cfFFE41Wbxz4bbaexAjNMInXPFUpRRUJ8NoMm0Cw+zxbqjdPmLhcjfazMw==
+"@babel/plugin-transform-numeric-separator@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-numeric-separator/-/plugin-transform-numeric-separator-7.22.5.tgz#57226a2ed9e512b9b446517ab6fa2d17abb83f58"
+  integrity sha512-NbslED1/6M+sXiwwtcAB/nieypGw02Ejf4KtDeMkCEpP6gWFMX1wI9WKYua+4oBneCCEmulOkRpwywypVZzs/g==
   dependencies:
-    "@babel/helper-module-transforms" "^7.14.0"
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-numeric-separator" "^7.10.4"
 
-"@babel/plugin-transform-named-capturing-groups-regex@^7.12.1", "@babel/plugin-transform-named-capturing-groups-regex@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.13.tgz#2213725a5f5bbbe364b50c3ba5998c9599c5c9d9"
-  integrity sha512-Xsm8P2hr5hAxyYblrfACXpQKdQbx4m2df9/ZZSQ8MAhsadw06+jW7s9zsSw6he+mJZXRlVMyEnVktJo4zjk1WA==
+"@babel/plugin-transform-object-rest-spread@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-rest-spread/-/plugin-transform-object-rest-spread-7.22.5.tgz#9686dc3447df4753b0b2a2fae7e8bc33cdc1f2e1"
+  integrity sha512-Kk3lyDmEslH9DnvCDA1s1kkd3YWQITiBOHngOtDL9Pt6BZjzqb6hiOlb8VfjiiQJ2unmegBqZu0rx5RxJb5vmQ==
   dependencies:
-    "@babel/helper-create-regexp-features-plugin" "^7.12.13"
+    "@babel/compat-data" "^7.22.5"
+    "@babel/helper-compilation-targets" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-object-rest-spread" "^7.8.3"
+    "@babel/plugin-transform-parameters" "^7.22.5"
 
-"@babel/plugin-transform-new-target@^7.12.1", "@babel/plugin-transform-new-target@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.13.tgz#e22d8c3af24b150dd528cbd6e685e799bf1c351c"
-  integrity sha512-/KY2hbLxrG5GTQ9zzZSc3xWiOy379pIETEhbtzwZcw9rvuaVV4Fqy7BYGYOWZnaoXIQYbbJ0ziXLa/sKcGCYEQ==
+"@babel/plugin-transform-object-super@^7.0.0", "@babel/plugin-transform-object-super@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.22.5.tgz#794a8d2fcb5d0835af722173c1a9d704f44e218c"
+  integrity sha512-klXqyaT9trSjIUrcsYIfETAzmOEZL3cBYqOYLJxBHfMFFggmXOv+NYSX/Jbs9mzMVESw/WycLFPRx8ba/b2Ipw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-replace-supers" "^7.22.5"
 
-"@babel/plugin-transform-object-super@^7.0.0":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz#fb3c6ccdd15939b6ff7939944b51971ddc35912c"
-  integrity sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==
+"@babel/plugin-transform-optional-catch-binding@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-optional-catch-binding/-/plugin-transform-optional-catch-binding-7.22.5.tgz#842080be3076703be0eaf32ead6ac8174edee333"
+  integrity sha512-pH8orJahy+hzZje5b8e2QIlBWQvGpelS76C63Z+jhZKsmzfNaPQ+LaW6dcJ9bxTpo1mtXbgHwy765Ro3jftmUg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
-    "@babel/helper-replace-supers" "^7.18.6"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-optional-catch-binding" "^7.8.3"
 
-"@babel/plugin-transform-object-super@^7.12.1", "@babel/plugin-transform-object-super@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.12.13.tgz#b4416a2d63b8f7be314f3d349bd55a9c1b5171f7"
-  integrity sha512-JzYIcj3XtYspZDV8j9ulnoMPZZnF/Cj0LUxPOjR89BdBVx+zYJI9MdMIlUZjbXDX+6YVeS6I3e8op+qQ3BYBoQ==
+"@babel/plugin-transform-optional-chaining@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.22.5.tgz#1003762b9c14295501beb41be72426736bedd1e0"
+  integrity sha512-AconbMKOMkyG+xCng2JogMCDcqW8wedQAqpVIL4cOSescZ7+iW8utC6YDZLMCSUIReEA733gzRSaOSXMAt/4WQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
-    "@babel/helper-replace-supers" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-skip-transparent-expression-wrappers" "^7.22.5"
+    "@babel/plugin-syntax-optional-chaining" "^7.8.3"
 
-"@babel/plugin-transform-parameters@^7.0.0", "@babel/plugin-transform-parameters@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.20.7.tgz#0ee349e9d1bc96e78e3b37a7af423a4078a7083f"
-  integrity sha512-WiWBIkeHKVOSYPO0pWkxGPfKeWrCJyD3NJ53+Lrp/QMSZbsVPovrVl2aWZ19D/LTVnaDv5Ap7GJ/B2CTOZdrfA==
+"@babel/plugin-transform-parameters@^7.0.0", "@babel/plugin-transform-parameters@^7.20.7", "@babel/plugin-transform-parameters@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.22.5.tgz#c3542dd3c39b42c8069936e48717a8d179d63a18"
+  integrity sha512-AVkFUBurORBREOmHRKo06FjHYgjrabpdqRSwq6+C7R5iTCZOsM4QbcB27St0a4U6fffyAOqh3s/qEfybAhfivg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.20.2"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-parameters@^7.12.1", "@babel/plugin-transform-parameters@^7.14.2":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.14.2.tgz#e4290f72e0e9e831000d066427c4667098decc31"
-  integrity sha512-NxoVmA3APNCC1JdMXkdYXuQS+EMdqy0vIwyDHeKHiJKRxmp1qGSdb0JLEIoPRhkx6H/8Qi3RJ3uqOCYw8giy9A==
+"@babel/plugin-transform-private-methods@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-private-methods/-/plugin-transform-private-methods-7.22.5.tgz#21c8af791f76674420a147ae62e9935d790f8722"
+  integrity sha512-PPjh4gyrQnGe97JTalgRGMuU4icsZFnWkzicB/fUtzlKUqvsWBKEpPPfr5a2JiyirZkHxnAqkQMO5Z5B2kK3fA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-create-class-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-property-literals@^7.0.0":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz#e22498903a483448e94e032e9bbb9c5ccbfc93a3"
-  integrity sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==
+"@babel/plugin-transform-private-property-in-object@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-private-property-in-object/-/plugin-transform-private-property-in-object-7.22.5.tgz#07a77f28cbb251546a43d175a1dda4cf3ef83e32"
+  integrity sha512-/9xnaTTJcVoBtSSmrVyhtSvO3kbqS2ODoh2juEU72c3aYonNF0OMGiaz2gjukyKM2wBBYJP38S4JiE0Wfb5VMQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-create-class-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-private-property-in-object" "^7.14.5"
 
-"@babel/plugin-transform-property-literals@^7.12.1", "@babel/plugin-transform-property-literals@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.12.13.tgz#4e6a9e37864d8f1b3bc0e2dce7bf8857db8b1a81"
-  integrity sha512-nqVigwVan+lR+g8Fj8Exl0UQX2kymtjcWfMOYM1vTYEKujeyv2SkMgazf2qNcK7l4SDiKyTA/nHCPqL4e2zo1A==
+"@babel/plugin-transform-property-literals@^7.0.0", "@babel/plugin-transform-property-literals@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.22.5.tgz#b5ddabd73a4f7f26cd0e20f5db48290b88732766"
+  integrity sha512-TiOArgddK3mK/x1Qwf5hay2pxI6wCZnvQqrFSqbtg1GLl2JcNMitVH/YnqjP+M31pLUeTfzY1HAXFDnUBV30rQ==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
 "@babel/plugin-transform-react-constant-elements@^7.12.1":
   version "7.13.13"
@@ -1422,286 +1038,163 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.13.0"
 
-"@babel/plugin-transform-react-display-name@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.12.1.tgz#1cbcd0c3b1d6648c55374a22fc9b6b7e5341c00d"
-  integrity sha512-cAzB+UzBIrekfYxyLlFqf/OagTvHLcVBb5vpouzkYkBclRPraiygVnafvAoipErZLI8ANv8Ecn6E/m5qPXD26w==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-
-"@babel/plugin-transform-react-display-name@^7.0.0":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.18.6.tgz#8b1125f919ef36ebdfff061d664e266c666b9415"
-  integrity sha512-TV4sQ+T013n61uMoygyMRm+xf04Bd5oqFpv2jAEQwSZ8NwQA7zeRPg1LMVg2PWi3zWBz+CLKD+v5bcpZ/BS0aA==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
-
-"@babel/plugin-transform-react-display-name@^7.12.1", "@babel/plugin-transform-react-display-name@^7.12.13":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.14.2.tgz#2e854544d42ab3bb9c21f84e153d62e800fbd593"
-  integrity sha512-zCubvP+jjahpnFJvPaHPiGVfuVUjXHhFvJKQdNnsmSsiU9kR/rCZ41jHc++tERD2zV+p7Hr6is+t5b6iWTCqSw==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-
-"@babel/plugin-transform-react-jsx-development@^7.12.1", "@babel/plugin-transform-react-jsx-development@^7.12.17":
-  version "7.12.17"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.12.17.tgz#f510c0fa7cd7234153539f9a362ced41a5ca1447"
-  integrity sha512-BPjYV86SVuOaudFhsJR1zjgxxOhJDt6JHNoD48DxWEIxUCAMjV1ys6DYw4SDYZh0b1QsS2vfIA9t/ZsQGsDOUQ==
+"@babel/plugin-transform-react-display-name@^7.0.0", "@babel/plugin-transform-react-display-name@^7.16.0", "@babel/plugin-transform-react-display-name@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.22.5.tgz#3c4326f9fce31c7968d6cb9debcaf32d9e279a2b"
+  integrity sha512-PVk3WPYudRF5z4GKMEYUrLjPl38fJSKNaEOkFuoprioowGuWN6w2RKznuFNSlJx7pzzXXStPUnNSOEO0jL5EVw==
   dependencies:
-    "@babel/plugin-transform-react-jsx" "^7.12.17"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-react-jsx-self@^7.12.1":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.12.13.tgz#422d99d122d592acab9c35ea22a6cfd9bf189f60"
-  integrity sha512-FXYw98TTJ125GVCCkFLZXlZ1qGcsYqNQhVBQcZjyrwf8FEUtVfKIoidnO8S0q+KBQpDYNTmiGo1gn67Vti04lQ==
+"@babel/plugin-transform-react-jsx-development@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.22.5.tgz#e716b6edbef972a92165cd69d92f1255f7e73e87"
+  integrity sha512-bDhuzwWMuInwCYeDeMzyi7TaBgRQei6DqxhbyniL7/VG4RSS7HtSL2QbY4eESy1KJqlWt8g3xeEBGPuo+XqC8A==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/plugin-transform-react-jsx" "^7.22.5"
 
-"@babel/plugin-transform-react-jsx-source@^7.12.1":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.14.2.tgz#2620b57e7de775c0687f65d464026d15812941da"
-  integrity sha512-OMorspVyjxghAjzgeAWc6O7W7vHbJhV69NeTGdl9Mxgz6PaweAuo7ffB9T5A1OQ9dGcw0As4SYMUhyNC4u7mVg==
+"@babel/plugin-transform-react-jsx@^7.0.0", "@babel/plugin-transform-react-jsx@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.22.5.tgz#932c291eb6dd1153359e2a90cb5e557dcf068416"
+  integrity sha512-rog5gZaVbUip5iWDMTYbVM15XQq+RkUKhET/IHR6oizR+JEoN6CAfTTuHcK4vwUyzca30qqHqEpzBOnaRMWYMA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-module-imports" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-jsx" "^7.22.5"
+    "@babel/types" "^7.22.5"
 
-"@babel/plugin-transform-react-jsx@^7.0.0":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.7.tgz#025d85a1935fd7e19dfdcb1b1d4df34d4da484f7"
-  integrity sha512-Tfq7qqD+tRj3EoDhY00nn2uP2hsRxgYGi5mLQ5TimKav0a9Lrpd4deE+fcLXU8zFYRjlKPHZhpCvfEA6qnBxqQ==
+"@babel/plugin-transform-react-pure-annotations@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.22.5.tgz#1f58363eef6626d6fa517b95ac66fe94685e32c0"
+  integrity sha512-gP4k85wx09q+brArVinTXhWiyzLl9UpmGva0+mWyKxk6JZequ05x3eUcIUE+FyttPKJFRRVtAvQaJ6YF9h1ZpA==
   dependencies:
-    "@babel/helper-annotate-as-pure" "^7.18.6"
-    "@babel/helper-module-imports" "^7.18.6"
-    "@babel/helper-plugin-utils" "^7.20.2"
-    "@babel/plugin-syntax-jsx" "^7.18.6"
-    "@babel/types" "^7.20.7"
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-react-jsx@^7.12.1", "@babel/plugin-transform-react-jsx@^7.12.17", "@babel/plugin-transform-react-jsx@^7.13.12":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.14.3.tgz#0e26597805cf0862da735f264550933c38babb66"
-  integrity sha512-uuxuoUNVhdgYzERiHHFkE4dWoJx+UFVyuAl0aqN8P2/AKFHwqgUC5w2+4/PjpKXJsFgBlYAFXlUmDQ3k3DUkXw==
+"@babel/plugin-transform-regenerator@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.22.5.tgz#cd8a68b228a5f75fa01420e8cc2fc400f0fc32aa"
+  integrity sha512-rR7KePOE7gfEtNTh9Qw+iO3Q/e4DEsoQ+hdvM6QUDH7JRJ5qxq5AA52ZzBWbI5i9lfNuvySgOGP8ZN7LAmaiPw==
   dependencies:
-    "@babel/helper-annotate-as-pure" "^7.12.13"
-    "@babel/helper-module-imports" "^7.13.12"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-jsx" "^7.12.13"
-    "@babel/types" "^7.14.2"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    regenerator-transform "^0.15.1"
 
-"@babel/plugin-transform-react-pure-annotations@^7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.12.1.tgz#05d46f0ab4d1339ac59adf20a1462c91b37a1a42"
-  integrity sha512-RqeaHiwZtphSIUZ5I85PEH19LOSzxfuEazoY7/pWASCAIBuATQzpSVD+eT6MebeeZT2F4eSL0u4vw6n4Nm0Mjg==
+"@babel/plugin-transform-reserved-words@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.22.5.tgz#832cd35b81c287c4bcd09ce03e22199641f964fb"
+  integrity sha512-DTtGKFRQUDm8svigJzZHzb/2xatPc6TzNvAIJ5GqOKDsGFYgAskjRulbR/vGsPKq3OPqtexnz327qYpP57RFyA==
   dependencies:
-    "@babel/helper-annotate-as-pure" "^7.10.4"
-    "@babel/helper-plugin-utils" "^7.10.4"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-regenerator@^7.12.1", "@babel/plugin-transform-regenerator@^7.13.15":
-  version "7.13.15"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.13.15.tgz#e5eb28945bf8b6563e7f818945f966a8d2997f39"
-  integrity sha512-Bk9cOLSz8DiurcMETZ8E2YtIVJbFCPGW28DJWUakmyVWtQSm6Wsf0p4B4BfEr/eL2Nkhe/CICiUiMOCi1TPhuQ==
+"@babel/plugin-transform-runtime@^7.16.4":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.22.5.tgz#ca975fb5e260044473c8142e1b18b567d33c2a3b"
+  integrity sha512-bg4Wxd1FWeFx3daHFTWk1pkSWK/AyQuiyAoeZAOkAOUBjnZPH6KT7eMxouV47tQ6hl6ax2zyAWBdWZXbrvXlaw==
   dependencies:
-    regenerator-transform "^0.14.2"
+    "@babel/helper-module-imports" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    babel-plugin-polyfill-corejs2 "^0.4.3"
+    babel-plugin-polyfill-corejs3 "^0.8.1"
+    babel-plugin-polyfill-regenerator "^0.5.0"
+    semver "^6.3.0"
 
-"@babel/plugin-transform-reserved-words@^7.12.1", "@babel/plugin-transform-reserved-words@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.12.13.tgz#7d9988d4f06e0fe697ea1d9803188aa18b472695"
-  integrity sha512-xhUPzDXxZN1QfiOy/I5tyye+TRz6lA7z6xaT4CLOjPRMVg1ldRf0LHw0TDBpYL4vG78556WuHdyO9oi5UmzZBg==
+"@babel/plugin-transform-shorthand-properties@^7.0.0", "@babel/plugin-transform-shorthand-properties@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.22.5.tgz#6e277654be82b5559fc4b9f58088507c24f0c624"
+  integrity sha512-vM4fq9IXHscXVKzDv5itkO1X52SmdFBFcMIBZ2FRn2nqVYqw6dBexUgMvAjHW+KXpPPViD/Yo3GrDEBaRC0QYA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-runtime@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.12.1.tgz#04b792057eb460389ff6a4198e377614ea1e7ba5"
-  integrity sha512-Ac/H6G9FEIkS2tXsZjL4RAdS3L3WHxci0usAnz7laPWUmFiGtj7tIASChqKZMHTSQTQY6xDbOq+V1/vIq3QrWg==
+"@babel/plugin-transform-spread@^7.0.0", "@babel/plugin-transform-spread@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.22.5.tgz#6487fd29f229c95e284ba6c98d65eafb893fea6b"
+  integrity sha512-5ZzDQIGyvN4w8+dMmpohL6MBo+l2G7tfC/O2Dg7/hjpgeWvUx8FzfeOKxGog9IimPa4YekaQ9PlDqTLOljkcxg==
   dependencies:
-    "@babel/helper-module-imports" "^7.12.1"
-    "@babel/helper-plugin-utils" "^7.10.4"
-    resolve "^1.8.1"
-    semver "^5.5.1"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-skip-transparent-expression-wrappers" "^7.22.5"
 
-"@babel/plugin-transform-shorthand-properties@^7.0.0":
-  version "7.18.6"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz#6d6df7983d67b195289be24909e3f12a8f664dc9"
-  integrity sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==
+"@babel/plugin-transform-sticky-regex@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.22.5.tgz#295aba1595bfc8197abd02eae5fc288c0deb26aa"
+  integrity sha512-zf7LuNpHG0iEeiyCNwX4j3gDg1jgt1k3ZdXBKbZSoA3BbGQGvMiSvfbZRR3Dr3aeJe3ooWFZxOOG3IRStYp2Bw==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.6"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-shorthand-properties@^7.12.1", "@babel/plugin-transform-shorthand-properties@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.12.13.tgz#db755732b70c539d504c6390d9ce90fe64aff7ad"
-  integrity sha512-xpL49pqPnLtf0tVluuqvzWIgLEhuPpZzvs2yabUHSKRNlN7ScYU7aMlmavOeyXJZKgZKQRBlh8rHbKiJDraTSw==
+"@babel/plugin-transform-template-literals@^7.0.0", "@babel/plugin-transform-template-literals@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.22.5.tgz#8f38cf291e5f7a8e60e9f733193f0bcc10909bff"
+  integrity sha512-5ciOehRNf+EyUeewo8NkbQiUs4d6ZxiHo6BcBcnFlgiJfu16q0bQUw9Jvo0b0gBKFG1SMhDSjeKXSYuJLeFSMA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-spread@^7.0.0":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.20.7.tgz#c2d83e0b99d3bf83e07b11995ee24bf7ca09401e"
-  integrity sha512-ewBbHQ+1U/VnH1fxltbJqDeWBU1oNLG8Dj11uIv3xVf7nrQu0bPGe5Rf716r7K5Qz+SqtAOVswoVunoiBtGhxw==
+"@babel/plugin-transform-typeof-symbol@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.22.5.tgz#5e2ba478da4b603af8673ff7c54f75a97b716b34"
+  integrity sha512-bYkI5lMzL4kPii4HHEEChkD0rkc+nvnlR6+o/qdqR6zrm0Sv/nodmyLhlq2DO0YKLUNd2VePmPRjJXSBh9OIdA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.20.2"
-    "@babel/helper-skip-transparent-expression-wrappers" "^7.20.0"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-spread@^7.12.1", "@babel/plugin-transform-spread@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.13.0.tgz#84887710e273c1815ace7ae459f6f42a5d31d5fd"
-  integrity sha512-V6vkiXijjzYeFmQTr3dBxPtZYLPcUfY34DebOU27jIl2M/Y8Egm52Hw82CSjjPqd54GTlJs5x+CR7HeNr24ckg==
+"@babel/plugin-transform-typescript@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.22.5.tgz#5c0f7adfc1b5f38c4dbc8f79b1f0f8074134bd7d"
+  integrity sha512-SMubA9S7Cb5sGSFFUlqxyClTA9zWJ8qGQrppNUm05LtFuN1ELRFNndkix4zUJrC9F+YivWwa1dHMSyo0e0N9dA==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1"
+    "@babel/helper-annotate-as-pure" "^7.22.5"
+    "@babel/helper-create-class-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/plugin-syntax-typescript" "^7.22.5"
 
-"@babel/plugin-transform-sticky-regex@^7.12.1", "@babel/plugin-transform-sticky-regex@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.13.tgz#760ffd936face73f860ae646fb86ee82f3d06d1f"
-  integrity sha512-Jc3JSaaWT8+fr7GRvQP02fKDsYk4K/lYwWq38r/UGfaxo89ajud321NH28KRQ7xy1Ybc0VUE5Pz8psjNNDUglg==
+"@babel/plugin-transform-unicode-escapes@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.22.5.tgz#ce0c248522b1cb22c7c992d88301a5ead70e806c"
+  integrity sha512-biEmVg1IYB/raUO5wT1tgfacCef15Fbzhkx493D3urBI++6hpJ+RFG4SrWMn0NEZLfvilqKf3QDrRVZHo08FYg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-template-literals@^7.0.0":
-  version "7.18.9"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.9.tgz#04ec6f10acdaa81846689d63fae117dd9c243a5e"
-  integrity sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA==
+"@babel/plugin-transform-unicode-property-regex@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-property-regex/-/plugin-transform-unicode-property-regex-7.22.5.tgz#098898f74d5c1e86660dc112057b2d11227f1c81"
+  integrity sha512-HCCIb+CbJIAE6sXn5CjFQXMwkCClcOfPCzTlilJ8cUatfzwHlWQkbtV0zD338u9dZskwvuOYTuuaMaA8J5EI5A==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.18.9"
+    "@babel/helper-create-regexp-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-template-literals@^7.12.1", "@babel/plugin-transform-template-literals@^7.13.0":
-  version "7.13.0"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.13.0.tgz#a36049127977ad94438dee7443598d1cefdf409d"
-  integrity sha512-d67umW6nlfmr1iehCcBv69eSUSySk1EsIS8aTDX4Xo9qajAh6mYtcl4kJrBkGXuxZPEgVr7RVfAvNW6YQkd4Mw==
+"@babel/plugin-transform-unicode-regex@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.22.5.tgz#ce7e7bb3ef208c4ff67e02a22816656256d7a183"
+  integrity sha512-028laaOKptN5vHJf9/Arr/HiJekMd41hOEZYvNsrsXqJ7YPYuX2bQxh31fkZzGmq3YqHRJzYFFAVYvKfMPKqyg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
+    "@babel/helper-create-regexp-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-typeof-symbol@^7.12.1", "@babel/plugin-transform-typeof-symbol@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.13.tgz#785dd67a1f2ea579d9c2be722de8c84cb85f5a7f"
-  integrity sha512-eKv/LmUJpMnu4npgfvs3LiHhJua5fo/CysENxa45YCQXZwKnGCQKAg87bvoqSW1fFT+HA32l03Qxsm8ouTY3ZQ==
+"@babel/plugin-transform-unicode-sets-regex@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-sets-regex/-/plugin-transform-unicode-sets-regex-7.22.5.tgz#77788060e511b708ffc7d42fdfbc5b37c3004e91"
+  integrity sha512-lhMfi4FC15j13eKrh3DnYHjpGj6UKQHtNKTbtc1igvAhRy4+kLhV07OpLcsN0VgDEw/MjAvJO4BdMJsHwMhzCg==
   dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
+    "@babel/helper-create-regexp-features-plugin" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/plugin-transform-typescript@^7.12.1":
-  version "7.14.3"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.14.3.tgz#44f67f725a60cccee33d9d6fee5e4f338258f34f"
-  integrity sha512-G5Bb5pY6tJRTC4ag1visSgiDoGgJ1u1fMUgmc2ijLkcIdzP83Q1qyZX4ggFQ/SkR+PNOatkaYC+nKcTlpsX4ag==
+"@babel/preset-env@^7.11.0", "@babel/preset-env@^7.12.1", "@babel/preset-env@^7.16.4":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.22.5.tgz#3da66078b181f3d62512c51cf7014392c511504e"
+  integrity sha512-fj06hw89dpiZzGZtxn+QybifF07nNiZjZ7sazs2aVDcysAZVGjW7+7iFYxg6GLNM47R/thYfLdrXc+2f11Vi9A==
   dependencies:
-    "@babel/helper-create-class-features-plugin" "^7.14.3"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/plugin-syntax-typescript" "^7.12.13"
-
-"@babel/plugin-transform-unicode-escapes@^7.12.1", "@babel/plugin-transform-unicode-escapes@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.12.13.tgz#840ced3b816d3b5127dd1d12dcedc5dead1a5e74"
-  integrity sha512-0bHEkdwJ/sN/ikBHfSmOXPypN/beiGqjo+o4/5K+vxEFNPRPdImhviPakMKG4x96l85emoa0Z6cDflsdBusZbw==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.12.13"
-
-"@babel/plugin-transform-unicode-regex@^7.12.1", "@babel/plugin-transform-unicode-regex@^7.12.13":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.12.13.tgz#b52521685804e155b1202e83fc188d34bb70f5ac"
-  integrity sha512-mDRzSNY7/zopwisPZ5kM9XKCfhchqIYwAKRERtEnhYscZB79VRekuRSoYbN0+KVe3y8+q1h6A4svXtP7N+UoCA==
-  dependencies:
-    "@babel/helper-create-regexp-features-plugin" "^7.12.13"
-    "@babel/helper-plugin-utils" "^7.12.13"
-
-"@babel/preset-env@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.12.1.tgz#9c7e5ca82a19efc865384bb4989148d2ee5d7ac2"
-  integrity sha512-H8kxXmtPaAGT7TyBvSSkoSTUK6RHh61So05SyEbpmr0MCZrsNYn7mGMzzeYoOUCdHzww61k8XBft2TaES+xPLg==
-  dependencies:
-    "@babel/compat-data" "^7.12.1"
-    "@babel/helper-compilation-targets" "^7.12.1"
-    "@babel/helper-module-imports" "^7.12.1"
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/helper-validator-option" "^7.12.1"
-    "@babel/plugin-proposal-async-generator-functions" "^7.12.1"
-    "@babel/plugin-proposal-class-properties" "^7.12.1"
-    "@babel/plugin-proposal-dynamic-import" "^7.12.1"
-    "@babel/plugin-proposal-export-namespace-from" "^7.12.1"
-    "@babel/plugin-proposal-json-strings" "^7.12.1"
-    "@babel/plugin-proposal-logical-assignment-operators" "^7.12.1"
-    "@babel/plugin-proposal-nullish-coalescing-operator" "^7.12.1"
-    "@babel/plugin-proposal-numeric-separator" "^7.12.1"
-    "@babel/plugin-proposal-object-rest-spread" "^7.12.1"
-    "@babel/plugin-proposal-optional-catch-binding" "^7.12.1"
-    "@babel/plugin-proposal-optional-chaining" "^7.12.1"
-    "@babel/plugin-proposal-private-methods" "^7.12.1"
-    "@babel/plugin-proposal-unicode-property-regex" "^7.12.1"
-    "@babel/plugin-syntax-async-generators" "^7.8.0"
-    "@babel/plugin-syntax-class-properties" "^7.12.1"
-    "@babel/plugin-syntax-dynamic-import" "^7.8.0"
-    "@babel/plugin-syntax-export-namespace-from" "^7.8.3"
-    "@babel/plugin-syntax-json-strings" "^7.8.0"
-    "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4"
-    "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0"
-    "@babel/plugin-syntax-numeric-separator" "^7.10.4"
-    "@babel/plugin-syntax-object-rest-spread" "^7.8.0"
-    "@babel/plugin-syntax-optional-catch-binding" "^7.8.0"
-    "@babel/plugin-syntax-optional-chaining" "^7.8.0"
-    "@babel/plugin-syntax-top-level-await" "^7.12.1"
-    "@babel/plugin-transform-arrow-functions" "^7.12.1"
-    "@babel/plugin-transform-async-to-generator" "^7.12.1"
-    "@babel/plugin-transform-block-scoped-functions" "^7.12.1"
-    "@babel/plugin-transform-block-scoping" "^7.12.1"
-    "@babel/plugin-transform-classes" "^7.12.1"
-    "@babel/plugin-transform-computed-properties" "^7.12.1"
-    "@babel/plugin-transform-destructuring" "^7.12.1"
-    "@babel/plugin-transform-dotall-regex" "^7.12.1"
-    "@babel/plugin-transform-duplicate-keys" "^7.12.1"
-    "@babel/plugin-transform-exponentiation-operator" "^7.12.1"
-    "@babel/plugin-transform-for-of" "^7.12.1"
-    "@babel/plugin-transform-function-name" "^7.12.1"
-    "@babel/plugin-transform-literals" "^7.12.1"
-    "@babel/plugin-transform-member-expression-literals" "^7.12.1"
-    "@babel/plugin-transform-modules-amd" "^7.12.1"
-    "@babel/plugin-transform-modules-commonjs" "^7.12.1"
-    "@babel/plugin-transform-modules-systemjs" "^7.12.1"
-    "@babel/plugin-transform-modules-umd" "^7.12.1"
-    "@babel/plugin-transform-named-capturing-groups-regex" "^7.12.1"
-    "@babel/plugin-transform-new-target" "^7.12.1"
-    "@babel/plugin-transform-object-super" "^7.12.1"
-    "@babel/plugin-transform-parameters" "^7.12.1"
-    "@babel/plugin-transform-property-literals" "^7.12.1"
-    "@babel/plugin-transform-regenerator" "^7.12.1"
-    "@babel/plugin-transform-reserved-words" "^7.12.1"
-    "@babel/plugin-transform-shorthand-properties" "^7.12.1"
-    "@babel/plugin-transform-spread" "^7.12.1"
-    "@babel/plugin-transform-sticky-regex" "^7.12.1"
-    "@babel/plugin-transform-template-literals" "^7.12.1"
-    "@babel/plugin-transform-typeof-symbol" "^7.12.1"
-    "@babel/plugin-transform-unicode-escapes" "^7.12.1"
-    "@babel/plugin-transform-unicode-regex" "^7.12.1"
-    "@babel/preset-modules" "^0.1.3"
-    "@babel/types" "^7.12.1"
-    core-js-compat "^3.6.2"
-    semver "^5.5.0"
-
-"@babel/preset-env@^7.12.1", "@babel/preset-env@^7.8.4":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.14.2.tgz#e80612965da73579c84ad2f963c2359c71524ed5"
-  integrity sha512-7dD7lVT8GMrE73v4lvDEb85cgcQhdES91BSD7jS/xjC6QY8PnRhux35ac+GCpbiRhp8crexBvZZqnaL6VrY8TQ==
-  dependencies:
-    "@babel/compat-data" "^7.14.0"
-    "@babel/helper-compilation-targets" "^7.13.16"
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-validator-option" "^7.12.17"
-    "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining" "^7.13.12"
-    "@babel/plugin-proposal-async-generator-functions" "^7.14.2"
-    "@babel/plugin-proposal-class-properties" "^7.13.0"
-    "@babel/plugin-proposal-class-static-block" "^7.13.11"
-    "@babel/plugin-proposal-dynamic-import" "^7.14.2"
-    "@babel/plugin-proposal-export-namespace-from" "^7.14.2"
-    "@babel/plugin-proposal-json-strings" "^7.14.2"
-    "@babel/plugin-proposal-logical-assignment-operators" "^7.14.2"
-    "@babel/plugin-proposal-nullish-coalescing-operator" "^7.14.2"
-    "@babel/plugin-proposal-numeric-separator" "^7.14.2"
-    "@babel/plugin-proposal-object-rest-spread" "^7.14.2"
-    "@babel/plugin-proposal-optional-catch-binding" "^7.14.2"
-    "@babel/plugin-proposal-optional-chaining" "^7.14.2"
-    "@babel/plugin-proposal-private-methods" "^7.13.0"
-    "@babel/plugin-proposal-private-property-in-object" "^7.14.0"
-    "@babel/plugin-proposal-unicode-property-regex" "^7.12.13"
+    "@babel/compat-data" "^7.22.5"
+    "@babel/helper-compilation-targets" "^7.22.5"
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-validator-option" "^7.22.5"
+    "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression" "^7.22.5"
+    "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining" "^7.22.5"
+    "@babel/plugin-proposal-private-property-in-object" "7.21.0-placeholder-for-preset-env.2"
     "@babel/plugin-syntax-async-generators" "^7.8.4"
     "@babel/plugin-syntax-class-properties" "^7.12.13"
-    "@babel/plugin-syntax-class-static-block" "^7.12.13"
+    "@babel/plugin-syntax-class-static-block" "^7.14.5"
     "@babel/plugin-syntax-dynamic-import" "^7.8.3"
     "@babel/plugin-syntax-export-namespace-from" "^7.8.3"
+    "@babel/plugin-syntax-import-assertions" "^7.22.5"
+    "@babel/plugin-syntax-import-attributes" "^7.22.5"
+    "@babel/plugin-syntax-import-meta" "^7.10.4"
     "@babel/plugin-syntax-json-strings" "^7.8.3"
     "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4"
     "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3"
@@ -1709,52 +1202,69 @@
     "@babel/plugin-syntax-object-rest-spread" "^7.8.3"
     "@babel/plugin-syntax-optional-catch-binding" "^7.8.3"
     "@babel/plugin-syntax-optional-chaining" "^7.8.3"
-    "@babel/plugin-syntax-private-property-in-object" "^7.14.0"
-    "@babel/plugin-syntax-top-level-await" "^7.12.13"
-    "@babel/plugin-transform-arrow-functions" "^7.13.0"
-    "@babel/plugin-transform-async-to-generator" "^7.13.0"
-    "@babel/plugin-transform-block-scoped-functions" "^7.12.13"
-    "@babel/plugin-transform-block-scoping" "^7.14.2"
-    "@babel/plugin-transform-classes" "^7.14.2"
-    "@babel/plugin-transform-computed-properties" "^7.13.0"
-    "@babel/plugin-transform-destructuring" "^7.13.17"
-    "@babel/plugin-transform-dotall-regex" "^7.12.13"
-    "@babel/plugin-transform-duplicate-keys" "^7.12.13"
-    "@babel/plugin-transform-exponentiation-operator" "^7.12.13"
-    "@babel/plugin-transform-for-of" "^7.13.0"
-    "@babel/plugin-transform-function-name" "^7.12.13"
-    "@babel/plugin-transform-literals" "^7.12.13"
-    "@babel/plugin-transform-member-expression-literals" "^7.12.13"
-    "@babel/plugin-transform-modules-amd" "^7.14.2"
-    "@babel/plugin-transform-modules-commonjs" "^7.14.0"
-    "@babel/plugin-transform-modules-systemjs" "^7.13.8"
-    "@babel/plugin-transform-modules-umd" "^7.14.0"
-    "@babel/plugin-transform-named-capturing-groups-regex" "^7.12.13"
-    "@babel/plugin-transform-new-target" "^7.12.13"
-    "@babel/plugin-transform-object-super" "^7.12.13"
-    "@babel/plugin-transform-parameters" "^7.14.2"
-    "@babel/plugin-transform-property-literals" "^7.12.13"
-    "@babel/plugin-transform-regenerator" "^7.13.15"
-    "@babel/plugin-transform-reserved-words" "^7.12.13"
-    "@babel/plugin-transform-shorthand-properties" "^7.12.13"
-    "@babel/plugin-transform-spread" "^7.13.0"
-    "@babel/plugin-transform-sticky-regex" "^7.12.13"
-    "@babel/plugin-transform-template-literals" "^7.13.0"
-    "@babel/plugin-transform-typeof-symbol" "^7.12.13"
-    "@babel/plugin-transform-unicode-escapes" "^7.12.13"
-    "@babel/plugin-transform-unicode-regex" "^7.12.13"
-    "@babel/preset-modules" "^0.1.4"
-    "@babel/types" "^7.14.2"
-    babel-plugin-polyfill-corejs2 "^0.2.0"
-    babel-plugin-polyfill-corejs3 "^0.2.0"
-    babel-plugin-polyfill-regenerator "^0.2.0"
-    core-js-compat "^3.9.0"
+    "@babel/plugin-syntax-private-property-in-object" "^7.14.5"
+    "@babel/plugin-syntax-top-level-await" "^7.14.5"
+    "@babel/plugin-syntax-unicode-sets-regex" "^7.18.6"
+    "@babel/plugin-transform-arrow-functions" "^7.22.5"
+    "@babel/plugin-transform-async-generator-functions" "^7.22.5"
+    "@babel/plugin-transform-async-to-generator" "^7.22.5"
+    "@babel/plugin-transform-block-scoped-functions" "^7.22.5"
+    "@babel/plugin-transform-block-scoping" "^7.22.5"
+    "@babel/plugin-transform-class-properties" "^7.22.5"
+    "@babel/plugin-transform-class-static-block" "^7.22.5"
+    "@babel/plugin-transform-classes" "^7.22.5"
+    "@babel/plugin-transform-computed-properties" "^7.22.5"
+    "@babel/plugin-transform-destructuring" "^7.22.5"
+    "@babel/plugin-transform-dotall-regex" "^7.22.5"
+    "@babel/plugin-transform-duplicate-keys" "^7.22.5"
+    "@babel/plugin-transform-dynamic-import" "^7.22.5"
+    "@babel/plugin-transform-exponentiation-operator" "^7.22.5"
+    "@babel/plugin-transform-export-namespace-from" "^7.22.5"
+    "@babel/plugin-transform-for-of" "^7.22.5"
+    "@babel/plugin-transform-function-name" "^7.22.5"
+    "@babel/plugin-transform-json-strings" "^7.22.5"
+    "@babel/plugin-transform-literals" "^7.22.5"
+    "@babel/plugin-transform-logical-assignment-operators" "^7.22.5"
+    "@babel/plugin-transform-member-expression-literals" "^7.22.5"
+    "@babel/plugin-transform-modules-amd" "^7.22.5"
+    "@babel/plugin-transform-modules-commonjs" "^7.22.5"
+    "@babel/plugin-transform-modules-systemjs" "^7.22.5"
+    "@babel/plugin-transform-modules-umd" "^7.22.5"
+    "@babel/plugin-transform-named-capturing-groups-regex" "^7.22.5"
+    "@babel/plugin-transform-new-target" "^7.22.5"
+    "@babel/plugin-transform-nullish-coalescing-operator" "^7.22.5"
+    "@babel/plugin-transform-numeric-separator" "^7.22.5"
+    "@babel/plugin-transform-object-rest-spread" "^7.22.5"
+    "@babel/plugin-transform-object-super" "^7.22.5"
+    "@babel/plugin-transform-optional-catch-binding" "^7.22.5"
+    "@babel/plugin-transform-optional-chaining" "^7.22.5"
+    "@babel/plugin-transform-parameters" "^7.22.5"
+    "@babel/plugin-transform-private-methods" "^7.22.5"
+    "@babel/plugin-transform-private-property-in-object" "^7.22.5"
+    "@babel/plugin-transform-property-literals" "^7.22.5"
+    "@babel/plugin-transform-regenerator" "^7.22.5"
+    "@babel/plugin-transform-reserved-words" "^7.22.5"
+    "@babel/plugin-transform-shorthand-properties" "^7.22.5"
+    "@babel/plugin-transform-spread" "^7.22.5"
+    "@babel/plugin-transform-sticky-regex" "^7.22.5"
+    "@babel/plugin-transform-template-literals" "^7.22.5"
+    "@babel/plugin-transform-typeof-symbol" "^7.22.5"
+    "@babel/plugin-transform-unicode-escapes" "^7.22.5"
+    "@babel/plugin-transform-unicode-property-regex" "^7.22.5"
+    "@babel/plugin-transform-unicode-regex" "^7.22.5"
+    "@babel/plugin-transform-unicode-sets-regex" "^7.22.5"
+    "@babel/preset-modules" "^0.1.5"
+    "@babel/types" "^7.22.5"
+    babel-plugin-polyfill-corejs2 "^0.4.3"
+    babel-plugin-polyfill-corejs3 "^0.8.1"
+    babel-plugin-polyfill-regenerator "^0.5.0"
+    core-js-compat "^3.30.2"
     semver "^6.3.0"
 
-"@babel/preset-modules@^0.1.3", "@babel/preset-modules@^0.1.4":
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/@babel/preset-modules/-/preset-modules-0.1.4.tgz#362f2b68c662842970fdb5e254ffc8fc1c2e415e"
-  integrity sha512-J36NhwnfdzpmH41M1DrnkkgAqhZaqr/NBdPfQ677mLzlaXo+oDiv1deyCDtgAhz8p328otdob0Du7+xgHGZbKg==
+"@babel/preset-modules@^0.1.5":
+  version "0.1.5"
+  resolved "https://registry.yarnpkg.com/@babel/preset-modules/-/preset-modules-0.1.5.tgz#ef939d6e7f268827e1841638dc6ff95515e115d9"
+  integrity sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==
   dependencies:
     "@babel/helper-plugin-utils" "^7.0.0"
     "@babel/plugin-proposal-unicode-property-regex" "^7.4.4"
@@ -1762,129 +1272,64 @@
     "@babel/types" "^7.4.4"
     esutils "^2.0.2"
 
-"@babel/preset-react@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.12.1.tgz#7f022b13f55b6dd82f00f16d1c599ae62985358c"
-  integrity sha512-euCExymHCi0qB9u5fKw7rvlw7AZSjw/NaB9h7EkdTt5+yHRrXdiRTh7fkG3uBPpJg82CqLfp1LHLqWGSCrab+g==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/plugin-transform-react-display-name" "^7.12.1"
-    "@babel/plugin-transform-react-jsx" "^7.12.1"
-    "@babel/plugin-transform-react-jsx-development" "^7.12.1"
-    "@babel/plugin-transform-react-jsx-self" "^7.12.1"
-    "@babel/plugin-transform-react-jsx-source" "^7.12.1"
-    "@babel/plugin-transform-react-pure-annotations" "^7.12.1"
-
-"@babel/preset-react@^7.12.5":
-  version "7.13.13"
-  resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.13.13.tgz#fa6895a96c50763fe693f9148568458d5a839761"
-  integrity sha512-gx+tDLIE06sRjKJkVtpZ/t3mzCDOnPG+ggHZG9lffUbX8+wC739x20YQc9V35Do6ZAxaUc/HhVHIiOzz5MvDmA==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.13.0"
-    "@babel/helper-validator-option" "^7.12.17"
-    "@babel/plugin-transform-react-display-name" "^7.12.13"
-    "@babel/plugin-transform-react-jsx" "^7.13.12"
-    "@babel/plugin-transform-react-jsx-development" "^7.12.17"
-    "@babel/plugin-transform-react-pure-annotations" "^7.12.1"
-
-"@babel/preset-typescript@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.12.1.tgz#86480b483bb97f75036e8864fe404cc782cc311b"
-  integrity sha512-hNK/DhmoJPsksdHuI/RVrcEws7GN5eamhi28JkO52MqIxU8Z0QpmiSOQxZHWOHV7I3P4UjHV97ay4TcamMA6Kw==
-  dependencies:
-    "@babel/helper-plugin-utils" "^7.10.4"
-    "@babel/plugin-transform-typescript" "^7.12.1"
+"@babel/preset-react@^7.12.5", "@babel/preset-react@^7.16.0":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.22.5.tgz#c4d6058fbf80bccad02dd8c313a9aaa67e3c3dd6"
+  integrity sha512-M+Is3WikOpEJHgR385HbuCITPTaPRaNkibTEa9oiofmJvIsrceb4yp9RL9Kb+TE8LznmeyZqpP+Lopwcx59xPQ==
+  dependencies:
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-validator-option" "^7.22.5"
+    "@babel/plugin-transform-react-display-name" "^7.22.5"
+    "@babel/plugin-transform-react-jsx" "^7.22.5"
+    "@babel/plugin-transform-react-jsx-development" "^7.22.5"
+    "@babel/plugin-transform-react-pure-annotations" "^7.22.5"
+
+"@babel/preset-typescript@^7.16.0":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.22.5.tgz#16367d8b01d640e9a507577ed4ee54e0101e51c8"
+  integrity sha512-YbPaal9LxztSGhmndR46FmAbkJ/1fAsw293tSU+I5E5h+cnJ3d4GTwyUgGYmOXJYdGA+uNePle4qbaRzj2NISQ==
+  dependencies:
+    "@babel/helper-plugin-utils" "^7.22.5"
+    "@babel/helper-validator-option" "^7.22.5"
+    "@babel/plugin-syntax-jsx" "^7.22.5"
+    "@babel/plugin-transform-modules-commonjs" "^7.22.5"
+    "@babel/plugin-transform-typescript" "^7.22.5"
+
+"@babel/regjsgen@^0.8.0":
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310"
+  integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==
 
 "@babel/runtime-corejs3@^7.10.2":
   version "7.14.0"
   resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.14.0.tgz#6bf5fbc0b961f8e3202888cb2cd0fb7a0a9a3f66"
-  integrity sha512-0R0HTZWHLk6G8jIk0FtoX+AatCtKnswS98VhXwGImFc759PJRp4Tru0PQYZofyijTFUr+gT8Mu7sgXVJLQ0ceg==
-  dependencies:
-    core-js-pure "^3.0.0"
-    regenerator-runtime "^0.13.4"
-
-"@babel/runtime@7.12.1":
-  version "7.12.1"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.12.1.tgz#b4116a6b6711d010b2dad3b7b6e43bf1b9954740"
-  integrity sha512-J5AIf3vPj3UwXaAzb5j1xM4WAQDX3EMgemF8rjCP3SoW09LfRKAXQKt6CoVYl230P6iWdRcBbnLDDdnqWxZSCA==
-  dependencies:
-    regenerator-runtime "^0.13.4"
-
-"@babel/runtime@7.13.10":
-  version "7.13.10"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.10.tgz#47d42a57b6095f4468da440388fdbad8bebf0d7d"
-  integrity sha512-4QPkjJq6Ns3V/RgpEahRk+AGfL0eO6RHHtTWoNNr5mO49G6B5+X6d6THgWEAvTrznU5xYpbAlVKRYcsCgh/Akw==
-  dependencies:
-    regenerator-runtime "^0.13.4"
-
-"@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.10.1", "@babel/runtime@^7.10.2", "@babel/runtime@^7.10.4", "@babel/runtime@^7.11.1", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.14.0", "@babel/runtime@^7.5.5", "@babel/runtime@^7.7.2", "@babel/runtime@^7.7.6", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
-  version "7.14.0"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.0.tgz#46794bc20b612c5f75e62dd071e24dfd95f1cbe6"
-  integrity sha512-JELkvo/DlpNdJ7dlyw/eY7E0suy5i5GQH+Vlxaq1nsNJ+H7f4Vtv3jMeCEgRhZZQFXTjldYfQgv2qmM6M1v5wA==
-  dependencies:
-    regenerator-runtime "^0.13.4"
-
-"@babel/runtime@^7.12.13":
-  version "7.20.1"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.1.tgz#1148bb33ab252b165a06698fde7576092a78b4a9"
-  integrity sha512-mrzLkl6U9YLF8qpqI7TB82PESyEGjm/0Ly91jG575eVxMMlb8fYfOXFZIJ8XfLrJZQbm7dlKry2bJmXBUEkdFg==
-  dependencies:
-    regenerator-runtime "^0.13.10"
-
-"@babel/runtime@^7.13.10", "@babel/runtime@^7.20.1", "@babel/runtime@^7.8.7":
-  version "7.20.13"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.13.tgz#7055ab8a7cff2b8f6058bf6ae45ff84ad2aded4b"
-  integrity sha512-gt3PKXs0DBoL9xCvOIIZ2NEqAGZqHjAnmVbfQtB620V0uReIQutpel14KcneZuer7UioY8ALKZ7iocavvzTNFA==
-  dependencies:
-    regenerator-runtime "^0.13.11"
-
-"@babel/runtime@^7.16.7", "@babel/runtime@^7.18.0", "@babel/runtime@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.7.tgz#fcb41a5a70550e04a7b708037c7c32f7f356d8fd"
-  integrity sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ==
-  dependencies:
-    regenerator-runtime "^0.13.11"
-
-"@babel/runtime@^7.18.3", "@babel/runtime@^7.20.0":
-  version "7.20.0"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.0.tgz#824a9ef325ffde6f78056059db3168c08785e24a"
-  integrity sha512-NDYdls71fTXoU8TZHfbBWg7DiZfNzClcKui/+kyi6ppD2L1qnWW3VV6CjtaBXSUGGhiTWJ6ereOIkUvenif66Q==
-  dependencies:
-    regenerator-runtime "^0.13.10"
-
-"@babel/runtime@^7.3.1":
-  version "7.15.3"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.15.3.tgz#2e1c2880ca118e5b2f9988322bd8a7656a32502b"
-  integrity sha512-OvwMLqNXkCXSz1kSm58sEsNuhqOx/fKpnUnKnFB5v8uDda5bLNEHNgKPvhDN6IU0LDcnHQ90LlJ0Q6jnyBSIBA==
+  integrity sha512-0R0HTZWHLk6G8jIk0FtoX+AatCtKnswS98VhXwGImFc759PJRp4Tru0PQYZofyijTFUr+gT8Mu7sgXVJLQ0ceg==
   dependencies:
+    core-js-pure "^3.0.0"
     regenerator-runtime "^0.13.4"
 
-"@babel/template@^7.10.4", "@babel/template@^7.12.13", "@babel/template@^7.3.3":
-  version "7.12.13"
-  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327"
-  integrity sha512-/7xxiGA57xMo/P2GVvdEumr8ONhFOhfgq2ihK3h1e6THqzTAkHbkXgB0xI9yeTfIUoH3+oAeHhqm/I43OTbbjA==
+"@babel/runtime@7.13.10":
+  version "7.13.10"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.10.tgz#47d42a57b6095f4468da440388fdbad8bebf0d7d"
+  integrity sha512-4QPkjJq6Ns3V/RgpEahRk+AGfL0eO6RHHtTWoNNr5mO49G6B5+X6d6THgWEAvTrznU5xYpbAlVKRYcsCgh/Akw==
   dependencies:
-    "@babel/code-frame" "^7.12.13"
-    "@babel/parser" "^7.12.13"
-    "@babel/types" "^7.12.13"
+    regenerator-runtime "^0.13.4"
 
-"@babel/template@^7.14.5", "@babel/template@^7.4.4":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/template/-/template-7.14.5.tgz#a9bc9d8b33354ff6e55a9c60d1109200a68974f4"
-  integrity sha512-6Z3Po85sfxRGachLULUhOmvAaOo7xCvqGQtxINai2mEGPFm6pQ4z5QInFnUrRpfoSV60BnjyF5F3c+15fxFV1g==
+"@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.10.1", "@babel/runtime@^7.10.2", "@babel/runtime@^7.10.4", "@babel/runtime@^7.11.1", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.13", "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.14.0", "@babel/runtime@^7.16.3", "@babel/runtime@^7.16.7", "@babel/runtime@^7.18.0", "@babel/runtime@^7.18.3", "@babel/runtime@^7.20.0", "@babel/runtime@^7.20.1", "@babel/runtime@^7.20.7", "@babel/runtime@^7.3.1", "@babel/runtime@^7.5.5", "@babel/runtime@^7.7.2", "@babel/runtime@^7.7.6", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.5.tgz#8564dd588182ce0047d55d7a75e93921107b57ec"
+  integrity sha512-ecjvYlnAaZ/KVneE/OdKYBYfgXV3Ptu6zQWmgEF7vwKhQnvVS6bjMD2XYgj+SNvQ1GfK/pjgokfPkC/2CO8CuA==
   dependencies:
-    "@babel/code-frame" "^7.14.5"
-    "@babel/parser" "^7.14.5"
-    "@babel/types" "^7.14.5"
+    regenerator-runtime "^0.13.11"
 
-"@babel/template@^7.18.10", "@babel/template@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.20.7.tgz#a15090c2839a83b02aa996c0b4994005841fd5a8"
-  integrity sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==
+"@babel/template@^7.22.5", "@babel/template@^7.3.3", "@babel/template@^7.4.4":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.5.tgz#0c8c4d944509875849bd0344ff0050756eefc6ec"
+  integrity sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==
   dependencies:
-    "@babel/code-frame" "^7.18.6"
-    "@babel/parser" "^7.20.7"
-    "@babel/types" "^7.20.7"
+    "@babel/code-frame" "^7.22.5"
+    "@babel/parser" "^7.22.5"
+    "@babel/types" "^7.22.5"
 
 "@babel/traverse@7.12.13":
   version "7.12.13"
@@ -1901,48 +1346,19 @@
     globals "^11.1.0"
     lodash "^4.17.19"
 
-"@babel/traverse@^7.1.0", "@babel/traverse@^7.12.1", "@babel/traverse@^7.13.0", "@babel/traverse@^7.13.15", "@babel/traverse@^7.14.0", "@babel/traverse@^7.14.2", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.0":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.2.tgz#9201a8d912723a831c2679c7ebbf2fe1416d765b"
-  integrity sha512-TsdRgvBFHMyHOOzcP9S6QU0QQtjxlRpEYOy3mcCO5RgmC305ki42aSAmfZEMSSYBla2oZ9BMqYlncBaKmD/7iA==
-  dependencies:
-    "@babel/code-frame" "^7.12.13"
-    "@babel/generator" "^7.14.2"
-    "@babel/helper-function-name" "^7.14.2"
-    "@babel/helper-split-export-declaration" "^7.12.13"
-    "@babel/parser" "^7.14.2"
-    "@babel/types" "^7.14.2"
-    debug "^4.1.0"
-    globals "^11.1.0"
-
-"@babel/traverse@^7.1.6":
-  version "7.14.7"
-  resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.14.7.tgz#64007c9774cfdc3abd23b0780bc18a3ce3631753"
-  integrity sha512-9vDr5NzHu27wgwejuKL7kIOm4bwEtaPQ4Z6cpCmjSuaRqpH/7xc4qcGEscwMqlkwgcXl6MvqoAjZkQ24uSdIZQ==
-  dependencies:
-    "@babel/code-frame" "^7.14.5"
-    "@babel/generator" "^7.14.5"
-    "@babel/helper-function-name" "^7.14.5"
-    "@babel/helper-hoist-variables" "^7.14.5"
-    "@babel/helper-split-export-declaration" "^7.14.5"
-    "@babel/parser" "^7.14.7"
-    "@babel/types" "^7.14.5"
-    debug "^4.1.0"
-    globals "^11.1.0"
-
-"@babel/traverse@^7.20.10", "@babel/traverse@^7.20.12", "@babel/traverse@^7.20.7":
-  version "7.20.12"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.20.12.tgz#7f0f787b3a67ca4475adef1f56cb94f6abd4a4b5"
-  integrity sha512-MsIbFN0u+raeja38qboyF8TIT7K0BFzz/Yd/77ta4MsUsmP2RAnidIlwq7d5HFQrH/OZJecGV6B71C4zAgpoSQ==
-  dependencies:
-    "@babel/code-frame" "^7.18.6"
-    "@babel/generator" "^7.20.7"
-    "@babel/helper-environment-visitor" "^7.18.9"
-    "@babel/helper-function-name" "^7.19.0"
-    "@babel/helper-hoist-variables" "^7.18.6"
-    "@babel/helper-split-export-declaration" "^7.18.6"
-    "@babel/parser" "^7.20.7"
-    "@babel/types" "^7.20.7"
+"@babel/traverse@^7.1.6", "@babel/traverse@^7.14.0", "@babel/traverse@^7.22.5", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.2":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.5.tgz#44bd276690db6f4940fdb84e1cb4abd2f729ccd1"
+  integrity sha512-7DuIjPgERaNo6r+PZwItpjCZEa5vyw4eJGufeLxrPdBXBoLcCJCIasvK6pK/9DVNrLZTLFhUGqaC6X/PA007TQ==
+  dependencies:
+    "@babel/code-frame" "^7.22.5"
+    "@babel/generator" "^7.22.5"
+    "@babel/helper-environment-visitor" "^7.22.5"
+    "@babel/helper-function-name" "^7.22.5"
+    "@babel/helper-hoist-variables" "^7.22.5"
+    "@babel/helper-split-export-declaration" "^7.22.5"
+    "@babel/parser" "^7.22.5"
+    "@babel/types" "^7.22.5"
     debug "^4.1.0"
     globals "^11.1.0"
 
@@ -1955,29 +1371,13 @@
     lodash "^4.17.19"
     to-fast-properties "^2.0.0"
 
-"@babel/types@^7.0.0", "@babel/types@^7.0.0-beta.49", "@babel/types@^7.12.1", "@babel/types@^7.12.13", "@babel/types@^7.12.6", "@babel/types@^7.13.0", "@babel/types@^7.13.12", "@babel/types@^7.13.16", "@babel/types@^7.14.0", "@babel/types@^7.14.2", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0":
-  version "7.14.2"
-  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.14.2.tgz#4208ae003107ef8a057ea8333e56eb64d2f6a2c3"
-  integrity sha512-SdjAG/3DikRHpUOjxZgnkbR11xUlyDMUFJdvnIgZEE16mqmY0BINMmc4//JMJglEmn6i7sq6p+mGrFWyZ98EEw==
-  dependencies:
-    "@babel/helper-validator-identifier" "^7.14.0"
-    to-fast-properties "^2.0.0"
-
-"@babel/types@^7.1.6", "@babel/types@^7.14.5":
-  version "7.14.5"
-  resolved "https://registry.npmjs.org/@babel/types/-/types-7.14.5.tgz#3bb997ba829a2104cedb20689c4a5b8121d383ff"
-  integrity sha512-M/NzBpEL95I5Hh4dwhin5JlE7EzO5PHMAuzjxss3tiOBD46KfQvVedN/3jEPZvdRvtsK2222XfdHogNIttFgcg==
-  dependencies:
-    "@babel/helper-validator-identifier" "^7.14.5"
-    to-fast-properties "^2.0.0"
-
-"@babel/types@^7.18.6", "@babel/types@^7.19.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.7":
-  version "7.20.7"
-  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.20.7.tgz#54ec75e252318423fc07fb644dc6a58a64c09b7f"
-  integrity sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg==
+"@babel/types@^7.0.0", "@babel/types@^7.1.6", "@babel/types@^7.12.13", "@babel/types@^7.12.6", "@babel/types@^7.20.7", "@babel/types@^7.22.5", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe"
+  integrity sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA==
   dependencies:
-    "@babel/helper-string-parser" "^7.19.4"
-    "@babel/helper-validator-identifier" "^7.19.1"
+    "@babel/helper-string-parser" "^7.22.5"
+    "@babel/helper-validator-identifier" "^7.22.5"
     to-fast-properties "^2.0.0"
 
 "@bcoe/v8-coverage@^0.2.3":
@@ -1985,84 +1385,142 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@cnakazawa/watch@^1.0.3":
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
-  integrity sha512-v9kIhKwjeZThiWrLmj0y17CWoyddASLj9O2yvbZkbvw/N3rWOYy9zkV66ursAoVr0mV15bL8g0c4QZUE6cdDoQ==
+"@craco/craco@^7.1.0":
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/@craco/craco/-/craco-7.1.0.tgz#12bd394c7f0334e214302e4d35a1768f68042fbb"
+  integrity sha512-oRAcPIKYrfPXp9rSzlsDNeOaVtDiKhoyqSXUoqiK24jCkHr4T8m/a2f74yXIzCbIheoUWDOIfWZyRgFgT+cpqA==
   dependencies:
-    exec-sh "^0.3.2"
-    minimist "^1.2.0"
+    autoprefixer "^10.4.12"
+    cosmiconfig "^7.0.1"
+    cosmiconfig-typescript-loader "^1.0.0"
+    cross-spawn "^7.0.3"
+    lodash "^4.17.21"
+    semver "^7.3.7"
+    webpack-merge "^5.8.0"
 
-"@craco/craco@^6.1.1":
-  version "6.1.2"
-  resolved "https://registry.yarnpkg.com/@craco/craco/-/craco-6.1.2.tgz#30e45288e4609ac6b8cf828085b34acebdc60e69"
-  integrity sha512-GlQZn+g+yNlaDvIL5m6mcCoBGyFDwO4kkNx3fNFf98wuldkdWyBFoQbtOFOIb4gvkTh4VntOOxtJEoZfKs7XXw==
+"@cspotcode/source-map-support@^0.8.0":
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
+  integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==
   dependencies:
-    cross-spawn "^7.0.0"
-    lodash "^4.17.15"
-    semver "^7.3.2"
-    webpack-merge "^4.2.2"
+    "@jridgewell/trace-mapping" "0.3.9"
 
-"@csstools/convert-colors@^1.4.0":
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/@csstools/convert-colors/-/convert-colors-1.4.0.tgz#ad495dc41b12e75d588c6db8b9834f08fa131eb7"
-  integrity sha512-5a6wqoJV/xEdbRNKVo6I4hO3VjyDq//8q2f9I6PBAvMesJHFauXDorcNCsr9RzvsZnaWi5NYCcfyqP1QeFHFbw==
+"@csstools/normalize.css@*":
+  version "12.0.0"
+  resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-12.0.0.tgz#a9583a75c3f150667771f30b60d9f059473e62c4"
+  integrity sha512-M0qqxAcwCsIVfpFQSlGN5XjXWu8l5JDZN+fPt1LeW5SZexQTgnaEvgXAY+CeygRw0EeppWHi12JxESWiWrB0Sg==
 
-"@csstools/normalize.css@^10.1.0":
-  version "10.1.0"
-  resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-10.1.0.tgz#f0950bba18819512d42f7197e56c518aa491cf18"
-  integrity sha512-ij4wRiunFfaJxjB0BdrYHIH8FxBJpOwNPhhAcunlmPdXudL1WQV1qoP9un6JsEBAgQH+7UXyyjh0g7jTxXK6tg==
+"@csstools/postcss-cascade-layers@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-cascade-layers/-/postcss-cascade-layers-1.1.1.tgz#8a997edf97d34071dd2e37ea6022447dd9e795ad"
+  integrity sha512-+KdYrpKC5TgomQr2DlZF4lDEpHcoxnj5IGddYYfBWJAKfj1JtuHUIqMa+E1pJJ+z3kvDViWMqyqPlG4Ja7amQA==
+  dependencies:
+    "@csstools/selector-specificity" "^2.0.2"
+    postcss-selector-parser "^6.0.10"
+
+"@csstools/postcss-color-function@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-color-function/-/postcss-color-function-1.1.1.tgz#2bd36ab34f82d0497cfacdc9b18d34b5e6f64b6b"
+  integrity sha512-Bc0f62WmHdtRDjf5f3e2STwRAl89N2CLb+9iAwzrv4L2hncrbDwnQD9PCq0gtAt7pOI2leIV08HIBUd4jxD8cw==
+  dependencies:
+    "@csstools/postcss-progressive-custom-properties" "^1.1.0"
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-font-format-keywords@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-font-format-keywords/-/postcss-font-format-keywords-1.0.1.tgz#677b34e9e88ae997a67283311657973150e8b16a"
+  integrity sha512-ZgrlzuUAjXIOc2JueK0X5sZDjCtgimVp/O5CEqTcs5ShWBa6smhWYbS0x5cVc/+rycTDbjjzoP0KTDnUneZGOg==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-hwb-function@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-hwb-function/-/postcss-hwb-function-1.0.2.tgz#ab54a9fce0ac102c754854769962f2422ae8aa8b"
+  integrity sha512-YHdEru4o3Rsbjmu6vHy4UKOXZD+Rn2zmkAmLRfPet6+Jz4Ojw8cbWxe1n42VaXQhD3CQUXXTooIy8OkVbUcL+w==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-ic-unit@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-ic-unit/-/postcss-ic-unit-1.0.1.tgz#28237d812a124d1a16a5acc5c3832b040b303e58"
+  integrity sha512-Ot1rcwRAaRHNKC9tAqoqNZhjdYBzKk1POgWfhN4uCOE47ebGcLRqXjKkApVDpjifL6u2/55ekkpnFcp+s/OZUw==
+  dependencies:
+    "@csstools/postcss-progressive-custom-properties" "^1.1.0"
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-is-pseudo-class@^2.0.7":
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-is-pseudo-class/-/postcss-is-pseudo-class-2.0.7.tgz#846ae6c0d5a1eaa878fce352c544f9c295509cd1"
+  integrity sha512-7JPeVVZHd+jxYdULl87lvjgvWldYu+Bc62s9vD/ED6/QTGjy0jy0US/f6BG53sVMTBJ1lzKZFpYmofBN9eaRiA==
+  dependencies:
+    "@csstools/selector-specificity" "^2.0.0"
+    postcss-selector-parser "^6.0.10"
+
+"@csstools/postcss-nested-calc@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-nested-calc/-/postcss-nested-calc-1.0.0.tgz#d7e9d1d0d3d15cf5ac891b16028af2a1044d0c26"
+  integrity sha512-JCsQsw1wjYwv1bJmgjKSoZNvf7R6+wuHDAbi5f/7MbFhl2d/+v+TvBTU4BJH3G1X1H87dHl0mh6TfYogbT/dJQ==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-normalize-display-values@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-normalize-display-values/-/postcss-normalize-display-values-1.0.1.tgz#15da54a36e867b3ac5163ee12c1d7f82d4d612c3"
+  integrity sha512-jcOanIbv55OFKQ3sYeFD/T0Ti7AMXc9nM1hZWu8m/2722gOTxFg7xYu4RDLJLeZmPUVQlGzo4jhzvTUq3x4ZUw==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-oklab-function@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-oklab-function/-/postcss-oklab-function-1.1.1.tgz#88cee0fbc8d6df27079ebd2fa016ee261eecf844"
+  integrity sha512-nJpJgsdA3dA9y5pgyb/UfEzE7W5Ka7u0CX0/HIMVBNWzWemdcTH3XwANECU6anWv/ao4vVNLTMxhiPNZsTK6iA==
+  dependencies:
+    "@csstools/postcss-progressive-custom-properties" "^1.1.0"
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-progressive-custom-properties@^1.1.0", "@csstools/postcss-progressive-custom-properties@^1.3.0":
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-progressive-custom-properties/-/postcss-progressive-custom-properties-1.3.0.tgz#542292558384361776b45c85226b9a3a34f276fa"
+  integrity sha512-ASA9W1aIy5ygskZYuWams4BzafD12ULvSypmaLJT2jvQ8G0M3I8PRQhC0h7mG0Z3LI05+agZjqSR9+K9yaQQjA==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-stepped-value-functions@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-stepped-value-functions/-/postcss-stepped-value-functions-1.0.1.tgz#f8772c3681cc2befed695e2b0b1d68e22f08c4f4"
+  integrity sha512-dz0LNoo3ijpTOQqEJLY8nyaapl6umbmDcgj4AD0lgVQ572b2eqA1iGZYTTWhrcrHztWDDRAX2DGYyw2VBjvCvQ==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-text-decoration-shorthand@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-text-decoration-shorthand/-/postcss-text-decoration-shorthand-1.0.0.tgz#ea96cfbc87d921eca914d3ad29340d9bcc4c953f"
+  integrity sha512-c1XwKJ2eMIWrzQenN0XbcfzckOLLJiczqy+YvfGmzoVXd7pT9FfObiSEfzs84bpE/VqfpEuAZ9tCRbZkZxxbdw==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-trigonometric-functions@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-trigonometric-functions/-/postcss-trigonometric-functions-1.0.2.tgz#94d3e4774c36d35dcdc88ce091336cb770d32756"
+  integrity sha512-woKaLO///4bb+zZC2s80l+7cm07M7268MsyG3M0ActXXEFi6SuhvriQYcb58iiKGbjwwIU7n45iRLEHypB47Og==
+  dependencies:
+    postcss-value-parser "^4.2.0"
+
+"@csstools/postcss-unset-value@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@csstools/postcss-unset-value/-/postcss-unset-value-1.0.2.tgz#c99bb70e2cdc7312948d1eb41df2412330b81f77"
+  integrity sha512-c8J4roPBILnelAsdLr4XOAR/GsTm0GJi4XpcfvoWk3U6KiTCqiFYc63KhRMQQX35jYMp4Ao8Ij9+IZRgMfJp1g==
+
+"@csstools/selector-specificity@^2.0.0", "@csstools/selector-specificity@^2.0.2":
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/@csstools/selector-specificity/-/selector-specificity-2.2.0.tgz#2cbcf822bf3764c9658c4d2e568bd0c0cb748016"
+  integrity sha512-+OJ9konv95ClSTOJCmMZqpd5+YGsB2S+x6w3E1oaM8UuR5j8nTNHYSz8c9BEPGDOCMQYIEEGlVPj/VY64iTbGw==
 
 "@ctrl/tinycolor@^3.3.1", "@ctrl/tinycolor@^3.4.0":
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/@ctrl/tinycolor/-/tinycolor-3.4.0.tgz#c3c5ae543c897caa9c2a68630bed355be5f9990f"
   integrity sha512-JZButFdZ1+/xAfpguQHoabIXkcqRRKpMrWKBkpEZZyxfY9C1DpADFB8PEqGSTeFr135SaTRfKqGKx5xSCLI7ZQ==
 
-"@data-ui/shared@^0.0.84":
-  version "0.0.84"
-  resolved "https://registry.yarnpkg.com/@data-ui/shared/-/shared-0.0.84.tgz#42bd025d677f9be2beada3e1a84a53d33ac0eb10"
-  integrity sha512-MsDLsFzBHFEREr/eF2/RX1o/cXioEg+VQTsM8gViW5ywGQ7Xo5+EqUOaBSrwqKAkvp3e8PaEZVkchPC54IBhrA==
-  dependencies:
-    "@data-ui/theme" "^0.0.84"
-    "@vx/event" "^0.0.165"
-    "@vx/group" "^0.0.165"
-    "@vx/shape" "^0.0.168"
-    "@vx/tooltip" "0.0.165"
-    d3-array "^1.2.1"
-    prop-types "^15.5.10"
-
-"@data-ui/theme@^0.0.84":
-  version "0.0.84"
-  resolved "https://registry.yarnpkg.com/@data-ui/theme/-/theme-0.0.84.tgz#b75c23d7f38c582adbb8d2159d0d703159f8e3b2"
-  integrity sha512-jIoHftC/5c/LVJYF4VSBjjVjrjc0yj4mLkGe8p0eVO7qUYKVvlWx7PrpM7ucyefvuAaKIwlr+Nh2xPGPdADjaA==
-
-"@data-ui/xy-chart@^0.0.84":
-  version "0.0.84"
-  resolved "https://registry.yarnpkg.com/@data-ui/xy-chart/-/xy-chart-0.0.84.tgz#28bc41592d81d7be1da65ddd50fe3b79ea65649a"
-  integrity sha512-4mRWEGfeQJ2kFXmQ81k1gDPx2zdkty6lt0+srui4zleSyhnBv1dmm9J03dq+qwr7+bpzjfq77nINV5HXWb31Bg==
-  dependencies:
-    "@data-ui/shared" "^0.0.84"
-    "@data-ui/theme" "^0.0.84"
-    "@vx/axis" "^0.0.175"
-    "@vx/curve" "^0.0.165"
-    "@vx/event" "^0.0.165"
-    "@vx/glyph" "^0.0.165"
-    "@vx/gradient" "^0.0.165"
-    "@vx/grid" "^0.0.180"
-    "@vx/group" "^0.0.165"
-    "@vx/pattern" "^0.0.165"
-    "@vx/point" "^0.0.165"
-    "@vx/responsive" "^0.0.192"
-    "@vx/scale" "^0.0.165"
-    "@vx/shape" "^0.0.165"
-    "@vx/stats" "^0.0.165"
-    "@vx/text" "^0.0.192"
-    "@vx/threshold" "0.0.170"
-    "@vx/tooltip" "^0.0.165"
-    "@vx/voronoi" "^0.0.165"
-    d3-array "^1.2.0"
-    prop-types "^15.5.10"
-
 "@emotion/babel-plugin@^11.10.5":
   version "11.10.5"
   resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz#65fa6e1790ddc9e23cc22658a4c5dea423c55c3c"
@@ -2214,21 +1672,38 @@
     ts-node "^9"
     tslib "^2"
 
-"@eslint/eslintrc@^0.4.1":
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.1.tgz#442763b88cecbe3ee0ec7ca6d6dd6168550cbf14"
-  integrity sha512-5v7TDE9plVhvxQeWLXDTvFvJBdH6pEsdnl2g/dAptmuFEPedQ4Erq5rsDsX+mvAM610IhNaO2W5V1dOOnDKxkQ==
+"@eslint-community/eslint-utils@^4.2.0":
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59"
+  integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==
+  dependencies:
+    eslint-visitor-keys "^3.3.0"
+
+"@eslint-community/regexpp@^4.4.0":
+  version "4.5.1"
+  resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.5.1.tgz#cdd35dce4fa1a89a4fd42b1599eb35b3af408884"
+  integrity sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ==
+
+"@eslint/eslintrc@^2.0.3":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.0.3.tgz#4910db5505f4d503f27774bf356e3704818a0331"
+  integrity sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==
   dependencies:
     ajv "^6.12.4"
-    debug "^4.1.1"
-    espree "^7.3.0"
-    globals "^12.1.0"
-    ignore "^4.0.6"
+    debug "^4.3.2"
+    espree "^9.5.2"
+    globals "^13.19.0"
+    ignore "^5.2.0"
     import-fresh "^3.2.1"
-    js-yaml "^3.13.1"
-    minimatch "^3.0.4"
+    js-yaml "^4.1.0"
+    minimatch "^3.1.2"
     strip-json-comments "^3.1.1"
 
+"@eslint/js@8.43.0":
+  version "8.43.0"
+  resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.43.0.tgz#559ca3d9ddbd6bf907ad524320a0d14b85586af0"
+  integrity sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg==
+
 "@graphql-codegen/add@^2.0.2":
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/add/-/add-2.0.2.tgz#4acbb95be9ebb859a3cebfe7132fdf49ffe06dd8"
@@ -2317,18 +1792,7 @@
     parse-filepath "^1.0.2"
     tslib "~2.3.0"
 
-"@graphql-codegen/plugin-helpers@^1.18.2":
-  version "1.18.7"
-  resolved "https://registry.yarnpkg.com/@graphql-codegen/plugin-helpers/-/plugin-helpers-1.18.7.tgz#465af3e5b02de89e49ddc76ad2546b880fe240f2"
-  integrity sha512-8ICOrXlsvyL1dpVz8C9b7H31d4DJpDd75WfjMn6Xjqz81Ah8xDn1Bi+7YXRCCILCBmvI94k6fi8qpsIVhFBBjQ==
-  dependencies:
-    "@graphql-tools/utils" "^7.9.1"
-    common-tags "1.8.0"
-    import-from "3.0.0"
-    lodash "~4.17.0"
-    tslib "~2.2.0"
-
-"@graphql-codegen/plugin-helpers@^1.18.8":
+"@graphql-codegen/plugin-helpers@^1.18.2", "@graphql-codegen/plugin-helpers@^1.18.8":
   version "1.18.8"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/plugin-helpers/-/plugin-helpers-1.18.8.tgz#39aac745b9e22e28c781cc07cf74836896a3a905"
   integrity sha512-mb4I9j9lMGqvGggYuZ0CV+Hme08nar68xkpPbAVotg/ZBmlhZIok/HqW2BcMQi7Rj+Il5HQMeQ1wQ1M7sv/TlQ==
@@ -2653,43 +2117,11 @@
   resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.2.0.tgz#5f3d96ec6b2354ad6d8a28bf216a1d97b5426861"
   integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==
 
-"@hapi/address@2.x.x":
-  version "2.1.4"
-  resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5"
-  integrity sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ==
-
-"@hapi/bourne@1.x.x":
-  version "1.3.2"
-  resolved "https://registry.yarnpkg.com/@hapi/bourne/-/bourne-1.3.2.tgz#0a7095adea067243ce3283e1b56b8a8f453b242a"
-  integrity sha512-1dVNHT76Uu5N3eJNTYcvxee+jzX4Z9lfciqRRHCU27ihbUcYi+iSc2iml5Ke1LXe1SyJCLA0+14Jh4tXJgOppA==
-
-"@hapi/hoek@8.x.x", "@hapi/hoek@^8.3.0":
-  version "8.5.1"
-  resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-8.5.1.tgz#fde96064ca446dec8c55a8c2f130957b070c6e06"
-  integrity sha512-yN7kbciD87WzLGc5539Tn0sApjyiGHAJgKvG9W8C7O+6c7qmoQMfVs0W4bX17eqz6C78QJqqFrtgdK5EWf6Qow==
-
 "@hapi/hoek@^9.0.0":
   version "9.2.0"
   resolved "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.2.0.tgz#f3933a44e365864f4dad5db94158106d511e8131"
   integrity sha512-sqKVVVOe5ivCaXDWivIJYVSaEgdQK9ul7a4Kity5Iw7u9+wBAPbX1RMSnLLmp7O4Vzj0WOWwMAJsTL00xwaNug==
 
-"@hapi/joi@^15.1.0":
-  version "15.1.1"
-  resolved "https://registry.yarnpkg.com/@hapi/joi/-/joi-15.1.1.tgz#c675b8a71296f02833f8d6d243b34c57b8ce19d7"
-  integrity sha512-entf8ZMOK8sc+8YfeOlM8pCfg3b5+WZIKBfUaaJT8UsjAAPjartzxIYm3TIbjvA4u+u++KbcXD38k682nVHDAQ==
-  dependencies:
-    "@hapi/address" "2.x.x"
-    "@hapi/bourne" "1.x.x"
-    "@hapi/hoek" "8.x.x"
-    "@hapi/topo" "3.x.x"
-
-"@hapi/topo@3.x.x":
-  version "3.1.6"
-  resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-3.1.6.tgz#68d935fa3eae7fdd5ab0d7f953f3205d8b2bfc29"
-  integrity sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ==
-  dependencies:
-    "@hapi/hoek" "^8.3.0"
-
 "@hapi/topo@^5.0.0":
   version "5.1.0"
   resolved "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012"
@@ -2697,6 +2129,25 @@
   dependencies:
     "@hapi/hoek" "^9.0.0"
 
+"@humanwhocodes/config-array@^0.11.10":
+  version "0.11.10"
+  resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.10.tgz#5a3ffe32cc9306365fb3fd572596cd602d5e12d2"
+  integrity sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==
+  dependencies:
+    "@humanwhocodes/object-schema" "^1.2.1"
+    debug "^4.1.1"
+    minimatch "^3.0.5"
+
+"@humanwhocodes/module-importer@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c"
+  integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==
+
+"@humanwhocodes/object-schema@^1.2.1":
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
+  integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
+
 "@iarna/toml@^2.2.5":
   version "2.2.5"
   resolved "https://registry.yarnpkg.com/@iarna/toml/-/toml-2.2.5.tgz#b32366c89b43c6f8cefbdefac778b9c828e3ba8c"
@@ -2723,167 +2174,194 @@
   resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98"
   integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==
 
-"@jest/console@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/console/-/console-26.6.2.tgz#4e04bc464014358b03ab4937805ee36a0aeb98f2"
-  integrity sha512-IY1R2i2aLsLr7Id3S6p2BA82GNWryt4oSvEXLAKc+L2zdi89dSkE8xC1C+0kpATG4JhBJREnQOH7/zmccM2B0g==
+"@jest/console@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/console/-/console-27.5.1.tgz#260fe7239602fe5130a94f1aa386eff54b014bba"
+  integrity sha512-kZ/tNpS3NXn0mlXXXPNuDZnb4c0oZ20r4K5eemM2k30ZC3G0T02nXUvyhf5YdbXWHPEJLc9qGLxEZ216MdL+Zg==
   dependencies:
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
     chalk "^4.0.0"
-    jest-message-util "^26.6.2"
-    jest-util "^26.6.2"
+    jest-message-util "^27.5.1"
+    jest-util "^27.5.1"
     slash "^3.0.0"
 
-"@jest/core@^26.6.0", "@jest/core@^26.6.3":
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/@jest/core/-/core-26.6.3.tgz#7639fcb3833d748a4656ada54bde193051e45fad"
-  integrity sha512-xvV1kKbhfUqFVuZ8Cyo+JPpipAHHAV3kcDBftiduK8EICXmTFddryy3P7NfZt8Pv37rA9nEJBKCCkglCPt/Xjw==
+"@jest/console@^28.1.3":
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/@jest/console/-/console-28.1.3.tgz#2030606ec03a18c31803b8a36382762e447655df"
+  integrity sha512-QPAkP5EwKdK/bxIr6C1I4Vs0rm2nHiANzj/Z5X2JQkrZo6IqvC4ldZ9K95tF0HdidhA8Bo6egxSzUFPYKcEXLw==
   dependencies:
-    "@jest/console" "^26.6.2"
-    "@jest/reporters" "^26.6.2"
-    "@jest/test-result" "^26.6.2"
-    "@jest/transform" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^28.1.3"
+    "@types/node" "*"
+    chalk "^4.0.0"
+    jest-message-util "^28.1.3"
+    jest-util "^28.1.3"
+    slash "^3.0.0"
+
+"@jest/core@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/core/-/core-27.5.1.tgz#267ac5f704e09dc52de2922cbf3af9edcd64b626"
+  integrity sha512-AK6/UTrvQD0Cd24NSqmIA6rKsu0tKIxfiCducZvqxYdmMisOYAsdItspT+fQDQYARPf8XgjAFZi0ogW2agH5nQ==
+  dependencies:
+    "@jest/console" "^27.5.1"
+    "@jest/reporters" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/transform" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
     ansi-escapes "^4.2.1"
     chalk "^4.0.0"
+    emittery "^0.8.1"
     exit "^0.1.2"
-    graceful-fs "^4.2.4"
-    jest-changed-files "^26.6.2"
-    jest-config "^26.6.3"
-    jest-haste-map "^26.6.2"
-    jest-message-util "^26.6.2"
-    jest-regex-util "^26.0.0"
-    jest-resolve "^26.6.2"
-    jest-resolve-dependencies "^26.6.3"
-    jest-runner "^26.6.3"
-    jest-runtime "^26.6.3"
-    jest-snapshot "^26.6.2"
-    jest-util "^26.6.2"
-    jest-validate "^26.6.2"
-    jest-watcher "^26.6.2"
-    micromatch "^4.0.2"
-    p-each-series "^2.1.0"
+    graceful-fs "^4.2.9"
+    jest-changed-files "^27.5.1"
+    jest-config "^27.5.1"
+    jest-haste-map "^27.5.1"
+    jest-message-util "^27.5.1"
+    jest-regex-util "^27.5.1"
+    jest-resolve "^27.5.1"
+    jest-resolve-dependencies "^27.5.1"
+    jest-runner "^27.5.1"
+    jest-runtime "^27.5.1"
+    jest-snapshot "^27.5.1"
+    jest-util "^27.5.1"
+    jest-validate "^27.5.1"
+    jest-watcher "^27.5.1"
+    micromatch "^4.0.4"
     rimraf "^3.0.0"
     slash "^3.0.0"
     strip-ansi "^6.0.0"
 
-"@jest/environment@^26.6.0", "@jest/environment@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-26.6.2.tgz#ba364cc72e221e79cc8f0a99555bf5d7577cf92c"
-  integrity sha512-nFy+fHl28zUrRsCeMB61VDThV1pVTtlEokBRgqPrcT1JNq4yRNIyTHfyht6PqtUvY9IsuLGTrbG8kPXjSZIZwA==
+"@jest/environment@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-27.5.1.tgz#d7425820511fe7158abbecc010140c3fd3be9c74"
+  integrity sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA==
   dependencies:
-    "@jest/fake-timers" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/fake-timers" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
-    jest-mock "^26.6.2"
+    jest-mock "^27.5.1"
 
-"@jest/fake-timers@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-26.6.2.tgz#459c329bcf70cee4af4d7e3f3e67848123535aad"
-  integrity sha512-14Uleatt7jdzefLPYM3KLcnUl1ZNikaKq34enpb5XG9i81JpppDb5muZvonvKyrl7ftEHkKS5L5/eB/kxJ+bvA==
+"@jest/fake-timers@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-27.5.1.tgz#76979745ce0579c8a94a4678af7a748eda8ada74"
+  integrity sha512-/aPowoolwa07k7/oM3aASneNeBGCmGQsc3ugN4u6s4C/+s5M64MFo/+djTdiwcbQlRfFElGuDXWzaWj6QgKObQ==
   dependencies:
-    "@jest/types" "^26.6.2"
-    "@sinonjs/fake-timers" "^6.0.1"
+    "@jest/types" "^27.5.1"
+    "@sinonjs/fake-timers" "^8.0.1"
     "@types/node" "*"
-    jest-message-util "^26.6.2"
-    jest-mock "^26.6.2"
-    jest-util "^26.6.2"
+    jest-message-util "^27.5.1"
+    jest-mock "^27.5.1"
+    jest-util "^27.5.1"
 
-"@jest/globals@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-26.6.2.tgz#5b613b78a1aa2655ae908eba638cc96a20df720a"
-  integrity sha512-85Ltnm7HlB/KesBUuALwQ68YTU72w9H2xW9FjZ1eL1U3lhtefjjl5c2MiUbpXt/i6LaPRvoOFJ22yCBSfQ0JIA==
+"@jest/globals@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-27.5.1.tgz#7ac06ce57ab966566c7963431cef458434601b2b"
+  integrity sha512-ZEJNB41OBQQgGzgyInAv0UUfDDj3upmHydjieSxFvTRuZElrx7tXg/uVQ5hYVEwiXs3+aMsAeEc9X7xiSKCm4Q==
   dependencies:
-    "@jest/environment" "^26.6.2"
-    "@jest/types" "^26.6.2"
-    expect "^26.6.2"
+    "@jest/environment" "^27.5.1"
+    "@jest/types" "^27.5.1"
+    expect "^27.5.1"
 
-"@jest/reporters@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-26.6.2.tgz#1f518b99637a5f18307bd3ecf9275f6882a667f6"
-  integrity sha512-h2bW53APG4HvkOnVMo8q3QXa6pcaNt1HkwVsOPMBV6LD/q9oSpxNSYZQYkAnjdMjrJ86UuYeLo+aEZClV6opnw==
+"@jest/reporters@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-27.5.1.tgz#ceda7be96170b03c923c37987b64015812ffec04"
+  integrity sha512-cPXh9hWIlVJMQkVk84aIvXuBB4uQQmFqZiacloFuGiP3ah1sbCxCosidXFDfqG8+6fO1oR2dTJTlsOy4VFmUfw==
   dependencies:
     "@bcoe/v8-coverage" "^0.2.3"
-    "@jest/console" "^26.6.2"
-    "@jest/test-result" "^26.6.2"
-    "@jest/transform" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/console" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/transform" "^27.5.1"
+    "@jest/types" "^27.5.1"
+    "@types/node" "*"
     chalk "^4.0.0"
     collect-v8-coverage "^1.0.0"
     exit "^0.1.2"
     glob "^7.1.2"
-    graceful-fs "^4.2.4"
+    graceful-fs "^4.2.9"
     istanbul-lib-coverage "^3.0.0"
-    istanbul-lib-instrument "^4.0.3"
+    istanbul-lib-instrument "^5.1.0"
     istanbul-lib-report "^3.0.0"
     istanbul-lib-source-maps "^4.0.0"
-    istanbul-reports "^3.0.2"
-    jest-haste-map "^26.6.2"
-    jest-resolve "^26.6.2"
-    jest-util "^26.6.2"
-    jest-worker "^26.6.2"
+    istanbul-reports "^3.1.3"
+    jest-haste-map "^27.5.1"
+    jest-resolve "^27.5.1"
+    jest-util "^27.5.1"
+    jest-worker "^27.5.1"
     slash "^3.0.0"
     source-map "^0.6.0"
     string-length "^4.0.1"
     terminal-link "^2.0.0"
-    v8-to-istanbul "^7.0.0"
-  optionalDependencies:
-    node-notifier "^8.0.0"
+    v8-to-istanbul "^8.1.0"
 
-"@jest/source-map@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-26.6.2.tgz#29af5e1e2e324cafccc936f218309f54ab69d535"
-  integrity sha512-YwYcCwAnNmOVsZ8mr3GfnzdXDAl4LaenZP5z+G0c8bzC9/dugL8zRmxZzdoTl4IaS3CryS1uWnROLPFmb6lVvA==
+"@jest/schemas@^28.1.3":
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-28.1.3.tgz#ad8b86a66f11f33619e3d7e1dcddd7f2d40ff905"
+  integrity sha512-/l/VWsdt/aBXgjshLWOFyFt3IVdYypu5y2Wn2rOO1un6nkqIn8SLXzgIMYXFyYsRWDyF5EthmKJMIdJvk08grg==
+  dependencies:
+    "@sinclair/typebox" "^0.24.1"
+
+"@jest/source-map@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-27.5.1.tgz#6608391e465add4205eae073b55e7f279e04e8cf"
+  integrity sha512-y9NIHUYF3PJRlHk98NdC/N1gl88BL08aQQgu4k4ZopQkCw9t9cV8mtl3TV8b/YCB8XaVTFrmUTAJvjsntDireg==
   dependencies:
     callsites "^3.0.0"
-    graceful-fs "^4.2.4"
+    graceful-fs "^4.2.9"
     source-map "^0.6.0"
 
-"@jest/test-result@^26.6.0", "@jest/test-result@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-26.6.2.tgz#55da58b62df134576cc95476efa5f7949e3f5f18"
-  integrity sha512-5O7H5c/7YlojphYNrK02LlDIV2GNPYisKwHm2QTKjNZeEzezCbwYs9swJySv2UfPMyZ0VdsmMv7jIlD/IKYQpQ==
+"@jest/test-result@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-27.5.1.tgz#56a6585fa80f7cdab72b8c5fc2e871d03832f5bb"
+  integrity sha512-EW35l2RYFUcUQxFJz5Cv5MTOxlJIQs4I7gxzi2zVU7PJhOwfYq1MdC5nhSmYjX1gmMmLPvB3sIaC+BkcHRBfag==
   dependencies:
-    "@jest/console" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/console" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/istanbul-lib-coverage" "^2.0.0"
     collect-v8-coverage "^1.0.0"
 
-"@jest/test-sequencer@^26.6.3":
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-26.6.3.tgz#98e8a45100863886d074205e8ffdc5a7eb582b17"
-  integrity sha512-YHlVIjP5nfEyjlrSr8t/YdNfU/1XEt7c5b4OxcXCjyRhjzLYu/rO69/WHPuYcbCWkz8kAeZVZp2N2+IOLLEPGw==
+"@jest/test-result@^28.1.3":
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-28.1.3.tgz#5eae945fd9f4b8fcfce74d239e6f725b6bf076c5"
+  integrity sha512-kZAkxnSE+FqE8YjW8gNuoVkkC9I7S1qmenl8sGcDOLropASP+BkcGKwhXoyqQuGOGeYY0y/ixjrd/iERpEXHNg==
   dependencies:
-    "@jest/test-result" "^26.6.2"
-    graceful-fs "^4.2.4"
-    jest-haste-map "^26.6.2"
-    jest-runner "^26.6.3"
-    jest-runtime "^26.6.3"
+    "@jest/console" "^28.1.3"
+    "@jest/types" "^28.1.3"
+    "@types/istanbul-lib-coverage" "^2.0.0"
+    collect-v8-coverage "^1.0.0"
 
-"@jest/transform@^26.6.2":
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-26.6.2.tgz#5ac57c5fa1ad17b2aae83e73e45813894dcf2e4b"
-  integrity sha512-E9JjhUgNzvuQ+vVAL21vlyfy12gP0GhazGgJC4h6qUt1jSdUXGWJ1wfu/X7Sd8etSgxV4ovT1pb9v5D6QW4XgA==
+"@jest/test-sequencer@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-27.5.1.tgz#4057e0e9cea4439e544c6353c6affe58d095745b"
+  integrity sha512-LCheJF7WB2+9JuCS7VB/EmGIdQuhtqjRNI9A43idHv3E4KltCTsPsLxvdaubFHSYwY/fNjMWjl6vNRhDiN7vpQ==
+  dependencies:
+    "@jest/test-result" "^27.5.1"
+    graceful-fs "^4.2.9"
+    jest-haste-map "^27.5.1"
+    jest-runtime "^27.5.1"
+
+"@jest/transform@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-27.5.1.tgz#6c3501dcc00c4c08915f292a600ece5ecfe1f409"
+  integrity sha512-ipON6WtYgl/1329g5AIJVbUuEh0wZVbdpGwC99Jw4LwuoBNS95MVphU6zOeD9pDkon+LLbFL7lOQRapbB8SCHw==
   dependencies:
     "@babel/core" "^7.1.0"
-    "@jest/types" "^26.6.2"
-    babel-plugin-istanbul "^6.0.0"
+    "@jest/types" "^27.5.1"
+    babel-plugin-istanbul "^6.1.1"
     chalk "^4.0.0"
     convert-source-map "^1.4.0"
     fast-json-stable-stringify "^2.0.0"
-    graceful-fs "^4.2.4"
-    jest-haste-map "^26.6.2"
-    jest-regex-util "^26.0.0"
-    jest-util "^26.6.2"
-    micromatch "^4.0.2"
-    pirates "^4.0.1"
+    graceful-fs "^4.2.9"
+    jest-haste-map "^27.5.1"
+    jest-regex-util "^27.5.1"
+    jest-util "^27.5.1"
+    micromatch "^4.0.4"
+    pirates "^4.0.4"
     slash "^3.0.0"
     source-map "^0.6.1"
     write-file-atomic "^3.0.0"
 
-"@jest/types@^26.6.0", "@jest/types@^26.6.2":
+"@jest/types@^26.6.2":
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.6.2.tgz#bef5a532030e1d88a2f5a6d933f84e97226ed48e"
   integrity sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==
@@ -2894,18 +2372,33 @@
     "@types/yargs" "^15.0.0"
     chalk "^4.0.0"
 
-"@jridgewell/gen-mapping@^0.1.0":
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996"
-  integrity sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==
+"@jest/types@^27.5.1":
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/@jest/types/-/types-27.5.1.tgz#3c79ec4a8ba61c170bf937bcf9e98a9df175ec80"
+  integrity sha512-Cx46iJ9QpwQTjIdq5VJu2QTMMs3QlEjI0x1QbBP5W1+nMzyc2XmimiRR/CbX9TO0cPTeUlxWMOu8mslYsJ8DEw==
   dependencies:
-    "@jridgewell/set-array" "^1.0.0"
-    "@jridgewell/sourcemap-codec" "^1.4.10"
+    "@types/istanbul-lib-coverage" "^2.0.0"
+    "@types/istanbul-reports" "^3.0.0"
+    "@types/node" "*"
+    "@types/yargs" "^16.0.0"
+    chalk "^4.0.0"
 
-"@jridgewell/gen-mapping@^0.3.2":
-  version "0.3.2"
-  resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9"
-  integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==
+"@jest/types@^28.1.3":
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/@jest/types/-/types-28.1.3.tgz#b05de80996ff12512bc5ceb1d208285a7d11748b"
+  integrity sha512-RyjiyMUZrKz/c+zlMFO1pm70DcIlST8AeWTkoUdZevew44wcNZQHsEVOiCVtgVnlFFD82FPaXycys58cf2muVQ==
+  dependencies:
+    "@jest/schemas" "^28.1.3"
+    "@types/istanbul-lib-coverage" "^2.0.0"
+    "@types/istanbul-reports" "^3.0.0"
+    "@types/node" "*"
+    "@types/yargs" "^17.0.8"
+    chalk "^4.0.0"
+
+"@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2":
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098"
+  integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==
   dependencies:
     "@jridgewell/set-array" "^1.0.1"
     "@jridgewell/sourcemap-codec" "^1.4.10"
@@ -2916,24 +2409,50 @@
   resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78"
   integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==
 
-"@jridgewell/set-array@^1.0.0", "@jridgewell/set-array@^1.0.1":
+"@jridgewell/resolve-uri@^3.0.3":
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721"
+  integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==
+
+"@jridgewell/set-array@^1.0.1":
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72"
   integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==
 
+"@jridgewell/source-map@^0.3.3":
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.3.tgz#8108265659d4c33e72ffe14e33d6cc5eb59f2fda"
+  integrity sha512-b+fsZXeLYi9fEULmfBrhxn4IrPlINf8fiNarzTof004v3lFdntdwa9PF7vFJqm3mg7s+ScJMxXaE3Acp1irZcg==
+  dependencies:
+    "@jridgewell/gen-mapping" "^0.3.0"
+    "@jridgewell/trace-mapping" "^0.3.9"
+
 "@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10":
   version "1.4.14"
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
   integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
 
-"@jridgewell/trace-mapping@^0.3.9":
-  version "0.3.17"
-  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985"
-  integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==
+"@jridgewell/trace-mapping@0.3.9":
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
+  integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==
+  dependencies:
+    "@jridgewell/resolve-uri" "^3.0.3"
+    "@jridgewell/sourcemap-codec" "^1.4.10"
+
+"@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9":
+  version "0.3.18"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6"
+  integrity sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==
   dependencies:
     "@jridgewell/resolve-uri" "3.1.0"
     "@jridgewell/sourcemap-codec" "1.4.14"
 
+"@leichtgewicht/ip-codec@^2.0.1":
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b"
+  integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==
+
 "@linaria/core@3.0.0-beta.13":
   version "3.0.0-beta.13"
   resolved "https://registry.yarnpkg.com/@linaria/core/-/core-3.0.0-beta.13.tgz#049c5be5faa67e341e413a0f6b641d5d78d91056"
@@ -3081,35 +2600,34 @@
     prop-types "^15.8.1"
     react-is "^18.2.0"
 
-"@nodelib/fs.scandir@2.1.4":
-  version "2.1.4"
-  resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz#d4b3549a5db5de2683e0c1071ab4f140904bbf69"
-  integrity sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA==
+"@nicolo-ribaudo/eslint-scope-5-internals@5.1.1-v1":
+  version "5.1.1-v1"
+  resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz#dbf733a965ca47b1973177dc0bb6c889edcfb129"
+  integrity sha512-54/JRvkLIzzDWshCWfuhadfrfZVPiElY8Fcgmg1HroEly/EDSszzhBAsarCux+D/kOslTRquNzuyGSmUSTTHGg==
+  dependencies:
+    eslint-scope "5.1.1"
+
+"@nodelib/fs.scandir@2.1.5":
+  version "2.1.5"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
+  integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==
   dependencies:
-    "@nodelib/fs.stat" "2.0.4"
+    "@nodelib/fs.stat" "2.0.5"
     run-parallel "^1.1.9"
 
-"@nodelib/fs.stat@2.0.4", "@nodelib/fs.stat@^2.0.2":
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.4.tgz#a3f2dd61bab43b8db8fa108a121cfffe4c676655"
-  integrity sha512-IYlHJA0clt2+Vg7bccq+TzRdJvv19c2INqBSsoOLp1je7xjtr7J26+WXR72MCdvU9q1qTzIWDfhMf+DRvQJK4Q==
+"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b"
+  integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==
 
-"@nodelib/fs.walk@^1.2.3":
-  version "1.2.6"
-  resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.6.tgz#cce9396b30aa5afe9e3756608f5831adcb53d063"
-  integrity sha512-8Broas6vTtW4GIXTAHDoE32hnN2M5ykgCpWGbuXHQ15vEMqr23pB76e/GZcYsZCHALv50ktd24qhEyKr6wBtow==
+"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8":
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a"
+  integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==
   dependencies:
-    "@nodelib/fs.scandir" "2.1.4"
+    "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
-"@npmcli/move-file@^1.0.1":
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-1.1.2.tgz#1a82c3e372f7cae9253eb66d72543d6b8685c674"
-  integrity sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==
-  dependencies:
-    mkdirp "^1.0.4"
-    rimraf "^3.0.2"
-
 "@ocavue/svgmoji-cjs@^0.1.1":
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/@ocavue/svgmoji-cjs/-/svgmoji-cjs-0.1.1.tgz#7240e6860c907187e4cd95b39858507384e4025f"
@@ -3117,16 +2635,19 @@
   dependencies:
     svgmoji "^3.2.0"
 
-"@pmmmwh/react-refresh-webpack-plugin@0.4.3":
-  version "0.4.3"
-  resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.4.3.tgz#1eec460596d200c0236bf195b078a5d1df89b766"
-  integrity sha512-br5Qwvh8D2OQqSXpd1g/xqXKnK0r+Jz6qVKBbWmpUcrbGOxUrf39V5oZ1876084CGn18uMdR5uvPqBv9UqtBjQ==
+"@pmmmwh/react-refresh-webpack-plugin@^0.5.3":
+  version "0.5.10"
+  resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.10.tgz#2eba163b8e7dbabb4ce3609ab5e32ab63dda3ef8"
+  integrity sha512-j0Ya0hCFZPd4x40qLzbhGsh9TMtdb+CJQiso+WxLOPNasohq9cc5SNUcwsZaRH6++Xh91Xkm/xHCkuIiIu0LUA==
   dependencies:
-    ansi-html "^0.0.7"
+    ansi-html-community "^0.0.8"
+    common-path-prefix "^3.0.0"
+    core-js-pure "^3.23.3"
     error-stack-parser "^2.0.6"
-    html-entities "^1.2.1"
-    native-url "^0.2.6"
-    schema-utils "^2.6.5"
+    find-up "^5.0.0"
+    html-entities "^2.1.0"
+    loader-utils "^2.0.4"
+    schema-utils "^3.0.0"
     source-map "^0.7.3"
 
 "@popperjs/core@^2.11.6", "@popperjs/core@^2.9.2":
@@ -3199,6 +2720,45 @@
     "@react-hook/event" "^1.2.1"
     "@react-hook/throttle" "^2.2.0"
 
+"@react-spring/animated@~9.7.3":
+  version "9.7.3"
+  resolved "https://registry.yarnpkg.com/@react-spring/animated/-/animated-9.7.3.tgz#4211b1a6d48da0ff474a125e93c0f460ff816e0f"
+  integrity sha512-5CWeNJt9pNgyvuSzQH+uy2pvTg8Y4/OisoscZIR8/ZNLIOI+CatFBhGZpDGTF/OzdNFsAoGk3wiUYTwoJ0YIvw==
+  dependencies:
+    "@react-spring/shared" "~9.7.3"
+    "@react-spring/types" "~9.7.3"
+
+"@react-spring/core@~9.7.3":
+  version "9.7.3"
+  resolved "https://registry.yarnpkg.com/@react-spring/core/-/core-9.7.3.tgz#60056bcb397f2c4f371c6c9a5f882db77ae90095"
+  integrity sha512-IqFdPVf3ZOC1Cx7+M0cXf4odNLxDC+n7IN3MDcVCTIOSBfqEcBebSv+vlY5AhM0zw05PDbjKrNmBpzv/AqpjnQ==
+  dependencies:
+    "@react-spring/animated" "~9.7.3"
+    "@react-spring/shared" "~9.7.3"
+    "@react-spring/types" "~9.7.3"
+
+"@react-spring/shared@~9.7.3":
+  version "9.7.3"
+  resolved "https://registry.yarnpkg.com/@react-spring/shared/-/shared-9.7.3.tgz#4cf29797847c689912aec4e62e34c99a4d5d9e53"
+  integrity sha512-NEopD+9S5xYyQ0pGtioacLhL2luflh6HACSSDUZOwLHoxA5eku1UPuqcJqjwSD6luKjjLfiLOspxo43FUHKKSA==
+  dependencies:
+    "@react-spring/types" "~9.7.3"
+
+"@react-spring/types@~9.7.3":
+  version "9.7.3"
+  resolved "https://registry.yarnpkg.com/@react-spring/types/-/types-9.7.3.tgz#ea78fd447cbc2612c1f5d55852e3c331e8172a0b"
+  integrity sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw==
+
+"@react-spring/web@^9.7.3":
+  version "9.7.3"
+  resolved "https://registry.yarnpkg.com/@react-spring/web/-/web-9.7.3.tgz#d9f4e17fec259f1d65495a19502ada4f5b57fa3d"
+  integrity sha512-BXt6BpS9aJL/QdVqEIX9YoUy8CE6TJrU0mNCqSoxdXlIeNcEBWOfIyE6B14ENNsyQKS3wOWkiJfco0tCr/9tUg==
+  dependencies:
+    "@react-spring/animated" "~9.7.3"
+    "@react-spring/core" "~9.7.3"
+    "@react-spring/shared" "~9.7.3"
+    "@react-spring/types" "~9.7.3"
+
 "@remirror/core-constants@^2.0.0":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@remirror/core-constants/-/core-constants-2.0.0.tgz#a52f89059d93955e00810023cc76b4f7db9650bf"
@@ -4076,18 +3636,27 @@
   dependencies:
     type-fest "^2.0.0"
 
-"@rollup/plugin-node-resolve@^7.1.1":
-  version "7.1.3"
-  resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz#80de384edfbd7bfc9101164910f86078151a3eca"
-  integrity sha512-RxtSL3XmdTAE2byxekYLnx+98kEUOrPHF/KRVjLH+DEIHy6kjIw7YINQzn+NXiH/NTrQLAwYs0GWB+csWygA9Q==
+"@rollup/plugin-babel@^5.2.0":
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283"
+  integrity sha512-WFfdLWU/xVWKeRQnKmIAQULUI7Il0gZnBIH/ZFO069wYIfPu+8zrfp/KMW0atmELoRDq8FbiP3VCss9MhCut7Q==
   dependencies:
-    "@rollup/pluginutils" "^3.0.8"
-    "@types/resolve" "0.0.8"
+    "@babel/helper-module-imports" "^7.10.4"
+    "@rollup/pluginutils" "^3.1.0"
+
+"@rollup/plugin-node-resolve@^11.2.1":
+  version "11.2.1"
+  resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-11.2.1.tgz#82aa59397a29cd4e13248b106e6a4a1880362a60"
+  integrity sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg==
+  dependencies:
+    "@rollup/pluginutils" "^3.1.0"
+    "@types/resolve" "1.17.1"
     builtin-modules "^3.1.0"
+    deepmerge "^4.2.2"
     is-module "^1.0.0"
-    resolve "^1.14.2"
+    resolve "^1.19.0"
 
-"@rollup/plugin-replace@^2.3.1":
+"@rollup/plugin-replace@^2.4.1":
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-2.4.2.tgz#a2d539314fbc77c244858faa523012825068510a"
   integrity sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg==
@@ -4095,7 +3664,7 @@
     "@rollup/pluginutils" "^3.1.0"
     magic-string "^0.25.7"
 
-"@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0":
+"@rollup/pluginutils@^3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b"
   integrity sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==
@@ -4109,6 +3678,11 @@
   resolved "https://registry.yarnpkg.com/@rooks/use-mutation-observer/-/use-mutation-observer-4.11.2.tgz#a0466c4338e0a4487ea19253c86bcd427c29f4af"
   integrity sha512-vpsdrZdr6TkB1zZJcHx+fR1YC/pHs2BaqcuYiEGjBVbwY5xcC49+h0hAUtQKHth3oJqXfIX/Ng8S7s5HFHdM/A==
 
+"@rushstack/eslint-patch@^1.1.0":
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.3.2.tgz#31b9c510d8cada9683549e1dbb4284cca5001faf"
+  integrity sha512-V+MvGwaHH03hYhY+k6Ef/xKd6RYlc4q8WBx+2ANmipHJcKuktNcI/NgEsJgdSUF6Lw32njT6OnrRsKYCdgHjYw==
+
 "@samverschueren/stream-to-observable@^0.3.0":
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz#a21117b19ee9be70c379ec1877537ef2e1c63301"
@@ -4138,6 +3712,11 @@
   resolved "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df"
   integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==
 
+"@sinclair/typebox@^0.24.1":
+  version "0.24.51"
+  resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.24.51.tgz#645f33fe4e02defe26f2f5c0410e1c094eac7f5f"
+  integrity sha512-1P1OROm/rdubP5aFDSZQILU0vrLCJ4fvHt6EoqHEM+2D/G5MK3bIaymUKLit8Js9gbns5UyJnkP/TZROLw4tUA==
+
 "@sindresorhus/is@^0.14.0":
   version "0.14.0"
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
@@ -4150,13 +3729,6 @@
   dependencies:
     type-detect "4.0.8"
 
-"@sinonjs/fake-timers@^6.0.1":
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz#293674fccb3262ac782c7aadfdeca86b10c75c40"
-  integrity sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==
-  dependencies:
-    "@sinonjs/commons" "^1.7.0"
-
 "@sinonjs/fake-timers@^7.0.4", "@sinonjs/fake-timers@^7.1.0":
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-7.1.0.tgz#8f13af27d842cbf51ad4502e05562fe9391d084e"
@@ -4164,6 +3736,13 @@
   dependencies:
     "@sinonjs/commons" "^1.7.0"
 
+"@sinonjs/fake-timers@^8.0.1":
+  version "8.1.0"
+  resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-8.1.0.tgz#3fdc2b6cb58935b21bfb8d1625eb1300484316e7"
+  integrity sha512-OAPJUAtgeINhh/TAlUID4QTs53Njm7xzddaVlEs/SXwgtiD1tW22zAB/W1wdqfrpmikgaWQ9Fw6Ws+hsiRm5Vg==
+  dependencies:
+    "@sinonjs/commons" "^1.7.0"
+
 "@sinonjs/samsam@^6.0.2":
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-6.0.2.tgz#a0117d823260f282c04bff5f8704bdc2ac6910bb"
@@ -4178,13 +3757,15 @@
   resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5"
   integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==
 
-"@surma/rollup-plugin-off-main-thread@^1.1.1":
-  version "1.4.2"
-  resolved "https://registry.yarnpkg.com/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-1.4.2.tgz#e6786b6af5799f82f7ab3a82e53f6182d2b91a58"
-  integrity sha512-yBMPqmd1yEJo/280PAMkychuaALyQ9Lkb5q1ck3mjJrFuEobIfhnQ4J3mbvBoISmR3SWMWV+cGB/I0lCQee79A==
+"@surma/rollup-plugin-off-main-thread@^2.2.3":
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz#ee34985952ca21558ab0d952f00298ad2190c053"
+  integrity sha512-lR8q/9W7hZpMWweNiAKU7NQerBnzQQLvi8qnTDU/fxItPhtZVMbPV3lbCwjhIlNBe9Bbr5V+KHshvWmVSG9cxQ==
   dependencies:
-    ejs "^2.6.1"
+    ejs "^3.1.6"
+    json5 "^2.2.0"
     magic-string "^0.25.0"
+    string.prototype.matchall "^4.0.6"
 
 "@svgmoji/blob@^3.2.0":
   version "3.2.0"
@@ -4319,7 +3900,7 @@
     deepmerge "^4.2.2"
     svgo "^1.2.2"
 
-"@svgr/webpack@5.5.0":
+"@svgr/webpack@^5.5.0":
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/@svgr/webpack/-/webpack-5.5.0.tgz#aae858ee579f5fa8ce6c3166ef56c6a1b381b640"
   integrity sha512-DOBOK255wfQxguUta2INKkzPj6AIS6iafZYiYmHn6W3pHlycSRRlvWKCfLDG10fXfLWqE3DJHgRUOyJYmARa7g==
@@ -4381,23 +3962,48 @@
   resolved "https://registry.yarnpkg.com/@tommoor/remove-markdown/-/remove-markdown-0.3.2.tgz#5288ddd0e26b6b173e76ebb31c94653b0dcff45d"
   integrity sha512-awcc9hfLZqyyZHOGzAHbnjgZJpQGS1W1oZZ5GXOTTnbKVdKQ4OWYbrRWPUvXI2YAKJazrcS8rxPh67PX3rpGkQ==
 
-"@tootallnate/once@1":
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
-  integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
+"@tootallnate/once@1":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
+  integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
+
+"@trysound/sax@0.2.0":
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad"
+  integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==
+
+"@tsconfig/node10@^1.0.7":
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2"
+  integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==
+
+"@tsconfig/node12@^1.0.7":
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d"
+  integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==
+
+"@tsconfig/node14@^1.0.0":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1"
+  integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==
+
+"@tsconfig/node16@^1.0.2":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9"
+  integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==
 
 "@types/aria-query@^4.2.0":
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.1.tgz#78b5433344e2f92e8b306c06a5622c50c245bf6b"
   integrity sha512-S6oPal772qJZHoRZLFc/XoZW2gFvwXusYUmXPXkgxJLuEk2vOt7jc4Yo6z/vtI0EBkbPBVrJJ0B+prLIKiWqHg==
 
-"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7":
-  version "7.1.14"
-  resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402"
-  integrity sha512-zGZJzzBUVDo/eV6KgbE0f0ZI7dInEYvo12Rb70uNQDshC3SkRMb67ja0GgRHZgAX3Za6rhaWlvbDO8rrGyAb1g==
+"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14":
+  version "7.20.1"
+  resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.1.tgz#916ecea274b0c776fec721e333e55762d3a9614b"
+  integrity sha512-aACu/U/omhdk15O4Nfb+fHgH/z3QsfQzpnvRZhYhThms83ZnAOZz7zZAWO7mn2yyNQaA4xTO8GLK3uqFU4bYYw==
   dependencies:
-    "@babel/parser" "^7.1.0"
-    "@babel/types" "^7.0.0"
+    "@babel/parser" "^7.20.7"
+    "@babel/types" "^7.20.7"
     "@types/babel__generator" "*"
     "@types/babel__template" "*"
     "@types/babel__traverse" "*"
@@ -4424,12 +4030,20 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
-"@types/classnames@^2.2.9":
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/@types/classnames/-/classnames-2.3.1.tgz#3c2467aa0f1a93f1f021e3b9bcf938bd5dfdc0dd"
-  integrity sha512-zeOWb0JGBoVmlQoznvqXbE0tEC/HONsnoUNH19Hc96NFsTAwTXbTqb8FMYkru1F/iqp7a18Ws3nWJvtA1sHD1A==
+"@types/body-parser@*":
+  version "1.19.2"
+  resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0"
+  integrity sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==
   dependencies:
-    classnames "*"
+    "@types/connect" "*"
+    "@types/node" "*"
+
+"@types/bonjour@^3.5.9":
+  version "3.5.10"
+  resolved "https://registry.yarnpkg.com/@types/bonjour/-/bonjour-3.5.10.tgz#0f6aadfe00ea414edc86f5d106357cda9701e275"
+  integrity sha512-p7ienRMiS41Nu2/igbJxxLDWrSZ0WxM8UQgCeO9KhoVF7cOVFkrKsiDr1EsJIla8vV3oEEjGcz11jc5yimhzZw==
+  dependencies:
+    "@types/node" "*"
 
 "@types/codemirror@^5.60.2":
   version "5.60.5"
@@ -4438,34 +4052,59 @@
   dependencies:
     "@types/tern" "*"
 
-"@types/d3-color@^1":
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-1.4.1.tgz#0d9746c84dfef28807b2989eed4f37b2575e1f33"
-  integrity sha512-xkPLi+gbgUU9ED6QX4g6jqYL2KCB0/3AlM+ncMGqn49OgH0gFMY/ITGqPF8HwEiLzJaC+2L0I+gNwBgABv1Pvg==
+"@types/connect-history-api-fallback@^1.3.5":
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.0.tgz#9fd20b3974bdc2bcd4ac6567e2e0f6885cb2cf41"
+  integrity sha512-4x5FkPpLipqwthjPsF7ZRbOv3uoLUFkTA9G9v583qi4pACvq0uTELrB8OLUzPWUI4IJIyvM85vzkV1nyiI2Lig==
+  dependencies:
+    "@types/express-serve-static-core" "*"
+    "@types/node" "*"
+
+"@types/connect@*":
+  version "3.4.35"
+  resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1"
+  integrity sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==
+  dependencies:
+    "@types/node" "*"
+
+"@types/d3-array@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@types/d3-array/-/d3-array-3.0.3.tgz#87d990bf504d14ad6b16766979d04e943c046dac"
+  integrity sha512-Reoy+pKnvsksN0lQUlcH6dOGjRZ/3WRwXR//m+/8lt1BXeI4xyaUZoqULNjyXXRuh0Mj4LNpkCvhUpQlY3X5xQ==
+
+"@types/d3-color@*", "@types/d3-color@3.1.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-3.1.0.tgz#6594da178ded6c7c3842f3cc0ac84b156f12f2d4"
+  integrity sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA==
+
+"@types/d3-format@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@types/d3-format/-/d3-format-3.0.1.tgz#194f1317a499edd7e58766f96735bdc0216bb89d"
+  integrity sha512-5KY70ifCCzorkLuIkDe0Z9YTf9RR2CjBX1iaJG+rgM/cPP+sO+q9YdQ9WdhQcgPj1EQiJ2/0+yUkkziTG6Lubg==
 
 "@types/d3-hierarchy@^1.1.6":
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/@types/d3-hierarchy/-/d3-hierarchy-1.1.7.tgz#14a57b0539f8929015f8ad96490de50a16211040"
   integrity sha512-fvht6DOYKzqmXjMb/+xfgkmrWM4SD7rMA/ZbM+gGwr9ZTuIDfky95J8CARtaJo/ExeWyS0xGVdL2gqno2zrQ0Q==
 
-"@types/d3-interpolate@^1.3.1":
-  version "1.4.2"
-  resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-1.4.2.tgz#88902a205f682773a517612299a44699285eed7b"
-  integrity sha512-ylycts6llFf8yAEs1tXzx2loxxzDZHseuhPokrqKprTQSTcD3JbJI1omZP1rphsELZO3Q+of3ff0ZS7+O6yVzg==
+"@types/d3-interpolate@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-3.0.1.tgz#e7d17fa4a5830ad56fe22ce3b4fac8541a9572dc"
+  integrity sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw==
   dependencies:
-    "@types/d3-color" "^1"
+    "@types/d3-color" "*"
 
 "@types/d3-path@^1", "@types/d3-path@^1.0.8":
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/@types/d3-path/-/d3-path-1.0.9.tgz#73526b150d14cd96e701597cbf346cfd1fd4a58c"
   integrity sha512-NaIeSIBiFgSC6IGUBjZWcscUJEq7vpVu7KthHN8eieTV9d9MqkSOZLH4chq1PmcKy06PNe3axLeKmRIyxJ+PZQ==
 
-"@types/d3-scale@^2.1.1":
-  version "2.2.4"
-  resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-2.2.4.tgz#ca0d4b84d2f88fe058480f81354d14041a667b96"
-  integrity sha512-wkQXT+IfgfAnKB5rtS1qMJg3FS32r1rVFHvqtiqk8pX8o5aQR3VwX1P7ErHjzNIicTlkWsaMiUTrYB+E75HFeA==
+"@types/d3-scale@4.0.2":
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-4.0.2.tgz#41be241126af4630524ead9cb1008ab2f0f26e69"
+  integrity sha512-Yk4htunhPAwN0XGlIwArRomOjdoBFXC3+kCxK2Ubg7I9shQlVSJy/pG/Ht5ASN+gdMIalpk8TJ5xV74jFsetLA==
   dependencies:
-    "@types/d3-time" "^1"
+    "@types/d3-time" "*"
 
 "@types/d3-shape@^1.3.1":
   version "1.3.5"
@@ -4474,10 +4113,20 @@
   dependencies:
     "@types/d3-path" "^1"
 
-"@types/d3-time@^1", "@types/d3-time@^1.0.10":
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-1.1.1.tgz#6cf3a4242c3bbac00440dfb8ba7884f16bedfcbf"
-  integrity sha512-ULX7LoqXTCYtM+tLYOaeAJK7IwCT+4Gxlm2MaH0ErKLi07R5lh8NHCAyWcDkCCmx1AfRcBEV6H9QE9R25uP7jw==
+"@types/d3-time-format@2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@types/d3-time-format/-/d3-time-format-2.1.0.tgz#011e0fb7937be34a9a8f580ae1e2f2f1336a8a22"
+  integrity sha512-/myT3I7EwlukNOX2xVdMzb8FRgNzRMpsZddwst9Ld/VFe6LyJyRp0s32l/V9XoUzk+Gqu56F/oGk6507+8BxrA==
+
+"@types/d3-time@*", "@types/d3-time@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-3.0.0.tgz#e1ac0f3e9e195135361fa1a1d62f795d87e6e819"
+  integrity sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==
+
+"@types/d3-voronoi@^1.1.9":
+  version "1.1.9"
+  resolved "https://registry.yarnpkg.com/@types/d3-voronoi/-/d3-voronoi-1.1.9.tgz#7bbc210818a3a5c5e0bafb051420df206617c9e5"
+  integrity sha512-DExNQkaHd1F3dFPvGA/Aw2NGyjMln6E9QzsiqOcBgnE+VInYnFBHBBySbZQts6z6xD+5jTfKCP7M4OqMyVjdwQ==
 
 "@types/diff@^5.0.0":
   version "5.0.0"
@@ -4496,31 +4145,51 @@
   dependencies:
     "@types/trusted-types" "*"
 
-"@types/eslint@^7.2.6":
-  version "7.2.11"
-  resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-7.2.11.tgz#180b58f5bb7d7376e39d22496e2b08901aa52fd2"
-  integrity sha512-WYhv//5K8kQtsSc9F1Kn2vHzhYor6KpwPbARH7hwYe3C3ETD0EVx/3P5qQybUoaBEuUa9f/02JjBiXFWalYUmw==
+"@types/eslint-scope@^3.7.3":
+  version "3.7.4"
+  resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.4.tgz#37fc1223f0786c39627068a12e94d6e6fc61de16"
+  integrity sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==
+  dependencies:
+    "@types/eslint" "*"
+    "@types/estree" "*"
+
+"@types/eslint@*", "@types/eslint@^7.29.0 || ^8.4.1":
+  version "8.40.2"
+  resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-8.40.2.tgz#2833bc112d809677864a4b0e7d1de4f04d7dac2d"
+  integrity sha512-PRVjQ4Eh9z9pmmtaq8nTjZjQwKFk7YIHIud3lRoKRBgUQjgjRmoGxxGEPXQkF+lH7QkHJRNr5F4aBgYCW0lqpQ==
   dependencies:
     "@types/estree" "*"
     "@types/json-schema" "*"
 
-"@types/estree@*":
-  version "0.0.47"
-  resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.47.tgz#d7a51db20f0650efec24cd04994f523d93172ed4"
-  integrity sha512-c5ciR06jK8u9BstrmJyO97m+klJrrhCf9u3rLu3DEAJBirxRqSCvDQoYKmxuYwQI5SZChAWu+tq9oVlGRuzPAg==
+"@types/estree@*", "@types/estree@^1.0.0":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.1.tgz#aa22750962f3bf0e79d753d3cc067f010c95f194"
+  integrity sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==
 
 "@types/estree@0.0.39":
   version "0.0.39"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
   integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==
 
-"@types/glob@^7.1.1":
-  version "7.1.3"
-  resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.3.tgz#e6ba80f36b7daad2c685acd9266382e68985c183"
-  integrity sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==
+"@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.33":
+  version "4.17.35"
+  resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.35.tgz#c95dd4424f0d32e525d23812aa8ab8e4d3906c4f"
+  integrity sha512-wALWQwrgiB2AWTT91CB62b6Yt0sNHpznUXeZEcnPU3DRdlDIz74x8Qg1UUYKSVFi+va5vKOLYRBI1bRKiLLKIg==
   dependencies:
-    "@types/minimatch" "*"
     "@types/node" "*"
+    "@types/qs" "*"
+    "@types/range-parser" "*"
+    "@types/send" "*"
+
+"@types/express@*", "@types/express@^4.17.13":
+  version "4.17.17"
+  resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.17.tgz#01d5437f6ef9cfa8668e616e13c2f2ac9a491ae4"
+  integrity sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==
+  dependencies:
+    "@types/body-parser" "*"
+    "@types/express-serve-static-core" "^4.17.33"
+    "@types/qs" "*"
+    "@types/serve-static" "*"
 
 "@types/graceful-fs@^4.1.2":
   version "4.1.5"
@@ -4556,10 +4225,15 @@
     "@types/react" "*"
     hoist-non-react-statics "^3.3.0"
 
-"@types/html-minifier-terser@^5.0.0":
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/@types/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz#3c9ee980f1a10d6021ae6632ca3e79ca2ec4fb50"
-  integrity sha512-giAlZwstKbmvMk1OO7WXSj4OZ0keXAcl2TQq4LWHiiPH2ByaH7WeUzng+Qej8UPxxv+8lRTuouo0iaNDBuzIBA==
+"@types/html-minifier-terser@^6.0.0":
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz#4fc33a00c1d0c16987b1a20cf92d20614c55ac35"
+  integrity sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==
+
+"@types/http-errors@*":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.1.tgz#20172f9578b225f6c7da63446f56d4ce108d5a65"
+  integrity sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==
 
 "@types/http-proxy-agent@^2.0.2":
   version "2.0.2"
@@ -4568,10 +4242,10 @@
   dependencies:
     "@types/node" "*"
 
-"@types/http-proxy@^1.17.5":
-  version "1.17.7"
-  resolved "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.7.tgz#30ea85cc2c868368352a37f0d0d3581e24834c6f"
-  integrity sha512-9hdj6iXH64tHSLTY+Vt2eYOGzSogC+JQ2H7bdPWkuh7KXP5qLllWx++t+K9Wk556c3dkDdPws/SpMRi0sdCT1w==
+"@types/http-proxy@^1.17.5", "@types/http-proxy@^1.17.8":
+  version "1.17.11"
+  resolved "https://registry.yarnpkg.com/@types/http-proxy/-/http-proxy-1.17.11.tgz#0ca21949a5588d55ac2b659b69035c84bd5da293"
+  integrity sha512-HC8G7c1WmaF2ekqpnFq626xd3Zz0uvaqFmBJNRZCGEZCXkvSdJoNFn/8Ygbd9fKNQj8UzLdCETaI0UWPAjK7IA==
   dependencies:
     "@types/node" "*"
 
@@ -4612,10 +4286,10 @@
   resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.1.tgz#5544730b65a480b18ace6b6ce914e519cec2d43b"
   integrity sha512-xdOvNmXmrZqqPy3kuCQ+fz6wA0xU5pji9cd1nDrflWaAWtYLLGk5ykW0H6yg5TVyehHP1pfmuuSaZkhP+kspVA==
 
-"@types/json-schema@*", "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6":
-  version "7.0.7"
-  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad"
-  integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==
+"@types/json-schema@*", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9":
+  version "7.0.12"
+  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb"
+  integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==
 
 "@types/json-stable-stringify@^1.0.32":
   version "1.0.32"
@@ -4634,10 +4308,10 @@
   dependencies:
     "@types/node" "*"
 
-"@types/lodash@^4.14.146", "@types/lodash@^4.14.160":
-  version "4.14.170"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.170.tgz#0d67711d4bf7f4ca5147e9091b847479b87925d6"
-  integrity sha512-bpcvu/MKHHeYX+qeEN8GE7DIravODWdACVA1ctevD8CN24RhPZIKMn9ntfAsrvLfSX3cR5RrBKAbYm9bGs0A+Q==
+"@types/lodash@^4.14.172":
+  version "4.14.195"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.195.tgz#bafc975b252eb6cea78882ce8a7b6bf22a6de632"
+  integrity sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg==
 
 "@types/marked@^4.0.2":
   version "4.0.7"
@@ -4651,16 +4325,21 @@
   dependencies:
     "@types/unist" "*"
 
+"@types/mime@*":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@types/mime/-/mime-3.0.1.tgz#5f8f2bca0a5863cb69bc0b0acd88c96cb1d4ae10"
+  integrity sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==
+
+"@types/mime@^1":
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
+  integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==
+
 "@types/min-document@^2.19.0":
   version "2.19.0"
   resolved "https://registry.yarnpkg.com/@types/min-document/-/min-document-2.19.0.tgz#4f9919e789917c00de967a2c38fa8d234cbcd7d6"
   integrity sha512-lsYeSW1zfNqHTL1RuaOgfAhoiOWV1RAQDKT0BZ26z4Faz8llVIj1r1ablUo5QY6yzHMketuvu4+N0sv0eZpXTg==
 
-"@types/minimatch@*":
-  version "3.0.4"
-  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.4.tgz#f0ec25dbf2f0e4b18647313ac031134ca5b24b21"
-  integrity sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==
-
 "@types/node@*":
   version "15.6.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-15.6.1.tgz#32d43390d5c62c5b6ec486a9bc9c59544de39a08"
@@ -4671,11 +4350,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.13.tgz#e743bae112bd779ac9650f907197dd2caa7f0364"
   integrity sha512-1x8W5OpxPq+T85OUsHRP6BqXeosKmeXRtjoF39STcdf/UWLqUsoehstZKOi0CunhVqHG17AyZgpj20eRVooK6A==
 
-"@types/normalize-package-data@^2.4.0":
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
-  integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==
-
 "@types/object.omit@^3.0.0":
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/@types/object.omit/-/object.omit-3.0.0.tgz#0d31e1208eac8fe2ad5c9499a1016a8273bbfafc"
@@ -4696,22 +4370,17 @@
   resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.3.tgz#e7b5aebbac150f8b5fdd4a46e7f0bd8e65e19109"
   integrity sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==
 
-"@types/prettier@^2.0.0":
-  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.2.3.tgz#ef65165aea2924c9359205bf748865b8881753c0"
-  integrity sha512-PijRCG/K3s3w1We6ynUKdxEc5AcuuH3NBmMDP8uvKVp6X43UY7NQlTzczakXP3DJR0F4dfNQIGjU2cUeRYs2AA==
+"@types/prettier@^2.1.5":
+  version "2.7.3"
+  resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.3.tgz#3e51a17e291d01d17d3fc61422015a933af7a08f"
+  integrity sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==
 
 "@types/prismjs@*":
   version "1.26.0"
   resolved "https://registry.yarnpkg.com/@types/prismjs/-/prismjs-1.26.0.tgz#a1c3809b0ad61c62cac6d4e0c56d610c910b7654"
   integrity sha512-ZTaqn/qSqUuAq1YwvOFQfVW1AR/oQJlLSZVustdjwI+GZ8kr0MSHBj0tsXPW1EqHubx50gtBEjbPGsdZwQwCjQ==
 
-"@types/prop-types@*":
-  version "15.7.3"
-  resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.3.tgz#2ab0d5da2e5815f94b0b9d4b95d1e5f243ab2ca7"
-  integrity sha512-KfRL3PuHmqQLOG+2tGpRO26Ctg+Cq1E01D2DMriKEATHgWLfeNDmq9e29Q9WIky0dQ3NPkd1mzYH8Lm936Z9qw==
-
-"@types/prop-types@^15.7.5":
+"@types/prop-types@*", "@types/prop-types@^15.7.5":
   version "15.7.5"
   resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf"
   integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==
@@ -4721,6 +4390,11 @@
   resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.4.tgz#15925414e0ad2cd765bfef58842f7e26a7accb24"
   integrity sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug==
 
+"@types/qs@*":
+  version "6.9.7"
+  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
+  integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
+
 "@types/query-string@^6.3.0":
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/@types/query-string/-/query-string-6.3.0.tgz#b6fa172a01405abcaedac681118e78429d62ea39"
@@ -4733,6 +4407,11 @@
   resolved "https://registry.yarnpkg.com/@types/querystringify/-/querystringify-2.0.0.tgz#d1eab3214ee2b57c3bd7eba0ab94b231028522fb"
   integrity sha512-9WgEGTevECrXJC2LSWPqiPYWq8BRmeaOyZn47js/3V6UF0PWtcVfvvR43YjeO8BzBsthTz98jMczujOwTw+WYg==
 
+"@types/range-parser@*":
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
+  integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
+
 "@types/react-color@^3.0.6":
   version "3.0.6"
   resolved "https://registry.yarnpkg.com/@types/react-color/-/react-color-3.0.6.tgz#602fed023802b2424e7cd6ff3594ccd3d5055f9a"
@@ -4742,9 +4421,9 @@
     "@types/reactcss" "*"
 
 "@types/react-dom@*":
-  version "18.0.10"
-  resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.0.10.tgz#3b66dec56aa0f16a6cc26da9e9ca96c35c0b4352"
-  integrity sha512-E42GW/JA4Qv15wQdqJq8DL4JhNpB3prJgjgapN3qJT9K2zO5IIAQh4VXvCEDupoqAwnz0cY4RlXeC/ajX5SFHg==
+  version "18.2.6"
+  resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.2.6.tgz#ad621fa71a8db29af7c31b41b2ea3d8a6f4144d1"
+  integrity sha512-2et4PDvg6PVCyS7fuTc4gPoksV58bW0RwSxWKcPRcHZf0PRUGq03TKcD/rUHe3azfV6/5/biUBJw+HhCQjaP0A==
   dependencies:
     "@types/react" "*"
 
@@ -4809,22 +4488,58 @@
   dependencies:
     "@types/prismjs" "*"
 
-"@types/resolve@0.0.8":
-  version "0.0.8"
-  resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-0.0.8.tgz#f26074d238e02659e323ce1a13d041eee280e194"
-  integrity sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ==
+"@types/resolve@1.17.1":
+  version "1.17.1"
+  resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.17.1.tgz#3afd6ad8967c77e4376c598a82ddd58f46ec45d6"
+  integrity sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==
   dependencies:
     "@types/node" "*"
 
+"@types/retry@0.12.0":
+  version "0.12.0"
+  resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
+  integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==
+
 "@types/scheduler@*":
   version "0.16.1"
   resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.1.tgz#18845205e86ff0038517aab7a18a62a6b9f71275"
   integrity sha512-EaCxbanVeyxDRTQBkdLb3Bvl/HK7PBK6UJjsSixB0iHKoWxE5uu2Q/DgtpOhPIojN0Zl1whvOd7PoHs2P0s5eA==
 
-"@types/source-list-map@*":
-  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9"
-  integrity sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==
+"@types/semver@^7.3.12":
+  version "7.5.0"
+  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.0.tgz#591c1ce3a702c45ee15f47a42ade72c2fd78978a"
+  integrity sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==
+
+"@types/send@*":
+  version "0.17.1"
+  resolved "https://registry.yarnpkg.com/@types/send/-/send-0.17.1.tgz#ed4932b8a2a805f1fe362a70f4e62d0ac994e301"
+  integrity sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==
+  dependencies:
+    "@types/mime" "^1"
+    "@types/node" "*"
+
+"@types/serve-index@^1.9.1":
+  version "1.9.1"
+  resolved "https://registry.yarnpkg.com/@types/serve-index/-/serve-index-1.9.1.tgz#1b5e85370a192c01ec6cec4735cf2917337a6278"
+  integrity sha512-d/Hs3nWDxNL2xAczmOVZNj92YZCS6RGxfBPjKzuu/XirCgXdpKEb88dYNbrYGint6IVWLNP+yonwVAuRC0T2Dg==
+  dependencies:
+    "@types/express" "*"
+
+"@types/serve-static@*", "@types/serve-static@^1.13.10":
+  version "1.15.2"
+  resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.2.tgz#3e5419ecd1e40e7405d34093f10befb43f63381a"
+  integrity sha512-J2LqtvFYCzaj8pVYKw8klQXrLLk7TBZmQ4ShlcdkELFKGwGMfevMLneMMRkMgZxotOD9wg497LpC7O8PcvAmfw==
+  dependencies:
+    "@types/http-errors" "*"
+    "@types/mime" "*"
+    "@types/node" "*"
+
+"@types/sockjs@^0.3.33":
+  version "0.3.33"
+  resolved "https://registry.yarnpkg.com/@types/sockjs/-/sockjs-0.3.33.tgz#570d3a0b99ac995360e3136fd6045113b1bd236f"
+  integrity sha512-f0KEEe05NvUnat+boPTZ0dgaLZ4SfSouXUgv5noUiefG2ajgKjmETo9ZJyuqsl7dfl2aHlLJUiki6B4ZYldiiw==
+  dependencies:
+    "@types/node" "*"
 
 "@types/stack-utils@^2.0.0":
   version "2.0.0"
@@ -4845,11 +4560,6 @@
     "@types/react" "*"
     csstype "^3.0.2"
 
-"@types/tapable@^1", "@types/tapable@^1.0.5":
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.7.tgz#545158342f949e8fd3bfd813224971ecddc3fac4"
-  integrity sha512-0VBprVqfgFD7Ehb2vd8Lh9TG3jP98gvr8rgehQqzztZNI7o8zS8Ad4jyZneKELphpuE212D8J70LnSNQSyO6bQ==
-
 "@types/tern@*":
   version "0.23.4"
   resolved "https://registry.yarnpkg.com/@types/tern/-/tern-0.23.4.tgz#03926eb13dbeaf3ae0d390caf706b2643a0127fb"
@@ -4869,49 +4579,21 @@
   resolved "https://registry.yarnpkg.com/@types/throttle-debounce/-/throttle-debounce-2.1.0.tgz#1c3df624bfc4b62f992d3012b84c56d41eab3776"
   integrity sha512-5eQEtSCoESnh2FsiLTxE121IiE60hnMqcb435fShf4bpLRjEu1Eoekht23y6zXS9Ts3l+Szu3TARnTsA0GkOkQ==
 
-"@types/trusted-types@*":
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.2.tgz#fc25ad9943bcac11cceb8168db4f275e0e72e756"
-  integrity sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg==
+"@types/trusted-types@*", "@types/trusted-types@^2.0.2":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.3.tgz#a136f83b0758698df454e328759dbd3d44555311"
+  integrity sha512-NfQ4gyz38SL8sDNrSixxU2Os1a5xcdFxipAFxYEuLUlvU2uDwS4NUpsImcf1//SlWItCVMMLiylsxbmNMToV/g==
 
 "@types/turndown@^5.0.1":
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/@types/turndown/-/turndown-5.0.1.tgz#fcda7b02cda4c9d445be1440036df20f335b9387"
   integrity sha512-N8Ad4e3oJxh9n9BiZx9cbe/0M3kqDpOTm2wzj13wdDUxDPjfjloWIJaquZzWE1cYTAHpjOH3rcTnXQdpEfS/SQ==
 
-"@types/uglify-js@*":
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.13.0.tgz#1cad8df1fb0b143c5aba08de5712ea9d1ff71124"
-  integrity sha512-EGkrJD5Uy+Pg0NUR8uA4bJ5WMfljyad0G+784vLCNUkD+QwOJXUbBYExXfVGf7YtyzdQp3L/XMYcliB987kL5Q==
-  dependencies:
-    source-map "^0.6.1"
-
 "@types/unist@*", "@types/unist@^2.0.0", "@types/unist@^2.0.2", "@types/unist@^2.0.3":
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e"
   integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==
 
-"@types/webpack-sources@*":
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/@types/webpack-sources/-/webpack-sources-2.1.0.tgz#8882b0bd62d1e0ce62f183d0d01b72e6e82e8c10"
-  integrity sha512-LXn/oYIpBeucgP1EIJbKQ2/4ZmpvRl+dlrFdX7+94SKRUV3Evy3FsfMZY318vGhkWUS5MPhtOM3w1/hCOAOXcg==
-  dependencies:
-    "@types/node" "*"
-    "@types/source-list-map" "*"
-    source-map "^0.7.3"
-
-"@types/webpack@^4.41.8":
-  version "4.41.29"
-  resolved "https://registry.yarnpkg.com/@types/webpack/-/webpack-4.41.29.tgz#2e66c1de8223c440366469415c50a47d97625773"
-  integrity sha512-6pLaORaVNZxiB3FSHbyBiWM7QdazAWda1zvAq4SbZObZqHSDbWLi62iFdblVea6SK9eyBIVp5yHhKt/yNQdR7Q==
-  dependencies:
-    "@types/node" "*"
-    "@types/tapable" "^1"
-    "@types/uglify-js" "*"
-    "@types/webpack-sources" "*"
-    anymatch "^3.0.0"
-    source-map "^0.6.0"
-
 "@types/websocket@1.0.2":
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/@types/websocket/-/websocket-1.0.2.tgz#d2855c6a312b7da73ed16ba6781815bf30c6187a"
@@ -4919,6 +4601,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/ws@^8.5.5":
+  version "8.5.5"
+  resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.5.tgz#af587964aa06682702ee6dcbc7be41a80e4b28eb"
+  integrity sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
   version "20.2.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9"
@@ -4931,117 +4620,115 @@
   dependencies:
     "@types/yargs-parser" "*"
 
-"@types/zen-observable@^0.8.0":
-  version "0.8.2"
-  resolved "https://registry.yarnpkg.com/@types/zen-observable/-/zen-observable-0.8.2.tgz#808c9fa7e4517274ed555fa158f2de4b4f468e71"
-  integrity sha512-HrCIVMLjE1MOozVoD86622S7aunluLb2PJdPfb3nYiEtohm8mIB/vyv0Fd37AdeMFrTUQXEunw78YloMA3Qilg==
-
-"@typescript-eslint/eslint-plugin@^4.25.0", "@typescript-eslint/eslint-plugin@^4.5.0":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.25.0.tgz#d82657b6ab4caa4c3f888ff923175fadc2f31f2a"
-  integrity sha512-Qfs3dWkTMKkKwt78xp2O/KZQB8MPS1UQ5D3YW2s6LQWBE1074BE+Rym+b1pXZIX3M3fSvPUDaCvZLKV2ylVYYQ==
-  dependencies:
-    "@typescript-eslint/experimental-utils" "4.25.0"
-    "@typescript-eslint/scope-manager" "4.25.0"
-    debug "^4.1.1"
-    functional-red-black-tree "^1.0.1"
-    lodash "^4.17.15"
-    regexpp "^3.0.0"
-    semver "^7.3.2"
-    tsutils "^3.17.1"
-
-"@typescript-eslint/experimental-utils@4.25.0", "@typescript-eslint/experimental-utils@^4.0.1":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.25.0.tgz#b2febcfa715d2c1806fd5f0335193a6cd270df54"
-  integrity sha512-f0doRE76vq7NEEU0tw+ajv6CrmPelw5wLoaghEHkA2dNLFb3T/zJQqGPQ0OYt5XlZaS13MtnN+GTPCuUVg338w==
-  dependencies:
-    "@types/json-schema" "^7.0.3"
-    "@typescript-eslint/scope-manager" "4.25.0"
-    "@typescript-eslint/types" "4.25.0"
-    "@typescript-eslint/typescript-estree" "4.25.0"
-    eslint-scope "^5.0.0"
-    eslint-utils "^2.0.0"
-
-"@typescript-eslint/experimental-utils@^3.10.1":
-  version "3.10.1"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-3.10.1.tgz#e179ffc81a80ebcae2ea04e0332f8b251345a686"
-  integrity sha512-DewqIgscDzmAfd5nOGe4zm6Bl7PKtMG2Ad0KG8CUZAHlXfAKTF9Ol5PXhiMh39yRL2ChRH1cuuUGOcVyyrhQIw==
-  dependencies:
-    "@types/json-schema" "^7.0.3"
-    "@typescript-eslint/types" "3.10.1"
-    "@typescript-eslint/typescript-estree" "3.10.1"
-    eslint-scope "^5.0.0"
-    eslint-utils "^2.0.0"
-
-"@typescript-eslint/parser@^4.25.0", "@typescript-eslint/parser@^4.4.1", "@typescript-eslint/parser@^4.5.0":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.25.0.tgz#6b2cb6285aa3d55bfb263c650739091b0f19aceb"
-  integrity sha512-OZFa1SKyEJpAhDx8FcbWyX+vLwh7OEtzoo2iQaeWwxucyfbi0mT4DijbOSsTgPKzGHr6GrF2V5p/CEpUH/VBxg==
-  dependencies:
-    "@typescript-eslint/scope-manager" "4.25.0"
-    "@typescript-eslint/types" "4.25.0"
-    "@typescript-eslint/typescript-estree" "4.25.0"
-    debug "^4.1.1"
-
-"@typescript-eslint/scope-manager@4.25.0":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.25.0.tgz#9d86a5bcc46ef40acd03d85ad4e908e5aab8d4ca"
-  integrity sha512-2NElKxMb/0rya+NJG1U71BuNnp1TBd1JgzYsldsdA83h/20Tvnf/HrwhiSlNmuq6Vqa0EzidsvkTArwoq+tH6w==
+"@types/yargs@^16.0.0":
+  version "16.0.5"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-16.0.5.tgz#12cc86393985735a283e387936398c2f9e5f88e3"
+  integrity sha512-AxO/ADJOBFJScHbWhq2xAhlWP24rY4aCEG/NFaMvbT3X2MgRsLjhjQwsn0Zi5zn0LG9jUhCCZMeX9Dkuw6k+vQ==
   dependencies:
-    "@typescript-eslint/types" "4.25.0"
-    "@typescript-eslint/visitor-keys" "4.25.0"
-
-"@typescript-eslint/types@3.10.1":
-  version "3.10.1"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-3.10.1.tgz#1d7463fa7c32d8a23ab508a803ca2fe26e758727"
-  integrity sha512-+3+FCUJIahE9q0lDi1WleYzjCwJs5hIsbugIgnbB+dSCYUxl8L6PwmsyOPFZde2hc1DlTo/xnkOgiTLSyAbHiQ==
-
-"@typescript-eslint/types@4.25.0":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.25.0.tgz#0e444a5c5e3c22d7ffa5e16e0e60510b3de5af87"
-  integrity sha512-+CNINNvl00OkW6wEsi32wU5MhHti2J25TJsJJqgQmJu3B3dYDBcmOxcE5w9cgoM13TrdE/5ND2HoEnBohasxRQ==
+    "@types/yargs-parser" "*"
 
-"@typescript-eslint/typescript-estree@3.10.1":
-  version "3.10.1"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-3.10.1.tgz#fd0061cc38add4fad45136d654408569f365b853"
-  integrity sha512-QbcXOuq6WYvnB3XPsZpIwztBoquEYLXh2MtwVU+kO8jgYCiv4G5xrSP/1wg4tkvrEE+esZVquIPX/dxPlePk1w==
+"@types/yargs@^17.0.8":
+  version "17.0.24"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.24.tgz#b3ef8d50ad4aa6aecf6ddc97c580a00f5aa11902"
+  integrity sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==
   dependencies:
-    "@typescript-eslint/types" "3.10.1"
-    "@typescript-eslint/visitor-keys" "3.10.1"
-    debug "^4.1.1"
-    glob "^7.1.6"
-    is-glob "^4.0.1"
-    lodash "^4.17.15"
-    semver "^7.3.2"
-    tsutils "^3.17.1"
+    "@types/yargs-parser" "*"
 
-"@typescript-eslint/typescript-estree@4.25.0":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.25.0.tgz#942e4e25888736bff5b360d9b0b61e013d0cfa25"
-  integrity sha512-1B8U07TGNAFMxZbSpF6jqiDs1cVGO0izVkf18Q/SPcUAc9LhHxzvSowXDTvkHMWUVuPpagupaW63gB6ahTXVlg==
-  dependencies:
-    "@typescript-eslint/types" "4.25.0"
-    "@typescript-eslint/visitor-keys" "4.25.0"
-    debug "^4.1.1"
-    globby "^11.0.1"
-    is-glob "^4.0.1"
-    semver "^7.3.2"
-    tsutils "^3.17.1"
+"@types/zen-observable@^0.8.0":
+  version "0.8.2"
+  resolved "https://registry.yarnpkg.com/@types/zen-observable/-/zen-observable-0.8.2.tgz#808c9fa7e4517274ed555fa158f2de4b4f468e71"
+  integrity sha512-HrCIVMLjE1MOozVoD86622S7aunluLb2PJdPfb3nYiEtohm8mIB/vyv0Fd37AdeMFrTUQXEunw78YloMA3Qilg==
 
-"@typescript-eslint/visitor-keys@3.10.1":
-  version "3.10.1"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-3.10.1.tgz#cd4274773e3eb63b2e870ac602274487ecd1e931"
-  integrity sha512-9JgC82AaQeglebjZMgYR5wgmfUdUc+EitGUUMW8u2nDckaeimzW+VsoLV6FoimPv2id3VQzfjwBxEMVz08ameQ==
-  dependencies:
-    eslint-visitor-keys "^1.1.0"
+"@typescript-eslint/eslint-plugin@^5.38.1", "@typescript-eslint/eslint-plugin@^5.5.0":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.60.1.tgz#81382d6ecb92b8dda70e91f9035611cb2fecd1c3"
+  integrity sha512-KSWsVvsJsLJv3c4e73y/Bzt7OpqMCADUO846bHcuWYSYM19bldbAeDv7dYyV0jwkbMfJ2XdlzwjhXtuD7OY6bw==
+  dependencies:
+    "@eslint-community/regexpp" "^4.4.0"
+    "@typescript-eslint/scope-manager" "5.60.1"
+    "@typescript-eslint/type-utils" "5.60.1"
+    "@typescript-eslint/utils" "5.60.1"
+    debug "^4.3.4"
+    grapheme-splitter "^1.0.4"
+    ignore "^5.2.0"
+    natural-compare-lite "^1.4.0"
+    semver "^7.3.7"
+    tsutils "^3.21.0"
+
+"@typescript-eslint/experimental-utils@^5.0.0":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-5.60.1.tgz#d783bb63b9183541019a945eda6a9d96b096d985"
+  integrity sha512-TXUdLxv2t8181nh5yLXl/Gr/zKj1ZofQ7m+ZdmG2+El0TYOHCvlZfc35D4nturemC3RUnf3KmLuFp3bVBjkG5w==
+  dependencies:
+    "@typescript-eslint/utils" "5.60.1"
+
+"@typescript-eslint/parser@^5.38.1", "@typescript-eslint/parser@^5.5.0":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.60.1.tgz#0f2f58209c0862a73e3d5a56099abfdfa21d0fd3"
+  integrity sha512-pHWlc3alg2oSMGwsU/Is8hbm3XFbcrb6P5wIxcQW9NsYBfnrubl/GhVVD/Jm/t8HXhA2WncoIRfBtnCgRGV96Q==
+  dependencies:
+    "@typescript-eslint/scope-manager" "5.60.1"
+    "@typescript-eslint/types" "5.60.1"
+    "@typescript-eslint/typescript-estree" "5.60.1"
+    debug "^4.3.4"
+
+"@typescript-eslint/scope-manager@5.60.1":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.60.1.tgz#35abdb47f500c68c08f2f2b4f22c7c79472854bb"
+  integrity sha512-Dn/LnN7fEoRD+KspEOV0xDMynEmR3iSHdgNsarlXNLGGtcUok8L4N71dxUgt3YvlO8si7E+BJ5Fe3wb5yUw7DQ==
+  dependencies:
+    "@typescript-eslint/types" "5.60.1"
+    "@typescript-eslint/visitor-keys" "5.60.1"
+
+"@typescript-eslint/type-utils@5.60.1":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.60.1.tgz#17770540e98d65ab4730c7aac618003f702893f4"
+  integrity sha512-vN6UztYqIu05nu7JqwQGzQKUJctzs3/Hg7E2Yx8rz9J+4LgtIDFWjjl1gm3pycH0P3mHAcEUBd23LVgfrsTR8A==
+  dependencies:
+    "@typescript-eslint/typescript-estree" "5.60.1"
+    "@typescript-eslint/utils" "5.60.1"
+    debug "^4.3.4"
+    tsutils "^3.21.0"
+
+"@typescript-eslint/types@5.60.1":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.60.1.tgz#a17473910f6b8d388ea83c9d7051af89c4eb7561"
+  integrity sha512-zDcDx5fccU8BA0IDZc71bAtYIcG9PowaOwaD8rjYbqwK7dpe/UMQl3inJ4UtUK42nOCT41jTSCwg76E62JpMcg==
+
+"@typescript-eslint/typescript-estree@5.60.1":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.60.1.tgz#8c71824b7165b64d5ebd7aa42968899525959834"
+  integrity sha512-hkX70J9+2M2ZT6fhti5Q2FoU9zb+GeZK2SLP1WZlvUDqdMbEKhexZODD1WodNRyO8eS+4nScvT0dts8IdaBzfw==
+  dependencies:
+    "@typescript-eslint/types" "5.60.1"
+    "@typescript-eslint/visitor-keys" "5.60.1"
+    debug "^4.3.4"
+    globby "^11.1.0"
+    is-glob "^4.0.3"
+    semver "^7.3.7"
+    tsutils "^3.21.0"
+
+"@typescript-eslint/utils@5.60.1", "@typescript-eslint/utils@^5.58.0":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.60.1.tgz#6861ebedbefba1ac85482d2bdef6f2ff1eb65b80"
+  integrity sha512-tiJ7FFdFQOWssFa3gqb94Ilexyw0JVxj6vBzaSpfN/8IhoKkDuSAenUKvsSHw2A/TMpJb26izIszTXaqygkvpQ==
+  dependencies:
+    "@eslint-community/eslint-utils" "^4.2.0"
+    "@types/json-schema" "^7.0.9"
+    "@types/semver" "^7.3.12"
+    "@typescript-eslint/scope-manager" "5.60.1"
+    "@typescript-eslint/types" "5.60.1"
+    "@typescript-eslint/typescript-estree" "5.60.1"
+    eslint-scope "^5.1.1"
+    semver "^7.3.7"
 
-"@typescript-eslint/visitor-keys@4.25.0":
-  version "4.25.0"
-  resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.25.0.tgz#863e7ed23da4287c5b469b13223255d0fde6aaa7"
-  integrity sha512-AmkqV9dDJVKP/TcZrbf6s6i1zYXt5Hl8qOLrRDTFfRNae4+LB8A4N3i+FLZPW85zIxRy39BgeWOfMS3HoH5ngg==
+"@typescript-eslint/visitor-keys@5.60.1":
+  version "5.60.1"
+  resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.60.1.tgz#19a877358bf96318ec35d90bfe6bd1445cce9434"
+  integrity sha512-xEYIxKcultP6E/RMKqube11pGjXH1DCo60mQoWhVYyKfLkwbIVVjYxmOenNMxILx0TjCujPTjjnTIVzm09TXIw==
   dependencies:
-    "@typescript-eslint/types" "4.25.0"
-    eslint-visitor-keys "^2.0.0"
+    "@typescript-eslint/types" "5.60.1"
+    eslint-visitor-keys "^3.3.0"
 
 "@uiw/react-markdown-preview@3.0.6":
   version "3.0.6"
@@ -5063,540 +4750,402 @@
     "@uiw/react-markdown-preview" "3.0.6"
     rehype "11.0.0"
 
-"@vx/axis@^0.0.175":
-  version "0.0.175"
-  resolved "https://registry.yarnpkg.com/@vx/axis/-/axis-0.0.175.tgz#312c07b81e8b043876436cab3bd8e90f30a1f7ec"
-  integrity sha512-qVRIHurnbPnRF4p0KQITArOUSF564tWW1pc48giLz+DJGlcJ4H9RfOSTpV6rnnP15xto6pQdQehBgBAvFRmoig==
+"@use-gesture/core@10.2.27":
+  version "10.2.27"
+  resolved "https://registry.yarnpkg.com/@use-gesture/core/-/core-10.2.27.tgz#0f24b17c036cd828ba07e3451ff45e2df959c6f5"
+  integrity sha512-V4XV7hn9GAD2MYu8yBBVi5iuWBsAMfjPRMsEVzoTNGYH72tf0kFP+OKqGKc8YJFQIJx6yj+AOqxmEHOmx2/MEA==
+
+"@use-gesture/react@^10.0.0-beta.22":
+  version "10.2.27"
+  resolved "https://registry.yarnpkg.com/@use-gesture/react/-/react-10.2.27.tgz#7fbd50d14449ec5bc49c9b6cfef8a2845f5e0608"
+  integrity sha512-7E5vnWCxeslWlxwZ8uKIcnUZVMTRMZ8cvSnLLKF1NkyNb3PnNiAzoXM4G1vTKJKRhgOTeI6wK1YsEpwo9ABV5w==
   dependencies:
-    "@vx/group" "0.0.170"
-    "@vx/point" "0.0.165"
-    "@vx/shape" "0.0.175"
-    "@vx/text" "0.0.175"
-    classnames "^2.2.5"
-    prop-types "^15.6.0"
+    "@use-gesture/core" "10.2.27"
 
-"@vx/axis@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/axis/-/axis-0.0.199.tgz#b7dc829e21cb49773f11d29f8fe901ae2794f215"
-  integrity sha512-pDP5Lf7bzVneh4YrPmJnIdiheHqiiVqWku8LKxTckIrvR55E3S1etlK9RSiT/3I9iERC9l/CXi7W7pJLKi7TaQ==
+"@visx/annotation@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@visx/annotation/-/annotation-3.0.1.tgz#007c0030155d95897a74422a9d3aad65c7ef82a7"
+  integrity sha512-otf2AZDlt/XCpOBG0gkPowXVerLJW5yXPFis94Km0bj629htyKOsY5GeJKlBeB89ddxdi4zWHkaFxGFJ79Pqog==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/react" "*"
-    "@vx/group" "0.0.199"
-    "@vx/point" "0.0.199"
-    "@vx/scale" "0.0.199"
-    "@vx/shape" "0.0.199"
-    "@vx/text" "0.0.199"
-    classnames "^2.2.5"
-    prop-types "^15.6.0"
+    "@visx/drag" "3.0.1"
+    "@visx/group" "3.0.0"
+    "@visx/text" "3.0.0"
+    classnames "^2.3.1"
+    prop-types "^15.5.10"
+    react-use-measure "^2.0.4"
 
-"@vx/bounds@0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/bounds/-/bounds-0.0.165.tgz#75f107a6deb58223c6878db5053382eff3174567"
-  integrity sha512-ZvRb72/4QNs1ZrytZTZxd0hfAb/KKfhsdkcYtIQkmdF6dTsjigMQZ+h2bLvLnbZb/RxyCCoxdiZSGXd+T1c//Q==
+"@visx/axis@3.2.0", "@visx/axis@^3.1.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/axis/-/axis-3.2.0.tgz#6c6fc9bae92388f94a4b1c97b913f77b9b352fae"
+  integrity sha512-EqRsJshvXDB2R2o9U8BHiWNLz+HtHsVcPF0PKNO6BN20okV5KogOl1KPb0wUB0xD4vbAzc94fIm+MBaijpEWQg==
   dependencies:
-    prop-types "^15.5.10"
+    "@types/react" "*"
+    "@visx/group" "3.0.0"
+    "@visx/point" "3.0.1"
+    "@visx/scale" "3.2.0"
+    "@visx/shape" "3.2.0"
+    "@visx/text" "3.0.0"
+    classnames "^2.3.1"
+    prop-types "^15.6.0"
 
-"@vx/bounds@0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/bounds/-/bounds-0.0.199.tgz#9a57aa5ea59c96d7f2369921ee6d7751d65f731d"
-  integrity sha512-3hq9MmfiSB5ciKTzHhB4/UV77JIIUEd1hDj5EBfLUUOL27wXgM6+RSOB+FxFt3tRJniltp3iZs5Gb75SDecpGw==
+"@visx/bounds@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/bounds/-/bounds-3.0.0.tgz#cf357cbff90a1fe5f95eb9d9288dd8794b744a7f"
+  integrity sha512-YQaSSER9erxlhppzRms6cvYdKqcIwk6eksrGdbJkBoHobhPo1JCIUXlmrA4qgrEnXInPJpueGE+PE5F+Dk12DA==
   dependencies:
     "@types/react" "*"
     "@types/react-dom" "*"
     prop-types "^15.5.10"
 
-"@vx/clip-path@0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/clip-path/-/clip-path-0.0.165.tgz#93cd65cc6a35319c7e403ce7b973ac1c8045b741"
-  integrity sha512-mBCbgguLMVyGvar5FbxqyyY4NQFlnXoSLF0TrhgWYkF/FCXdE1CzBC+Y4iXIJOY0ZTtluqL9XrNdIDpx49AmuA==
-
-"@vx/curve@0.0.165", "@vx/curve@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/curve/-/curve-0.0.165.tgz#330d1512dceae0af43dd3eb4c85523132030a3a0"
-  integrity sha512-fiQAGrKNGjJbL+eixUckJqIZDWXH/1NtIyyDbSz3J7ksk0QpYr5BgWcNJN76HLNt7wfcLwNzCHeNs4iVYyFGTg==
-  dependencies:
-    d3-shape "^1.0.6"
-
-"@vx/curve@0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/curve/-/curve-0.0.199.tgz#c462f11989068390333607fc254ff178a2f5227a"
-  integrity sha512-dJ84gIbFdZ/3KvYmmCrXiA4kDmM23NdhB3/8DGEFqsMn16b881hiN2YtXvkluaXfs01e/s4WcHLgAR7Fn9KYaA==
+"@visx/curve@3.0.0", "@visx/curve@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/curve/-/curve-3.0.0.tgz#c54568472e00a38483c58cf52e4a6ddb2887c2d4"
+  integrity sha512-kvHJDLBeczTQ87ZExSTfRxej06l6o6UiQ0NHf9+xpAin06y6Qk1ThOHHWJTGM6KGzwlu7jEauJGHwZs6nMhDvA==
   dependencies:
     "@types/d3-shape" "^1.3.1"
     d3-shape "^1.0.6"
 
-"@vx/event@0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/event/-/event-0.0.199.tgz#f8cc129fe3a6dfd9aa416cce4c0c2d843ad01fec"
-  integrity sha512-y/5z9hBGQ4XDfruGObzk3vFsafJyUmUea9X9JWpMNqD8uyHzgmitNtGxcno0zUXjJAUv65ALHPmcxdLzsEMfsQ==
+"@visx/drag@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@visx/drag/-/drag-3.0.1.tgz#753d5f471d4e31679ca4fddcb476cf2d5c7eb6e4"
+  integrity sha512-yi2AB/unUfNYBRKS4pmUOuz8MjaAAYjsQGYcD/s4LqeQjd+lBZF7CuNcYZ/maGNQAEUfgLr2czIzADanOMtMaw==
   dependencies:
     "@types/react" "*"
-    "@vx/point" "0.0.199"
-
-"@vx/event@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/event/-/event-0.0.165.tgz#675d89fdfdc08d0c99c36ff1a381ea50fccfba2e"
-  integrity sha512-FsQiw0f3s5DQB6aBQmBcoWk9e4q65LcDobHIyV8qrmpW2QgV2NvQFM1w0Q300ohpRMgJDzGk68HHHQgFOJvApw==
-  dependencies:
-    "@vx/point" "0.0.165"
+    "@visx/event" "3.0.1"
+    "@visx/point" "3.0.1"
+    prop-types "^15.5.10"
 
-"@vx/glyph@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/glyph/-/glyph-0.0.165.tgz#ba6fe31700dae852c60468e00dd732fa7521d1fc"
-  integrity sha512-kccUm40e/VCtayxqvcwc2K2M6oNXO7IafwIfw1RRv6Fj4Iutto9ZpI+PGOf/zPnYVueoLnWBXT/HE7IRS+C2gw==
+"@visx/event@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@visx/event/-/event-3.0.1.tgz#d5358f52ff5ef30036d955bd2b68b96472ff2d6f"
+  integrity sha512-tK1EUYQLLStBuoCMbm8LJ3VbDyCVI8HjT0pMRQxm+C75FSIVWvrThgrfrC9sWOFnEMEYWspZO7hI5zjsPKjLQA==
   dependencies:
-    "@vx/group" "0.0.165"
-    classnames "^2.2.5"
-    d3-shape "^1.2.0"
+    "@types/react" "*"
+    "@visx/point" "3.0.1"
 
-"@vx/glyph@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/glyph/-/glyph-0.0.199.tgz#1af503d97cbf8713f32374fce274e3be1514b0e6"
-  integrity sha512-/IKLRa3jvycJsdNwyqksUwOQisLWWE46DNQCTHuRe6uDtlRLsHcjUF1/vxDW4g7W//8os8TSKFwuWLqjwQyO6A==
+"@visx/glyph@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/glyph/-/glyph-3.0.0.tgz#218a96aa0ccba95dc77e46ab08d26ad89198f3a8"
+  integrity sha512-r1B0IocfWfhTABKjam0qqsWKjxLxZfGwefnwn8IcfELSd9iAUtLbI/46nP4roQRHhB/Wl3RBbgA97fZw8f1MxA==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/d3-shape" "^1.3.1"
     "@types/react" "*"
-    "@vx/group" "0.0.199"
-    classnames "^2.2.5"
+    "@visx/group" "3.0.0"
+    classnames "^2.3.1"
     d3-shape "^1.2.0"
     prop-types "^15.6.2"
 
-"@vx/gradient@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/gradient/-/gradient-0.0.165.tgz#0cc0fe873e6acded4943fa274f68601ad5a50a38"
-  integrity sha512-FjRXMTmcy7k0TWsfDzWWXw6T9WXKP+6LS/GRgnguq271pab/P+AdOJThsVxtBgUc8ZOAPbub3/2Gggz9d8tocg==
-  dependencies:
-    classnames "^2.2.5"
-    prop-types "^15.5.7"
-
-"@vx/grid@^0.0.180":
-  version "0.0.180"
-  resolved "https://registry.yarnpkg.com/@vx/grid/-/grid-0.0.180.tgz#31f2f73c9055d0ab2bf38f03a0c7dc47b9c12327"
-  integrity sha512-+ugS0c6GbwHr6pFU0znnOG3/zTwRRadvWwj3E4ZOHmKUSz6ZEN6JNo+rD3WSZckYwLis6UivmYfJ5cV6AM4ufg==
+"@visx/grid@3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/grid/-/grid-3.2.0.tgz#1db936e0c20bf45d2e8280f8c70b62982c67fb59"
+  integrity sha512-f9EkYOB/acGVh4DwIzVZ39+uChvWiPRwO029L8Dlc6uTvG00sgnKttfZdDP2iYnGrAqyFTHsPLsuTQs4hKP9jA==
   dependencies:
-    "@vx/group" "0.0.170"
-    "@vx/point" "0.0.165"
-    "@vx/shape" "0.0.179"
-    classnames "^2.2.5"
+    "@types/react" "*"
+    "@visx/curve" "3.0.0"
+    "@visx/group" "3.0.0"
+    "@visx/point" "3.0.1"
+    "@visx/scale" "3.2.0"
+    "@visx/shape" "3.2.0"
+    classnames "^2.3.1"
     prop-types "^15.6.2"
 
-"@vx/group@0.0.165", "@vx/group@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/group/-/group-0.0.165.tgz#2342523225de94859b5be49c3072dc6bb6795e78"
-  integrity sha512-gi1DSg8AAaVRseyWiq8y4bzyvKiQIXT6vDUYBVRmv2LBcpHocBGaxNiNK0X602RgLG0XmNyRv6qSCWLOaBs3Mg==
-  dependencies:
-    classnames "^2.2.5"
-
-"@vx/group@0.0.170":
-  version "0.0.170"
-  resolved "https://registry.yarnpkg.com/@vx/group/-/group-0.0.170.tgz#8b30b3ea07c348fe22253812fe7cb6d4200d725d"
-  integrity sha512-RnDdRoy0YI5hokk+YWXc8t39Kp51i4BdCpiwkDJU4YypGycTYnDFjicam6jigUmZ/6wyMirDf/aQboWviFLt2Q==
-  dependencies:
-    classnames "^2.2.5"
-
-"@vx/group@0.0.199", "@vx/group@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/group/-/group-0.0.199.tgz#167410a83c83aa2011f6e7cabafd68ed5659c690"
-  integrity sha512-QSHHQn1a5z9H/vrZP8FIlJTznWjuyDcloXU/1jocf1FL9BxWV7RErvZ/IBWr+bIZVXLZXUpfVpX9JMry+eESIg==
+"@visx/group@3.0.0", "@visx/group@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/group/-/group-3.0.0.tgz#e7f9752599bcc7e141ff5317a2a9a502577ab8df"
+  integrity sha512-SFjXhTMcsaVAb1/TVL1KM5vn8gQTIVgSx0ATdDl4BJSFp2ym1lO8LY4jpV4SFweaHnWxVwrrfGLTn5QsYnvmjQ==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/react" "*"
-    classnames "^2.2.5"
+    classnames "^2.3.1"
     prop-types "^15.6.2"
 
-"@vx/hierarchy@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/hierarchy/-/hierarchy-0.0.199.tgz#cfb88a82984eac6a47613b4a6f134f50e3a89df7"
-  integrity sha512-cAkRnudQXqvqV2Fpg5yrOm4GD48c0pNJbnAY6cSGReIoxXySt8mPBAyI6VnlF/zfcs9w30AVkiUUekzo1NjaAA==
+"@visx/hierarchy@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/hierarchy/-/hierarchy-3.0.0.tgz#77c25f7f653542295f6c124563440e1d491b0e36"
+  integrity sha512-AZEHIiBdxgDhQqMQMwP7rFJlHiHN6KuNJK+SGjVYe1s+xoVisKBG7P/WGd3HCRNzh7hc6gffFow9enDT573uHQ==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/d3-hierarchy" "^1.1.6"
     "@types/react" "*"
-    "@vx/group" "0.0.199"
-    classnames "^2.2.5"
+    "@visx/group" "3.0.0"
+    classnames "^2.3.1"
     d3-hierarchy "^1.1.4"
     prop-types "^15.6.1"
 
-"@vx/legend@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/legend/-/legend-0.0.199.tgz#8c2611ccc50c710effe395941fa13efe2bf9d641"
-  integrity sha512-plGergquRrefNE00HrGlofo0kmX1iMDTs6VlRBW+OKWrgJVtCgCD6jAWxz9No/MbwleMgsQsWd3nzKj6oKFAhw==
+"@visx/legend@^3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/legend/-/legend-3.2.0.tgz#de8453b89ca4bc89f4c00a48b23c848fef570fcb"
+  integrity sha512-XyZjTDpTlzcTTFdrV16I58NKl1cvYtmFdr4Lb1dvpQkAJkFyWzjrWEqOSGw1yJkaimQKlmaemn8uRgzLXLfKhw==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/react" "*"
-    "@vx/group" "0.0.199"
-    "@vx/scale" "0.0.199"
-    classnames "^2.2.5"
-    prop-types "^15.5.10"
-
-"@vx/pattern@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/pattern/-/pattern-0.0.165.tgz#0d317cb9a13205e35691f702442739ff0256711d"
-  integrity sha512-h5nmfcYlQYYzNhlhqaYUvVnkmGnC0yWv5yU1snjHweGmIHTovV3RAbKgVFAP7kB3i2rbEtC3O8WkJN++cZdLzA==
-  dependencies:
-    classnames "^2.2.5"
+    "@visx/group" "3.0.0"
+    "@visx/scale" "3.2.0"
+    classnames "^2.3.1"
     prop-types "^15.5.10"
 
-"@vx/point@0.0.165", "@vx/point@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/point/-/point-0.0.165.tgz#7ebde5da3d86954fe31a56f923f31550f0b4b867"
-  integrity sha512-spoHilhjcWNgccrSzBUPw+PXV81tYxeyEWBkgr35aGVU4m7YT86Ywvfemwp7AVVGPn+XJHrhB0ujAhDoyqFPoA==
-
-"@vx/point@0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/point/-/point-0.0.199.tgz#2ae16d99ca8d351fe09857533ba1e9c995696706"
-  integrity sha512-8mlOHgaCqT8qKhMSAA5U7tsBdOQBku7DIJ5fteTWeZt6OOafVSGvfT8LjphN8NA1xbY3ZY3L1bn9ZnZurYHDJQ==
+"@visx/point@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@visx/point/-/point-3.0.1.tgz#77587ddaabf6f3023f09f8a0ce33a2c27c9d64c8"
+  integrity sha512-S5WOBMgEP2xHcgs3A2BFB2vwzrk0tMmn3PGZAbQJ+lu4HlnalDP72klUnxLTH8xclNNvpUHtHM5eLIJXyHx6Pw==
 
-"@vx/responsive@^0.0.192":
-  version "0.0.192"
-  resolved "https://registry.yarnpkg.com/@vx/responsive/-/responsive-0.0.192.tgz#721d032bec38b9e3ff5fde2e4d5d8ee5a81cc517"
-  integrity sha512-HaXVwhSJXUfRbzRV+glxsX0ki2Hi1mdpz42iuGArVQgDPJEmBHjkXyoiXU8U6v66M7FAH+OyKgtc5j2bfhyYzA==
+"@visx/react-spring@3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/react-spring/-/react-spring-3.2.0.tgz#7174ce2a89cc91efd6f0a43e2c90169800ebb6c5"
+  integrity sha512-gWr9/ERz+nq9B2PtaUe8keuj9tn1oXQfzuU9z8/n4PFx2xq6csx4IVEnQRtfLrQhsv3iU9GQ3wbQUcYsdJTdvQ==
   dependencies:
-    lodash "^4.17.10"
-    prop-types "^15.6.1"
-    resize-observer-polyfill "1.5.0"
-
-"@vx/scale@0.0.165", "@vx/scale@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/scale/-/scale-0.0.165.tgz#8575880c48296a80c0e9459057f826e8f903bc07"
-  integrity sha512-5jSgXJDU6J/KWIyCbpjHqysPCddp7tG3LbTV7UmtB1Qleb4m4slShTVSE7+EKU+zgiQPDGm0+E2ht4cet+7F7A==
-  dependencies:
-    d3-scale "^2.0.0"
-
-"@vx/scale@0.0.199", "@vx/scale@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/scale/-/scale-0.0.199.tgz#bd3b0b7add5e99c0b8240ca72edfad3b96bbd700"
-  integrity sha512-AbrPbQUEYFR0oxMHry7LqvLbyXxXek2aDAnwzzp7C+Yh1wfUk63hSuaRp13CO7+3r3gIy+99yrsaYPsgt2I6Nw==
-  dependencies:
-    "@types/d3-interpolate" "^1.3.1"
-    "@types/d3-scale" "^2.1.1"
-    "@types/d3-time" "^1.0.10"
-    d3-interpolate "^1.4.0"
-    d3-scale "^3.0.1"
-    d3-time "^1.1.0"
-
-"@vx/shape@0.0.170":
-  version "0.0.170"
-  resolved "https://registry.yarnpkg.com/@vx/shape/-/shape-0.0.170.tgz#55cf5968f7a6465484aac1a5541275b4813bb9a2"
-  integrity sha512-rm8oVRP0ejgwGhQTVhqP5awqphWX60FgbnRt9X+YBUqgv7Qyedfgs/CHd/5QFZX3aPp8d4F+b4+lghbIYiMgmQ==
-  dependencies:
-    "@vx/curve" "0.0.165"
-    "@vx/group" "0.0.170"
-    "@vx/point" "0.0.165"
-    classnames "^2.2.5"
-    d3-path "^1.0.5"
-    d3-shape "^1.2.0"
-    prop-types "^15.5.10"
+    "@types/react" "*"
+    "@visx/axis" "3.2.0"
+    "@visx/grid" "3.2.0"
+    "@visx/scale" "3.2.0"
+    "@visx/text" "3.0.0"
+    classnames "^2.3.1"
+    prop-types "^15.6.2"
 
-"@vx/shape@0.0.175":
-  version "0.0.175"
-  resolved "https://registry.yarnpkg.com/@vx/shape/-/shape-0.0.175.tgz#099bcd4fdc890988fa373526b170c11942af2255"
-  integrity sha512-bjAJoIIpKjUEPDV2xmTYGUvSvwRztv+6rd1c6NPZG/nIuqsMHFnFig/2xTcQJEQhRg6aKzvxIUo43zPSSq3fWA==
+"@visx/responsive@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/responsive/-/responsive-3.0.0.tgz#e183c54ce04cffe756378872d30ac88c66a137ac"
+  integrity sha512-immnxQwOWlrxbnlCIqJWuDpPfrM6tglgMTN1WsyXyGluLMJqhuuxqxllfXaRPkQFS4fcvs66KCEELdazh96U2w==
   dependencies:
-    "@vx/curve" "0.0.165"
-    "@vx/group" "0.0.170"
-    "@vx/point" "0.0.165"
-    classnames "^2.2.5"
-    d3-path "^1.0.5"
-    d3-shape "^1.2.0"
-    prop-types "^15.5.10"
+    "@types/lodash" "^4.14.172"
+    "@types/react" "*"
+    lodash "^4.17.21"
+    prop-types "^15.6.1"
 
-"@vx/shape@0.0.179":
-  version "0.0.179"
-  resolved "https://registry.yarnpkg.com/@vx/shape/-/shape-0.0.179.tgz#038c449743d1e05b7b2d20151e9ab6e739f73516"
-  integrity sha512-YHVNx4xGpbjolkW3Lb5pEgJB0+u349vfnLI976DJlinY0hRNa4TZbWXOB4ywLIrYzQEXXPMUR8WtdubNxg6g0w==
+"@visx/scale@3.2.0", "@visx/scale@^3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/scale/-/scale-3.2.0.tgz#a10a41e77331061819e4c9568e76158f3489a951"
+  integrity sha512-8XOZ+LDr/QVC61EBtO4xvcRJQ/M0bfQHme/EJsiZU5DabrD3cHJAOxNW6pLF5PkvDGQII2V3delijNPDqTiTNg==
   dependencies:
-    "@vx/curve" "0.0.165"
-    "@vx/group" "0.0.170"
-    "@vx/point" "0.0.165"
-    classnames "^2.2.5"
-    d3-path "^1.0.5"
-    d3-shape "^1.2.0"
-    prop-types "^15.5.10"
+    "@visx/vendor" "3.2.0"
 
-"@vx/shape@0.0.199", "@vx/shape@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/shape/-/shape-0.0.199.tgz#b25cc7c0b5a02e72d5f3ce95484e023d2e280fde"
-  integrity sha512-Nuod/HGbrs4n5BFKs7gp2YUmhbivkxXOfGRR4yZd72+l5nEdvTB/fj/69YkF6O2xvtbiJr2fKPJWe4wTZZQV5A==
+"@visx/shape@3.2.0", "@visx/shape@^3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/shape/-/shape-3.2.0.tgz#2ddc9bbb17bcea32835fea6d2a09b20a3d119791"
+  integrity sha512-NJw5lWokh6tvViOtj5eiZKJuqlE7QPhJUTi0CiBI5h3vH0h84PGxqvdH+PNYr+Hze4U+qKuppDmjFkFx/8MjTw==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/d3-path" "^1.0.8"
     "@types/d3-shape" "^1.3.1"
-    "@types/lodash" "^4.14.146"
+    "@types/lodash" "^4.14.172"
     "@types/react" "*"
-    "@vx/curve" "0.0.199"
-    "@vx/group" "0.0.199"
-    "@vx/scale" "0.0.199"
-    classnames "^2.2.5"
-    d3-path "^1.0.5"
-    d3-shape "^1.2.0"
-    lodash "^4.17.15"
-    prop-types "^15.5.10"
-
-"@vx/shape@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/shape/-/shape-0.0.165.tgz#3424121f45d83dd8e7ba62b6adf8b0aea2fb9cf8"
-  integrity sha512-D9naH/glDtw8J8IcdumpRz1ihaoCAYMwFNh2KTv73HiTKrLQSXvIjwYFv9C0b8BCPNOXkDZS8s+AlgMSqGlZNQ==
-  dependencies:
-    "@vx/curve" "0.0.165"
-    "@vx/group" "0.0.165"
-    "@vx/point" "0.0.165"
-    classnames "^2.2.5"
-    d3-path "^1.0.5"
-    d3-shape "^1.2.0"
-    prop-types "^15.5.10"
-
-"@vx/shape@^0.0.168":
-  version "0.0.168"
-  resolved "https://registry.yarnpkg.com/@vx/shape/-/shape-0.0.168.tgz#172bc1cf4dade47076018efd559e0ecc4e959aec"
-  integrity sha512-urKZkwSafMpPQ0wI/L5FJmufRiAR4UsgYUCKxROjfE1Cf4jWNlK6mlVIIASxCdHlh9CGBbIrRMdl5Yv5lzqhjA==
-  dependencies:
-    "@vx/curve" "0.0.165"
-    "@vx/group" "0.0.165"
-    "@vx/point" "0.0.165"
-    classnames "^2.2.5"
+    "@visx/curve" "3.0.0"
+    "@visx/group" "3.0.0"
+    "@visx/scale" "3.2.0"
+    classnames "^2.3.1"
     d3-path "^1.0.5"
     d3-shape "^1.2.0"
+    lodash "^4.17.21"
     prop-types "^15.5.10"
 
-"@vx/stats@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/stats/-/stats-0.0.165.tgz#32413a144383367e51628bbea61b3b0ce980f1fd"
-  integrity sha512-FRW5N+7pXLZrQxT8JA8OH28PGKq7YfiycmnSG7jzXOnvw+sPm9MRKCoyRDTpFrCiggcOhHhvqhE8RiO2qF7d3Q==
-  dependencies:
-    "@vx/group" "0.0.165"
-    "@vx/scale" "0.0.165"
-    classnames "^2.2.5"
-    d3-shape "^1.2.0"
-
-"@vx/text@0.0.175":
-  version "0.0.175"
-  resolved "https://registry.yarnpkg.com/@vx/text/-/text-0.0.175.tgz#70ff63b01abfc148132d36ca7cdebfb040bb48c5"
-  integrity sha512-SOBhctXXAGhhpCOiTjxOM/8NDaDqGRk3OGfsJ714Mt1UJX6VQaKxFocZJwn6IMw3mNG6/p7O4Eao/gGDcoM6+A==
-  dependencies:
-    babel-plugin-lodash "^3.3.2"
-    classnames "^2.2.5"
-    lodash "^4.17.4"
-    reduce-css-calc "^1.3.0"
-
-"@vx/text@0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/text/-/text-0.0.199.tgz#5d4fd172a7e1f5cc1d8ad6c175d816d6e9aa654f"
-  integrity sha512-YUIkATaN+GEyiyfMsMN3VqopmOHFjPZe2AtXPnIhjrLfOY/SjpPqqT7L2S3pgGbqZYiJbZd9VzGxXYXRtGZOzw==
+"@visx/text@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/text/-/text-3.0.0.tgz#9099c3605027b9ab4c54bde97518a648136c3629"
+  integrity sha512-LW6v5T/gpd9RGw83/ScXncYc6IlcfzXTpaN8WbbxLRI65gdvSqrykwAMR0cbpQmzoVFuZXljqOf0QslHGnBg1w==
   dependencies:
-    "@types/classnames" "^2.2.9"
-    "@types/lodash" "^4.14.160"
+    "@types/lodash" "^4.14.172"
     "@types/react" "*"
-    classnames "^2.2.5"
-    lodash "^4.17.20"
-    prop-types "^15.7.2"
-    reduce-css-calc "^1.3.0"
-
-"@vx/text@^0.0.192":
-  version "0.0.192"
-  resolved "https://registry.yarnpkg.com/@vx/text/-/text-0.0.192.tgz#cb71261e003c9d951d242ec43049cff2c4509302"
-  integrity sha512-lyy7eXfmQ8SJF7Qx+bCRcaEgvVSa18Lp6eRMo3GMANumUh9kSe7LwgqRFSdBJ85WkPqX+UOkJVyCH7AOlt0IWA==
-  dependencies:
-    classnames "^2.2.5"
-    lodash "^4.17.15"
+    classnames "^2.3.1"
+    lodash "^4.17.21"
     prop-types "^15.7.2"
     reduce-css-calc "^1.3.0"
 
-"@vx/threshold@0.0.170":
-  version "0.0.170"
-  resolved "https://registry.yarnpkg.com/@vx/threshold/-/threshold-0.0.170.tgz#59baeabc7e1687042cf4955130fc7a72704030f7"
-  integrity sha512-A3yWJrFqckbleXg3Q3iSsU6mdtHbMxEnE4jGZd8og4m9r2RDVTvFVP6ZRo4vunlfWj5YuMnNsKhx4ZSWKVMtXg==
-  dependencies:
-    "@vx/clip-path" "0.0.165"
-    "@vx/shape" "0.0.170"
-    classnames "^2.2.5"
-    prop-types "^15.5.10"
-
-"@vx/tooltip@0.0.165", "@vx/tooltip@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/tooltip/-/tooltip-0.0.165.tgz#0d17a1b445a7bc70d7840e36593b780a6e7b40e2"
-  integrity sha512-/x1NZc67QGQ4e/WNT7Ks5LYRyeLSqp8lG04gX5J6leUS0zscAVzo3aE5u65Qqbc0cnMyMPRZ2Qtb4klWTLg+eQ==
-  dependencies:
-    "@vx/bounds" "0.0.165"
-    classnames "^2.2.5"
-    prop-types "^15.5.10"
-
-"@vx/tooltip@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/tooltip/-/tooltip-0.0.199.tgz#e60147412eb5be4d8dc98664abaee1ac63ee6570"
-  integrity sha512-IZ6ExnEyTZ4jq9Ml+XECnrtQ52SnLi8WD1dbxnVMoAzNbMKNfnkET5ZqzYM/fM/yPEsKMwNadZWizdKBMjIGVw==
+"@visx/tooltip@3.1.2":
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/@visx/tooltip/-/tooltip-3.1.2.tgz#6c7bb36a296f4501adb99b59487412e39fe06f44"
+  integrity sha512-p46qztGRNkEDbxzc3V1virahvz3UQ29TzddUjA0oaTIBCrOd9UJuLvv1Tq9OpeUYPdbrO/ZRwaEeri2pbwv04Q==
   dependencies:
-    "@types/classnames" "^2.2.9"
     "@types/react" "*"
-    "@vx/bounds" "0.0.199"
-    classnames "^2.2.5"
+    "@visx/bounds" "3.0.0"
+    classnames "^2.3.1"
     prop-types "^15.5.10"
-    react-use-measure "2.0.1"
+    react-use-measure "^2.0.4"
 
-"@vx/voronoi@^0.0.165":
-  version "0.0.165"
-  resolved "https://registry.yarnpkg.com/@vx/voronoi/-/voronoi-0.0.165.tgz#11ab585199b0dccf403544a6ad378a505bfb913b"
-  integrity sha512-oZT9KBAjDLCEcOrrqW01TPz8pLtrNNAFPa7mB9ignXvgntqEd3yVXCBkxXScfZLS+O8UQc+7/pawu0PPkE2eMw==
+"@visx/vendor@3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/vendor/-/vendor-3.2.0.tgz#05f1ab97ced594d456abe60d2136b298c08d6f1b"
+  integrity sha512-QBvachNdlwnz5aimPiXJwErGaGL/FjP/lzXT1uoHuGQQeeSxvodUd6tuSt/sxaoVFzqup557qJdVLqxSCQYs5A==
+  dependencies:
+    "@types/d3-array" "3.0.3"
+    "@types/d3-color" "3.1.0"
+    "@types/d3-format" "3.0.1"
+    "@types/d3-interpolate" "3.0.1"
+    "@types/d3-scale" "4.0.2"
+    "@types/d3-time" "3.0.0"
+    "@types/d3-time-format" "2.1.0"
+    d3-array "3.2.1"
+    d3-color "3.1.0"
+    d3-format "3.1.0"
+    d3-interpolate "3.0.1"
+    d3-scale "4.0.2"
+    d3-time "3.1.0"
+    d3-time-format "4.1.0"
+    internmap "2.0.3"
+
+"@visx/voronoi@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@visx/voronoi/-/voronoi-3.0.0.tgz#24a3513252dde1d6e85b88144e7a65b6d5936275"
+  integrity sha512-ySX7+Ic+irfgZQMij/0RJnryETonuKDWA3Upw3V6YtIiodPOEQ5w8FW8TvEbhaBlAUfSwQtHJ5ECvv3ZDrJa2A==
   dependencies:
-    "@vx/group" "0.0.165"
-    classnames "^2.2.5"
+    "@types/d3-voronoi" "^1.1.9"
+    "@types/react" "*"
+    classnames "^2.3.1"
     d3-voronoi "^1.1.2"
     prop-types "^15.6.1"
 
-"@vx/zoom@^0.0.199":
-  version "0.0.199"
-  resolved "https://registry.yarnpkg.com/@vx/zoom/-/zoom-0.0.199.tgz#e5c500fd6d3bfe2c2e01df0eca93ac1069dda083"
-  integrity sha512-1naGZJ+UYynQENK1NqGP34i1r30Crrl5GSvvx7NaDXTpql7mowzy827UJdMXKBwgBh5ca5cONWOdPGh68SqqiA==
+"@visx/xychart@^3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@visx/xychart/-/xychart-3.2.0.tgz#20ecc4a7452bc68215eda97047193aeda79e579b"
+  integrity sha512-7OdPUeYJkKPlqZqpt4cc6qcpE6edE35rcAvvWmdurfCEpTyYOpUaIMgM4t0NX6mNHoAx6K6XAeS5ROCE7vpCNg==
   dependencies:
+    "@types/lodash" "^4.14.172"
     "@types/react" "*"
-    "@vx/event" "0.0.199"
+    "@visx/annotation" "3.0.1"
+    "@visx/axis" "3.2.0"
+    "@visx/event" "3.0.1"
+    "@visx/glyph" "3.0.0"
+    "@visx/grid" "3.2.0"
+    "@visx/react-spring" "3.2.0"
+    "@visx/responsive" "3.0.0"
+    "@visx/scale" "3.2.0"
+    "@visx/shape" "3.2.0"
+    "@visx/text" "3.0.0"
+    "@visx/tooltip" "3.1.2"
+    "@visx/vendor" "3.2.0"
+    "@visx/voronoi" "3.0.0"
+    classnames "^2.3.1"
+    d3-interpolate-path "2.2.1"
+    d3-shape "^2.0.0"
+    lodash "^4.17.21"
+    mitt "^2.1.0"
     prop-types "^15.6.2"
 
-"@webassemblyjs/ast@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.9.0.tgz#bd850604b4042459a5a41cd7d338cbed695ed964"
-  integrity sha512-C6wW5L+b7ogSDVqymbkkvuW9kruN//YisMED04xzeBBqjHa2FYnmvOlS6Xj68xWQRgWvI9cIglsjFowH/RJyEA==
+"@visx/zoom@^3.1.1":
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/@visx/zoom/-/zoom-3.1.1.tgz#24b5e966e95c0f71842668337b66bda0eb7ebf0f"
+  integrity sha512-/jwCAx3oHweZiF+VmdSz6kf4OnCnZI5NTP5i4vImvstt6M/8mWVZv5ICwOooZOWmceBOOwLYgEO35tJN9E34Cg==
   dependencies:
-    "@webassemblyjs/helper-module-context" "1.9.0"
-    "@webassemblyjs/helper-wasm-bytecode" "1.9.0"
-    "@webassemblyjs/wast-parser" "1.9.0"
-
-"@webassemblyjs/floating-point-hex-parser@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.9.0.tgz#3c3d3b271bddfc84deb00f71344438311d52ffb4"
-  integrity sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==
+    "@types/react" "*"
+    "@use-gesture/react" "^10.0.0-beta.22"
+    "@visx/event" "3.0.1"
+    prop-types "^15.6.2"
 
-"@webassemblyjs/helper-api-error@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.9.0.tgz#203f676e333b96c9da2eeab3ccef33c45928b6a2"
-  integrity sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==
+"@webassemblyjs/ast@1.11.6", "@webassemblyjs/ast@^1.11.5":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.6.tgz#db046555d3c413f8966ca50a95176a0e2c642e24"
+  integrity sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==
+  dependencies:
+    "@webassemblyjs/helper-numbers" "1.11.6"
+    "@webassemblyjs/helper-wasm-bytecode" "1.11.6"
 
-"@webassemblyjs/helper-buffer@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.9.0.tgz#a1442d269c5feb23fcbc9ef759dac3547f29de00"
-  integrity sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==
+"@webassemblyjs/floating-point-hex-parser@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz#dacbcb95aff135c8260f77fa3b4c5fea600a6431"
+  integrity sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==
 
-"@webassemblyjs/helper-code-frame@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.9.0.tgz#647f8892cd2043a82ac0c8c5e75c36f1d9159f27"
-  integrity sha512-ERCYdJBkD9Vu4vtjUYe8LZruWuNIToYq/ME22igL+2vj2dQ2OOujIZr3MEFvfEaqKoVqpsFKAGsRdBSBjrIvZA==
-  dependencies:
-    "@webassemblyjs/wast-printer" "1.9.0"
+"@webassemblyjs/helper-api-error@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz#6132f68c4acd59dcd141c44b18cbebbd9f2fa768"
+  integrity sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==
 
-"@webassemblyjs/helper-fsm@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-fsm/-/helper-fsm-1.9.0.tgz#c05256b71244214671f4b08ec108ad63b70eddb8"
-  integrity sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==
+"@webassemblyjs/helper-buffer@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.6.tgz#b66d73c43e296fd5e88006f18524feb0f2c7c093"
+  integrity sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA==
 
-"@webassemblyjs/helper-module-context@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-module-context/-/helper-module-context-1.9.0.tgz#25d8884b76839871a08a6c6f806c3979ef712f07"
-  integrity sha512-MJCW8iGC08tMk2enck1aPW+BE5Cw8/7ph/VGZxwyvGbJwjktKkDK7vy7gAmMDx88D7mhDTCNKAW5tED+gZ0W8g==
+"@webassemblyjs/helper-numbers@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz#cbce5e7e0c1bd32cf4905ae444ef64cea919f1b5"
+  integrity sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==
   dependencies:
-    "@webassemblyjs/ast" "1.9.0"
+    "@webassemblyjs/floating-point-hex-parser" "1.11.6"
+    "@webassemblyjs/helper-api-error" "1.11.6"
+    "@xtuc/long" "4.2.2"
 
-"@webassemblyjs/helper-wasm-bytecode@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.9.0.tgz#4fed8beac9b8c14f8c58b70d124d549dd1fe5790"
-  integrity sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==
+"@webassemblyjs/helper-wasm-bytecode@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz#bb2ebdb3b83aa26d9baad4c46d4315283acd51e9"
+  integrity sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==
 
-"@webassemblyjs/helper-wasm-section@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.9.0.tgz#5a4138d5a6292ba18b04c5ae49717e4167965346"
-  integrity sha512-XnMB8l3ek4tvrKUUku+IVaXNHz2YsJyOOmz+MMkZvh8h1uSJpSen6vYnw3IoQ7WwEuAhL8Efjms1ZWjqh2agvw==
+"@webassemblyjs/helper-wasm-section@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.6.tgz#ff97f3863c55ee7f580fd5c41a381e9def4aa577"
+  integrity sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g==
   dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/helper-buffer" "1.9.0"
-    "@webassemblyjs/helper-wasm-bytecode" "1.9.0"
-    "@webassemblyjs/wasm-gen" "1.9.0"
+    "@webassemblyjs/ast" "1.11.6"
+    "@webassemblyjs/helper-buffer" "1.11.6"
+    "@webassemblyjs/helper-wasm-bytecode" "1.11.6"
+    "@webassemblyjs/wasm-gen" "1.11.6"
 
-"@webassemblyjs/ieee754@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.9.0.tgz#15c7a0fbaae83fb26143bbacf6d6df1702ad39e4"
-  integrity sha512-dcX8JuYU/gvymzIHc9DgxTzUUTLexWwt8uCTWP3otys596io0L5aW02Gb1RjYpx2+0Jus1h4ZFqjla7umFniTg==
+"@webassemblyjs/ieee754@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz#bb665c91d0b14fffceb0e38298c329af043c6e3a"
+  integrity sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==
   dependencies:
     "@xtuc/ieee754" "^1.2.0"
 
-"@webassemblyjs/leb128@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.9.0.tgz#f19ca0b76a6dc55623a09cffa769e838fa1e1c95"
-  integrity sha512-ENVzM5VwV1ojs9jam6vPys97B/S65YQtv/aanqnU7D8aSoHFX8GyhGg0CMfyKNIHBuAVjy3tlzd5QMMINa7wpw==
+"@webassemblyjs/leb128@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.6.tgz#70e60e5e82f9ac81118bc25381a0b283893240d7"
+  integrity sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==
   dependencies:
     "@xtuc/long" "4.2.2"
 
-"@webassemblyjs/utf8@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.9.0.tgz#04d33b636f78e6a6813227e82402f7637b6229ab"
-  integrity sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==
+"@webassemblyjs/utf8@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.6.tgz#90f8bc34c561595fe156603be7253cdbcd0fab5a"
+  integrity sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==
 
-"@webassemblyjs/wasm-edit@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.9.0.tgz#3fe6d79d3f0f922183aa86002c42dd256cfee9cf"
-  integrity sha512-FgHzBm80uwz5M8WKnMTn6j/sVbqilPdQXTWraSjBwFXSYGirpkSWE2R9Qvz9tNiTKQvoKILpCuTjBKzOIm0nxw==
-  dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/helper-buffer" "1.9.0"
-    "@webassemblyjs/helper-wasm-bytecode" "1.9.0"
-    "@webassemblyjs/helper-wasm-section" "1.9.0"
-    "@webassemblyjs/wasm-gen" "1.9.0"
-    "@webassemblyjs/wasm-opt" "1.9.0"
-    "@webassemblyjs/wasm-parser" "1.9.0"
-    "@webassemblyjs/wast-printer" "1.9.0"
-
-"@webassemblyjs/wasm-gen@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.9.0.tgz#50bc70ec68ded8e2763b01a1418bf43491a7a49c"
-  integrity sha512-cPE3o44YzOOHvlsb4+E9qSqjc9Qf9Na1OO/BHFy4OI91XDE14MjFN4lTMezzaIWdPqHnsTodGGNP+iRSYfGkjA==
+"@webassemblyjs/wasm-edit@^1.11.5":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.6.tgz#c72fa8220524c9b416249f3d94c2958dfe70ceab"
+  integrity sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw==
+  dependencies:
+    "@webassemblyjs/ast" "1.11.6"
+    "@webassemblyjs/helper-buffer" "1.11.6"
+    "@webassemblyjs/helper-wasm-bytecode" "1.11.6"
+    "@webassemblyjs/helper-wasm-section" "1.11.6"
+    "@webassemblyjs/wasm-gen" "1.11.6"
+    "@webassemblyjs/wasm-opt" "1.11.6"
+    "@webassemblyjs/wasm-parser" "1.11.6"
+    "@webassemblyjs/wast-printer" "1.11.6"
+
+"@webassemblyjs/wasm-gen@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.6.tgz#fb5283e0e8b4551cc4e9c3c0d7184a65faf7c268"
+  integrity sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA==
   dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/helper-wasm-bytecode" "1.9.0"
-    "@webassemblyjs/ieee754" "1.9.0"
-    "@webassemblyjs/leb128" "1.9.0"
-    "@webassemblyjs/utf8" "1.9.0"
+    "@webassemblyjs/ast" "1.11.6"
+    "@webassemblyjs/helper-wasm-bytecode" "1.11.6"
+    "@webassemblyjs/ieee754" "1.11.6"
+    "@webassemblyjs/leb128" "1.11.6"
+    "@webassemblyjs/utf8" "1.11.6"
 
-"@webassemblyjs/wasm-opt@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.9.0.tgz#2211181e5b31326443cc8112eb9f0b9028721a61"
-  integrity sha512-Qkjgm6Anhm+OMbIL0iokO7meajkzQD71ioelnfPEj6r4eOFuqm4YC3VBPqXjFyyNwowzbMD+hizmprP/Fwkl2A==
+"@webassemblyjs/wasm-opt@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.6.tgz#d9a22d651248422ca498b09aa3232a81041487c2"
+  integrity sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g==
   dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/helper-buffer" "1.9.0"
-    "@webassemblyjs/wasm-gen" "1.9.0"
-    "@webassemblyjs/wasm-parser" "1.9.0"
+    "@webassemblyjs/ast" "1.11.6"
+    "@webassemblyjs/helper-buffer" "1.11.6"
+    "@webassemblyjs/wasm-gen" "1.11.6"
+    "@webassemblyjs/wasm-parser" "1.11.6"
 
-"@webassemblyjs/wasm-parser@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.9.0.tgz#9d48e44826df4a6598294aa6c87469d642fff65e"
-  integrity sha512-9+wkMowR2AmdSWQzsPEjFU7njh8HTO5MqO8vjwEHuM+AMHioNqSBONRdr0NQQ3dVQrzp0s8lTcYqzUdb7YgELA==
+"@webassemblyjs/wasm-parser@1.11.6", "@webassemblyjs/wasm-parser@^1.11.5":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.6.tgz#bb85378c527df824004812bbdb784eea539174a1"
+  integrity sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ==
   dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/helper-api-error" "1.9.0"
-    "@webassemblyjs/helper-wasm-bytecode" "1.9.0"
-    "@webassemblyjs/ieee754" "1.9.0"
-    "@webassemblyjs/leb128" "1.9.0"
-    "@webassemblyjs/utf8" "1.9.0"
+    "@webassemblyjs/ast" "1.11.6"
+    "@webassemblyjs/helper-api-error" "1.11.6"
+    "@webassemblyjs/helper-wasm-bytecode" "1.11.6"
+    "@webassemblyjs/ieee754" "1.11.6"
+    "@webassemblyjs/leb128" "1.11.6"
+    "@webassemblyjs/utf8" "1.11.6"
 
-"@webassemblyjs/wast-parser@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-parser/-/wast-parser-1.9.0.tgz#3031115d79ac5bd261556cecc3fa90a3ef451914"
-  integrity sha512-qsqSAP3QQ3LyZjNC/0jBJ/ToSxfYJ8kYyuiGvtn/8MK89VrNEfwj7BPQzJVHi0jGTRK2dGdJ5PRqhtjzoww+bw==
-  dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/floating-point-hex-parser" "1.9.0"
-    "@webassemblyjs/helper-api-error" "1.9.0"
-    "@webassemblyjs/helper-code-frame" "1.9.0"
-    "@webassemblyjs/helper-fsm" "1.9.0"
-    "@xtuc/long" "4.2.2"
-
-"@webassemblyjs/wast-printer@1.9.0":
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.9.0.tgz#4935d54c85fef637b00ce9f52377451d00d47899"
-  integrity sha512-2J0nE95rHXHyQ24cWjMKJ1tqB/ds8z/cyeOZxJhcb+rW+SQASVjuznUSmdz5GpVJTzU8JkhYut0D3siFDD6wsA==
+"@webassemblyjs/wast-printer@1.11.6":
+  version "1.11.6"
+  resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.6.tgz#a7bf8dd7e362aeb1668ff43f35cb849f188eff20"
+  integrity sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A==
   dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/wast-parser" "1.9.0"
+    "@webassemblyjs/ast" "1.11.6"
     "@xtuc/long" "4.2.2"
 
 "@wry/context@^0.6.0":
@@ -5672,40 +5221,45 @@ acorn-globals@^6.0.0:
     acorn "^7.1.1"
     acorn-walk "^7.1.1"
 
-acorn-jsx@^5.3.1:
-  version "5.3.1"
-  resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b"
-  integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==
+acorn-import-assertions@^1.9.0:
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac"
+  integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==
+
+acorn-jsx@^5.3.2:
+  version "5.3.2"
+  resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937"
+  integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==
 
 acorn-walk@^7.1.1:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
   integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
 
-acorn@^6.4.1:
-  version "6.4.2"
-  resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6"
-  integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==
+acorn-walk@^8.1.1:
+  version "8.2.0"
+  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
+  integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
 
-acorn@^7.1.0, acorn@^7.1.1, acorn@^7.4.0:
+acorn@^7.1.1:
   version "7.4.1"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
   integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
 
-acorn@^8.2.4:
-  version "8.2.4"
-  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.2.4.tgz#caba24b08185c3b56e3168e97d15ed17f4d31fd0"
-  integrity sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==
+acorn@^8.2.4, acorn@^8.4.1, acorn@^8.7.1, acorn@^8.8.0, acorn@^8.8.2:
+  version "8.9.0"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.9.0.tgz#78a16e3b2bcc198c10822786fa6679e245db5b59"
+  integrity sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ==
 
-address@1.1.2, address@^1.0.1:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/address/-/address-1.1.2.tgz#bf1116c9c758c51b7a933d296b72c221ed9428b6"
-  integrity sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==
+address@^1.0.1, address@^1.1.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/address/-/address-1.2.2.tgz#2b5248dac5485a6390532c6a517fda2e3faac89e"
+  integrity sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==
 
-adjust-sourcemap-loader@3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/adjust-sourcemap-loader/-/adjust-sourcemap-loader-3.0.0.tgz#5ae12fb5b7b1c585e80bbb5a63ec163a1a45e61e"
-  integrity sha512-YBrGyT2/uVQ/c6Rr+t6ZJXniY03YtHGMJQYal368burRGYKqhx9qGTWqcBU5s1CwYY9E/ri63RYyG1IacMZtqw==
+adjust-sourcemap-loader@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/adjust-sourcemap-loader/-/adjust-sourcemap-loader-4.0.0.tgz#fc4a0fd080f7d10471f30a7320f25560ade28c99"
+  integrity sha512-OXwN5b9pCUXNQHJpwwD2qP40byEmSgzj8B4ydSN0uMNYWiFmJ6x6KwUllMmfk8Rwu/HJDFR7U8ubsWBoN0Xp0A==
   dependencies:
     loader-utils "^2.0.0"
     regex-parser "^2.2.11"
@@ -5717,25 +5271,26 @@ agent-base@6:
   dependencies:
     debug "4"
 
-aggregate-error@^3.0.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a"
-  integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==
+ajv-formats@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520"
+  integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==
   dependencies:
-    clean-stack "^2.0.0"
-    indent-string "^4.0.0"
-
-ajv-errors@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d"
-  integrity sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==
+    ajv "^8.0.0"
 
-ajv-keywords@^3.1.0, ajv-keywords@^3.4.1, ajv-keywords@^3.5.2:
+ajv-keywords@^3.4.1, ajv-keywords@^3.5.2:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d"
   integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==
 
-ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5:
+ajv-keywords@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-5.1.0.tgz#69d4d385a4733cdbeab44964a1170a88f87f0e16"
+  integrity sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==
+  dependencies:
+    fast-deep-equal "^3.1.3"
+
+ajv@^6.10.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
   integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==
@@ -5745,21 +5300,16 @@ ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-ajv@^8.0.1:
-  version "8.5.0"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.5.0.tgz#695528274bcb5afc865446aa275484049a18ae4b"
-  integrity sha512-Y2l399Tt1AguU3BPRP9Fn4eN+Or+StUGWCUpbnFyXSo8NZ9S4uj+AG2pjs5apK+ZMOwYOz1+a+VKvKH7CudXgQ==
+ajv@^8.0.0, ajv@^8.6.0, ajv@^8.9.0:
+  version "8.12.0"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1"
+  integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==
   dependencies:
     fast-deep-equal "^3.1.1"
     json-schema-traverse "^1.0.0"
     require-from-string "^2.0.2"
     uri-js "^4.2.2"
 
-alphanum-sort@^1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3"
-  integrity sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM=
-
 analytics-utils@^1.0.12:
   version "1.0.12"
   resolved "https://registry.yarnpkg.com/analytics-utils/-/analytics-utils-1.0.12.tgz#07bd63471d238e80f42d557fba039365f09c50db"
@@ -5776,16 +5326,6 @@ analytics@^0.8.9:
     "@analytics/core" "^0.12.7"
     "@analytics/storage-utils" "^0.4.2"
 
-ansi-colors@^3.0.0:
-  version "3.2.4"
-  resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-3.2.4.tgz#e3a3da4bfbae6c86a9c285625de124a234026fbf"
-  integrity sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA==
-
-ansi-colors@^4.1.1:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348"
-  integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==
-
 ansi-escapes@^3.0.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b"
@@ -5798,10 +5338,10 @@ ansi-escapes@^4.2.1, ansi-escapes@^4.3.1:
   dependencies:
     type-fest "^0.21.3"
 
-ansi-html@0.0.7, ansi-html@^0.0.7:
-  version "0.0.7"
-  resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e"
-  integrity sha1-gTWEAhliqenm/QOflA0S9WynhZ4=
+ansi-html-community@^0.0.8:
+  version "0.0.8"
+  resolved "https://registry.yarnpkg.com/ansi-html-community/-/ansi-html-community-0.0.8.tgz#69fbc4d6ccbe383f9736934ae34c3f8290f1bf41"
+  integrity sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==
 
 ansi-regex@^2.0.0:
   version "2.1.1"
@@ -5813,22 +5353,22 @@ ansi-regex@^3.0.0:
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998"
   integrity sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=
 
-ansi-regex@^4.1.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997"
-  integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==
+ansi-regex@^5.0.0, ansi-regex@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
+  integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
 
-ansi-regex@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
-  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
+ansi-regex@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.0.1.tgz#3183e38fae9a65d7cb5e53945cd5897d0260a06a"
+  integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==
 
 ansi-styles@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
   integrity sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=
 
-ansi-styles@^3.2.0, ansi-styles@^3.2.1:
+ansi-styles@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d"
   integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==
@@ -5842,6 +5382,11 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0:
   dependencies:
     color-convert "^2.0.1"
 
+ansi-styles@^5.0.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b"
+  integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
+
 antd@4.24.7:
   version "4.24.7"
   resolved "https://registry.yarnpkg.com/antd/-/antd-4.24.7.tgz#ad90cc2d6225fe3e0030aeccdc64de6c26edc3e7"
@@ -5896,32 +5441,29 @@ any-observable@^0.3.0:
   resolved "https://registry.yarnpkg.com/any-observable/-/any-observable-0.3.0.tgz#af933475e5806a67d0d7df090dd5e8bef65d119b"
   integrity sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog==
 
-anymatch@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb"
-  integrity sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==
-  dependencies:
-    micromatch "^3.1.4"
-    normalize-path "^2.1.1"
+any-promise@^1.0.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
+  integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==
 
-anymatch@^3.0.0, anymatch@^3.0.3, anymatch@~3.1.1:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716"
-  integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==
+anymatch@^3.0.3, anymatch@~3.1.2:
+  version "3.1.3"
+  resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e"
+  integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==
   dependencies:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
-aproba@^1.1.1:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
-  integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==
-
 arg@^4.1.0:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
   integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
 
+arg@^5.0.2:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
+  integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
+
 argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -5942,10 +5484,12 @@ aria-query@^4.2.2:
     "@babel/runtime" "^7.10.2"
     "@babel/runtime-corejs3" "^7.10.2"
 
-arity-n@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/arity-n/-/arity-n-1.0.4.tgz#d9e76b11733e08569c0847ae7b39b2860b30b745"
-  integrity sha1-2edrEXM+CFacCEeuezmyhgswt0U=
+aria-query@^5.1.3:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.3.0.tgz#650c569e41ad90b51b3d7df5e5eed1c7549c103e"
+  integrity sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==
+  dependencies:
+    dequal "^2.0.3"
 
 arr-diff@^4.0.0:
   version "4.0.0"
@@ -5967,88 +5511,73 @@ array-flatten@1.1.1:
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
   integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==
 
-array-flatten@^2.1.0:
+array-flatten@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-2.1.2.tgz#24ef80a28c1a893617e2149b0c6d0d788293b099"
   integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==
 
-array-includes@^3.1.1, array-includes@^3.1.2, array-includes@^3.1.3:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.3.tgz#c7f619b382ad2afaf5326cddfdc0afc61af7690a"
-  integrity sha512-gcem1KlBU7c9rB+Rq8/3PPKsK2kjqeEBa3bD5kkQo4nYlOHQCJqIJFqBXDEfwaRuYTT4E+FxA9xez7Gf/e3Q7A==
+array-includes@^3.1.5, array-includes@^3.1.6:
+  version "3.1.6"
+  resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.6.tgz#9e9e720e194f198266ba9e18c29e6a9b0e4b225f"
+  integrity sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==
   dependencies:
     call-bind "^1.0.2"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.0-next.2"
-    get-intrinsic "^1.1.1"
-    is-string "^1.0.5"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
+    get-intrinsic "^1.1.3"
+    is-string "^1.0.7"
 
 array-tree-filter@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/array-tree-filter/-/array-tree-filter-2.1.0.tgz#873ac00fec83749f255ac8dd083814b4f6329190"
   integrity sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==
 
-array-union@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39"
-  integrity sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=
-  dependencies:
-    array-uniq "^1.0.1"
-
 array-union@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d"
   integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==
 
-array-uniq@^1.0.1:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6"
-  integrity sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=
-
 array-unique@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
   integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=
 
-array.prototype.flat@^1.2.4:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz#6ef638b43312bd401b4c6199fdec7e2dc9e9a123"
-  integrity sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg==
+array.prototype.flat@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz#ffc6576a7ca3efc2f46a143b9d1dda9b4b3cf5e2"
+  integrity sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==
   dependencies:
-    call-bind "^1.0.0"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.0-next.1"
+    call-bind "^1.0.2"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
+    es-shim-unscopables "^1.0.0"
 
-array.prototype.flatmap@^1.2.4:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.2.4.tgz#94cfd47cc1556ec0747d97f7c7738c58122004c9"
-  integrity sha512-r9Z0zYoxqHz60vvQbWEdXIEtCwHF0yxaWfno9qzXeNHvfyl3BZqygmGzb84dsubyaXLH4husF+NFgMSdpZhk2Q==
+array.prototype.flatmap@^1.2.4, array.prototype.flatmap@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz#1aae7903c2100433cb8261cd4ed310aab5c4a183"
+  integrity sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==
   dependencies:
-    call-bind "^1.0.0"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.0-next.1"
-    function-bind "^1.1.1"
+    call-bind "^1.0.2"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
+    es-shim-unscopables "^1.0.0"
 
-arrify@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
-  integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
+array.prototype.tosorted@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/array.prototype.tosorted/-/array.prototype.tosorted-1.1.1.tgz#ccf44738aa2b5ac56578ffda97c03fd3e23dd532"
+  integrity sha512-pZYPXPRl2PqWcsUs6LOMn+1f1532nEoPTYowBtqLwAW+W8vSVhkIGnmOX1t/UQjD6YGI0vcD2B1U7ZFGQH9jnQ==
+  dependencies:
+    call-bind "^1.0.2"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
+    es-shim-unscopables "^1.0.0"
+    get-intrinsic "^1.1.3"
 
 asap@~2.0.3, asap@~2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
   integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=
 
-asn1.js@^5.2.0:
-  version "5.4.1"
-  resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07"
-  integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==
-  dependencies:
-    bn.js "^4.0.0"
-    inherits "^2.0.1"
-    minimalistic-assert "^1.0.0"
-    safer-buffer "^2.1.0"
-
 asn1@~0.2.3:
   version "0.2.4"
   resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
@@ -6061,14 +5590,6 @@ assert-plus@1.0.0, assert-plus@^1.0.0:
   resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525"
   integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=
 
-assert@^1.1.1:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/assert/-/assert-1.5.0.tgz#55c109aaf6e0aefdb3dc4b71240c70bf574b18eb"
-  integrity sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA==
-  dependencies:
-    object-assign "^4.1.1"
-    util "0.10.3"
-
 assign-symbols@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367"
@@ -6079,16 +5600,6 @@ ast-types-flow@^0.0.7:
   resolved "https://registry.yarnpkg.com/ast-types-flow/-/ast-types-flow-0.0.7.tgz#f70b735c6bca1a5c9c22d982c3e39e7feba3bdad"
   integrity sha1-9wtzXGvKGlycItmCw+Oef+ujva0=
 
-astral-regex@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31"
-  integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==
-
-async-each@^1.0.1:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf"
-  integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==
-
 async-limiter@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
@@ -6099,17 +5610,10 @@ async-validator@^4.1.0:
   resolved "https://registry.yarnpkg.com/async-validator/-/async-validator-4.2.5.tgz#c96ea3332a521699d0afaaceed510a54656c6339"
   integrity sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==
 
-async@0.9.x:
-  version "0.9.2"
-  resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"
-  integrity sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=
-
-async@^2.6.2:
-  version "2.6.3"
-  resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff"
-  integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==
-  dependencies:
-    lodash "^4.17.14"
+async@^3.2.3:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c"
+  integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==
 
 asynckit@^0.4.0:
   version "0.4.0"
@@ -6131,18 +5635,17 @@ auto-bind@~4.0.0:
   resolved "https://registry.yarnpkg.com/auto-bind/-/auto-bind-4.0.0.tgz#e3589fc6c2da8f7ca43ba9f84fa52a744fc997fb"
   integrity sha512-Hdw8qdNiqdJ8LqT0iK0sVzkFbzg6fhnQqqfWhBDxcHZvU75+B+ayzTy8x+k5Ix0Y92XOhOUlx74ps+bA6BeYMQ==
 
-autoprefixer@^9.6.1:
-  version "9.8.6"
-  resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.6.tgz#3b73594ca1bf9266320c5acf1588d74dea74210f"
-  integrity sha512-XrvP4VVHdRBCdX1S3WXVD8+RyG9qeb1D5Sn1DeLiG2xfSpzellk5k54xbUERJ3M5DggQxes39UGOTP8CFrEGbg==
+autoprefixer@^10.4.12, autoprefixer@^10.4.13:
+  version "10.4.14"
+  resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-10.4.14.tgz#e28d49902f8e759dd25b153264e862df2705f79d"
+  integrity sha512-FQzyfOsTlwVzjHxKEqRIAdJx9niO6VCBCoEwax/VLSoQF29ggECcPuBqUMZ+u8jCZOPSy8b8/8KnuFbp0SaFZQ==
   dependencies:
-    browserslist "^4.12.0"
-    caniuse-lite "^1.0.30001109"
-    colorette "^1.2.1"
+    browserslist "^4.21.5"
+    caniuse-lite "^1.0.30001464"
+    fraction.js "^4.2.0"
     normalize-range "^0.1.2"
-    num2fraction "^1.2.2"
-    postcss "^7.0.32"
-    postcss-value-parser "^4.1.0"
+    picocolors "^1.0.0"
+    postcss-value-parser "^4.2.0"
 
 aws-sign2@~0.7.0:
   version "0.7.0"
@@ -6154,10 +5657,10 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
   integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
 
-axe-core@^4.0.2:
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.2.1.tgz#2e50bcf10ee5b819014f6e342e41e45096239e34"
-  integrity sha512-evY7DN8qSIbsW2H/TWQ1bX3sXN1d4MNb5Vb4n7BzPuCwRHdkZ1H2eNLuSh73EoQqkGKUtju2G2HCcjCfhvZIAA==
+axe-core@^4.6.2:
+  version "4.7.2"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.2.tgz#040a7342b20765cb18bb50b628394c21bccc17a0"
+  integrity sha512-zIURGIS1E1Q4pcrMjp+nnEh+16G56eG/MUllJH8yEvw7asDo7Ac9uhC9KIH5jzpITueEZolfYglnCGIuSBz39g==
 
 axios@^0.21.1:
   version "0.21.4"
@@ -6166,42 +5669,25 @@ axios@^0.21.1:
   dependencies:
     follow-redirects "^1.14.0"
 
-axobject-query@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be"
-  integrity sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==
-
-babel-eslint@^10.1.0:
-  version "10.1.0"
-  resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
-  integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==
-  dependencies:
-    "@babel/code-frame" "^7.0.0"
-    "@babel/parser" "^7.7.0"
-    "@babel/traverse" "^7.7.0"
-    "@babel/types" "^7.7.0"
-    eslint-visitor-keys "^1.0.0"
-    resolve "^1.12.0"
-
-babel-extract-comments@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/babel-extract-comments/-/babel-extract-comments-1.0.0.tgz#0a2aedf81417ed391b85e18b4614e693a0351a21"
-  integrity sha512-qWWzi4TlddohA91bFwgt6zO/J0X+io7Qp184Fw0m2JYRSTZnJbFR8+07KmzudHCZgOiKRCrjhylwv9Xd8gfhVQ==
+axobject-query@^3.1.1:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-3.2.1.tgz#39c378a6e3b06ca679f29138151e45b2b32da62a"
+  integrity sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==
   dependencies:
-    babylon "^6.18.0"
+    dequal "^2.0.3"
 
-babel-jest@^26.6.0, babel-jest@^26.6.3:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-26.6.3.tgz#d87d25cb0037577a0c89f82e5755c5d293c01056"
-  integrity sha512-pl4Q+GAVOHwvjrck6jKjvmGhnO3jHX/xuB9d27f+EJZ/6k+6nMuPjorrYp7s++bKKdANwzElBWnLWaObvTnaZA==
+babel-jest@^27.4.2, babel-jest@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444"
+  integrity sha512-cdQ5dXjGRd0IBRATiQ4mZGlGlRE8kJpjPOixdNRdT+m3UcNqmYWN6rK6nvtXYfY3D76cb8s/O1Ss8ea24PIwcg==
   dependencies:
-    "@jest/transform" "^26.6.2"
-    "@jest/types" "^26.6.2"
-    "@types/babel__core" "^7.1.7"
-    babel-plugin-istanbul "^6.0.0"
-    babel-preset-jest "^26.6.2"
+    "@jest/transform" "^27.5.1"
+    "@jest/types" "^27.5.1"
+    "@types/babel__core" "^7.1.14"
+    babel-plugin-istanbul "^6.1.1"
+    babel-preset-jest "^27.5.1"
     chalk "^4.0.0"
-    graceful-fs "^4.2.4"
+    graceful-fs "^4.2.9"
     slash "^3.0.0"
 
 babel-literal-to-ast@^2.1.0:
@@ -6213,75 +5699,45 @@ babel-literal-to-ast@^2.1.0:
     "@babel/traverse" "^7.1.6"
     "@babel/types" "^7.1.6"
 
-babel-loader@8.1.0:
-  version "8.1.0"
-  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.1.0.tgz#c611d5112bd5209abe8b9fa84c3e4da25275f1c3"
-  integrity sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==
-  dependencies:
-    find-cache-dir "^2.1.0"
-    loader-utils "^1.4.0"
-    mkdirp "^0.5.3"
-    pify "^4.0.1"
-    schema-utils "^2.6.5"
-
-babel-loader@8.2.2:
-  version "8.2.2"
-  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.2.2.tgz#9363ce84c10c9a40e6c753748e1441b60c8a0b81"
-  integrity sha512-JvTd0/D889PQBtUXJ2PXaKU/pjZDMtHA9V2ecm+eNRmmBCMR09a+fmpGTNwnJtFmFl5Ei7Vy47LjBb+L0wQ99g==
+babel-loader@^8.2.3:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.3.0.tgz#124936e841ba4fe8176786d6ff28add1f134d6a8"
+  integrity sha512-H8SvsMF+m9t15HNLMipppzkC+Y2Yq+v3SonZyU70RBL/h1gxPkH08Ot8pEE9Z4Kd+czyWJClmFS8qzIP9OZ04Q==
   dependencies:
     find-cache-dir "^3.3.1"
-    loader-utils "^1.4.0"
+    loader-utils "^2.0.0"
     make-dir "^3.1.0"
     schema-utils "^2.6.5"
 
-babel-plugin-dynamic-import-node@^2.3.3:
-  version "2.3.3"
-  resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz#84fda19c976ec5c6defef57f9427b3def66e17a3"
-  integrity sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==
-  dependencies:
-    object.assign "^4.1.0"
-
-babel-plugin-import@^1.13.1:
-  version "1.13.3"
-  resolved "https://registry.yarnpkg.com/babel-plugin-import/-/babel-plugin-import-1.13.3.tgz#9dbbba7d1ac72bd412917a830d445e00941d26d7"
-  integrity sha512-1qCWdljJOrDRH/ybaCZuDgySii4yYrtQ8OJQwrcDqdt0y67N30ng3X3nABg6j7gR7qUJgcMa9OMhc4AGViDwWw==
+babel-plugin-import@1.13.5:
+  version "1.13.5"
+  resolved "https://registry.yarnpkg.com/babel-plugin-import/-/babel-plugin-import-1.13.5.tgz#42eed1c5afd9a35ee1b1f8fe922b07c44077d753"
+  integrity sha512-IkqnoV+ov1hdJVofly9pXRJmeDm9EtROfrc5i6eII0Hix2xMs5FEm8FG3ExMvazbnZBbgHIt6qdO8And6lCloQ==
   dependencies:
     "@babel/helper-module-imports" "^7.0.0"
-    "@babel/runtime" "^7.0.0"
 
-babel-plugin-istanbul@^6.0.0:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-6.0.0.tgz#e159ccdc9af95e0b570c75b4573b7c34d671d765"
-  integrity sha512-AF55rZXpe7trmEylbaE1Gv54wn6rwU03aptvRoVIGP8YykoSxqdVLV1TfwflBCE/QtHmqtP8SWlTENqbK8GCSQ==
+babel-plugin-istanbul@^6.1.1:
+  version "6.1.1"
+  resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz#fa88ec59232fd9b4e36dbbc540a8ec9a9b47da73"
+  integrity sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==
   dependencies:
     "@babel/helper-plugin-utils" "^7.0.0"
     "@istanbuljs/load-nyc-config" "^1.0.0"
     "@istanbuljs/schema" "^0.1.2"
-    istanbul-lib-instrument "^4.0.0"
+    istanbul-lib-instrument "^5.0.4"
     test-exclude "^6.0.0"
 
-babel-plugin-jest-hoist@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-26.6.2.tgz#8185bd030348d254c6d7dd974355e6a28b21e62d"
-  integrity sha512-PO9t0697lNTmcEHH69mdtYiOIkkOlj9fySqfO3K1eCcdISevLAE0xY59VLLUj0SoiPiTX/JU2CYFpILydUa5Lw==
+babel-plugin-jest-hoist@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-27.5.1.tgz#9be98ecf28c331eb9f5df9c72d6f89deb8181c2e"
+  integrity sha512-50wCwD5EMNW4aRpOwtqzyZHIewTYNxLA4nhB+09d8BIssfNfzBRhkBIHiaPv1Si226TQSvp8gxAJm2iY2qs2hQ==
   dependencies:
     "@babel/template" "^7.3.3"
     "@babel/types" "^7.3.3"
     "@types/babel__core" "^7.0.0"
     "@types/babel__traverse" "^7.0.6"
 
-babel-plugin-lodash@^3.3.2:
-  version "3.3.4"
-  resolved "https://registry.yarnpkg.com/babel-plugin-lodash/-/babel-plugin-lodash-3.3.4.tgz#4f6844358a1340baed182adbeffa8df9967bc196"
-  integrity sha512-yDZLjK7TCkWl1gpBeBGmuaDIFhZKmkoL+Cu2MUUjv5VxUZx/z7tBGBCBcQs5RI1Bkz5LLmNdjx7paOyQtMovyg==
-  dependencies:
-    "@babel/helper-module-imports" "^7.0.0-beta.49"
-    "@babel/types" "^7.0.0-beta.49"
-    glob "^7.1.1"
-    lodash "^4.17.10"
-    require-package-name "^2.0.1"
-
-babel-plugin-macros@2.8.0, babel-plugin-macros@^2.5.0:
+babel-plugin-macros@^2.5.0:
   version "2.8.0"
   resolved "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-2.8.0.tgz#0f958a7cc6556b1e65344465d99111a1e5e10138"
   integrity sha512-SEP5kJpfGYqYKpBrj5XU3ahw5p5GOHJ0U5ssOSQ/WBVdwkD2Dzlce95exQTs3jOVWPPKLBN2rlEWkCK7dSmLvg==
@@ -6299,34 +5755,34 @@ babel-plugin-macros@^3.1.0:
     cosmiconfig "^7.0.0"
     resolve "^1.19.0"
 
-babel-plugin-named-asset-import@^0.3.7:
-  version "0.3.7"
-  resolved "https://registry.yarnpkg.com/babel-plugin-named-asset-import/-/babel-plugin-named-asset-import-0.3.7.tgz#156cd55d3f1228a5765774340937afc8398067dd"
-  integrity sha512-squySRkf+6JGnvjoUtDEjSREJEBirnXi9NqP6rjSYsylxQxqBTz+pkmf395i9E2zsvmYUaI40BHo6SqZUdydlw==
+babel-plugin-named-asset-import@^0.3.8:
+  version "0.3.8"
+  resolved "https://registry.yarnpkg.com/babel-plugin-named-asset-import/-/babel-plugin-named-asset-import-0.3.8.tgz#6b7fa43c59229685368683c28bc9734f24524cc2"
+  integrity sha512-WXiAc++qo7XcJ1ZnTYGtLxmBCVbddAml3CEXgWaBzNzLNoxtQ8AiGEFDMOhot9XjTCQbvP5E77Fj9Gk924f00Q==
 
-babel-plugin-polyfill-corejs2@^0.2.0:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.2.2.tgz#e9124785e6fd94f94b618a7954e5693053bf5327"
-  integrity sha512-kISrENsJ0z5dNPq5eRvcctITNHYXWOA4DUZRFYCz3jYCcvTb/A546LIddmoGNMVYg2U38OyFeNosQwI9ENTqIQ==
+babel-plugin-polyfill-corejs2@^0.4.3:
+  version "0.4.3"
+  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.3.tgz#75044d90ba5043a5fb559ac98496f62f3eb668fd"
+  integrity sha512-bM3gHc337Dta490gg+/AseNB9L4YLHxq1nGKZZSHbhXv4aTYU2MD2cjza1Ru4S6975YLTaL1K8uJf6ukJhhmtw==
   dependencies:
-    "@babel/compat-data" "^7.13.11"
-    "@babel/helper-define-polyfill-provider" "^0.2.2"
+    "@babel/compat-data" "^7.17.7"
+    "@babel/helper-define-polyfill-provider" "^0.4.0"
     semver "^6.1.1"
 
-babel-plugin-polyfill-corejs3@^0.2.0:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.2.2.tgz#7424a1682ee44baec817327710b1b094e5f8f7f5"
-  integrity sha512-l1Cf8PKk12eEk5QP/NQ6TH8A1pee6wWDJ96WjxrMXFLHLOBFzYM4moG80HFgduVhTqAFez4alnZKEhP/bYHg0A==
+babel-plugin-polyfill-corejs3@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.8.1.tgz#39248263c38191f0d226f928d666e6db1b4b3a8a"
+  integrity sha512-ikFrZITKg1xH6pLND8zT14UPgjKHiGLqex7rGEZCH2EvhsneJaJPemmpQaIZV5AL03II+lXylw3UmddDK8RU5Q==
   dependencies:
-    "@babel/helper-define-polyfill-provider" "^0.2.2"
-    core-js-compat "^3.9.1"
+    "@babel/helper-define-polyfill-provider" "^0.4.0"
+    core-js-compat "^3.30.1"
 
-babel-plugin-polyfill-regenerator@^0.2.0:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.2.2.tgz#b310c8d642acada348c1fa3b3e6ce0e851bee077"
-  integrity sha512-Goy5ghsc21HgPDFtzRkSirpZVW35meGoTmTOb2bxqdl60ghub4xOidgNTHaZfQ2FaxQsKmwvXtOAkcIS4SMBWg==
+babel-plugin-polyfill-regenerator@^0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.5.0.tgz#e7344d88d9ef18a3c47ded99362ae4a757609380"
+  integrity sha512-hDJtKjMLVa7Z+LwnTCxoDLQj6wdc+B8dun7ayF2fYieI6OzfuvcLMB32ihJZ4UhCBwNYGl5bg/x/P9cMdnkc2g==
   dependencies:
-    "@babel/helper-define-polyfill-provider" "^0.2.2"
+    "@babel/helper-define-polyfill-provider" "^0.4.0"
 
 "babel-plugin-styled-components@>= 1.12.0":
   version "1.12.0"
@@ -6343,25 +5799,12 @@ babel-plugin-syntax-jsx@^6.18.0:
   resolved "https://registry.yarnpkg.com/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz#0af32a9a6e13ca7a3fd5069e62d7b0f58d0d8946"
   integrity sha1-CvMqmm4Tyno/1QaeYtew9Y0NiUY=
 
-babel-plugin-syntax-object-rest-spread@^6.8.0:
-  version "6.13.0"
-  resolved "https://registry.yarnpkg.com/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz#fd6536f2bce13836ffa3a5458c4903a597bb3bf5"
-  integrity sha1-/WU28rzhODb/o6VFjEkDpZe7O/U=
-
 babel-plugin-syntax-trailing-function-commas@^7.0.0-beta.0:
   version "7.0.0-beta.0"
   resolved "https://registry.yarnpkg.com/babel-plugin-syntax-trailing-function-commas/-/babel-plugin-syntax-trailing-function-commas-7.0.0-beta.0.tgz#aa213c1435e2bffeb6fca842287ef534ad05d5cf"
   integrity sha512-Xj9XuRuz3nTSbaTXWv3itLOcxyF4oPD8douBBmj7U9BBC6nEBYfyOJYQMf/8PJAFotC62UY5dFfIGEPr7WswzQ==
 
-babel-plugin-transform-object-rest-spread@^6.26.0:
-  version "6.26.0"
-  resolved "https://registry.yarnpkg.com/babel-plugin-transform-object-rest-spread/-/babel-plugin-transform-object-rest-spread-6.26.0.tgz#0f36692d50fef6b7e2d4b3ac1478137a963b7b06"
-  integrity sha1-DzZpLVD+9rfi1LOsFHgTepY7ewY=
-  dependencies:
-    babel-plugin-syntax-object-rest-spread "^6.8.0"
-    babel-runtime "^6.26.0"
-
-babel-plugin-transform-react-remove-prop-types@0.4.24:
+babel-plugin-transform-react-remove-prop-types@^0.4.24:
   version "0.4.24"
   resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-remove-prop-types/-/babel-plugin-transform-react-remove-prop-types-0.4.24.tgz#f2edaf9b4c6a5fbe5c1d678bfb531078c1555f3a"
   integrity sha512-eqj0hVcJUR57/Ug2zE1Yswsw4LhuqqHhD+8v120T1cl3kjg76QwtyBrdIk4WVwK+lAhBJVYCd/v+4nc4y+8JsA==
@@ -6417,47 +5860,35 @@ babel-preset-fbjs@^3.4.0:
     "@babel/plugin-transform-template-literals" "^7.0.0"
     babel-plugin-syntax-trailing-function-commas "^7.0.0-beta.0"
 
-babel-preset-jest@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-26.6.2.tgz#747872b1171df032252426586881d62d31798fee"
-  integrity sha512-YvdtlVm9t3k777c5NPQIv6cxFFFapys25HiUmuSgHwIZhfifweR5c5Sf5nwE3MAbfu327CYSvps8Yx6ANLyleQ==
+babel-preset-jest@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-27.5.1.tgz#91f10f58034cb7989cb4f962b69fa6eef6a6bc81"
+  integrity sha512-Nptf2FzlPCWYuJg41HBqXVT8ym6bXOevuCTbhxlUpjwtysGaIWFvDEjp4y+G7fl13FgOdjs7P/DmErqH7da0Ag==
   dependencies:
-    babel-plugin-jest-hoist "^26.6.2"
+    babel-plugin-jest-hoist "^27.5.1"
     babel-preset-current-node-syntax "^1.0.0"
 
-babel-preset-react-app@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/babel-preset-react-app/-/babel-preset-react-app-10.0.0.tgz#689b60edc705f8a70ce87f47ab0e560a317d7045"
-  integrity sha512-itL2z8v16khpuKutx5IH8UdCdSTuzrOhRFTEdIhveZ2i1iBKDrVE0ATa4sFVy+02GLucZNVBWtoarXBy0Msdpg==
-  dependencies:
-    "@babel/core" "7.12.3"
-    "@babel/plugin-proposal-class-properties" "7.12.1"
-    "@babel/plugin-proposal-decorators" "7.12.1"
-    "@babel/plugin-proposal-nullish-coalescing-operator" "7.12.1"
-    "@babel/plugin-proposal-numeric-separator" "7.12.1"
-    "@babel/plugin-proposal-optional-chaining" "7.12.1"
-    "@babel/plugin-transform-flow-strip-types" "7.12.1"
-    "@babel/plugin-transform-react-display-name" "7.12.1"
-    "@babel/plugin-transform-runtime" "7.12.1"
-    "@babel/preset-env" "7.12.1"
-    "@babel/preset-react" "7.12.1"
-    "@babel/preset-typescript" "7.12.1"
-    "@babel/runtime" "7.12.1"
-    babel-plugin-macros "2.8.0"
-    babel-plugin-transform-react-remove-prop-types "0.4.24"
-
-babel-runtime@^6.26.0:
-  version "6.26.0"
-  resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe"
-  integrity sha1-llxwWGaOgrVde/4E/yM3vItWR/4=
-  dependencies:
-    core-js "^2.4.0"
-    regenerator-runtime "^0.11.0"
-
-babylon@^6.18.0:
-  version "6.18.0"
-  resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3"
-  integrity sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==
+babel-preset-react-app@^10.0.1:
+  version "10.0.1"
+  resolved "https://registry.yarnpkg.com/babel-preset-react-app/-/babel-preset-react-app-10.0.1.tgz#ed6005a20a24f2c88521809fa9aea99903751584"
+  integrity sha512-b0D9IZ1WhhCWkrTXyFuIIgqGzSkRIH5D5AmB0bXbzYAB1OBAwHcUeyWW2LorutLWF5btNo/N7r/cIdmvvKJlYg==
+  dependencies:
+    "@babel/core" "^7.16.0"
+    "@babel/plugin-proposal-class-properties" "^7.16.0"
+    "@babel/plugin-proposal-decorators" "^7.16.4"
+    "@babel/plugin-proposal-nullish-coalescing-operator" "^7.16.0"
+    "@babel/plugin-proposal-numeric-separator" "^7.16.0"
+    "@babel/plugin-proposal-optional-chaining" "^7.16.0"
+    "@babel/plugin-proposal-private-methods" "^7.16.0"
+    "@babel/plugin-transform-flow-strip-types" "^7.16.0"
+    "@babel/plugin-transform-react-display-name" "^7.16.0"
+    "@babel/plugin-transform-runtime" "^7.16.4"
+    "@babel/preset-env" "^7.16.4"
+    "@babel/preset-react" "^7.16.0"
+    "@babel/preset-typescript" "^7.16.0"
+    "@babel/runtime" "^7.16.3"
+    babel-plugin-macros "^3.1.0"
+    babel-plugin-transform-react-remove-prop-types "^0.4.24"
 
 backo2@^1.0.2:
   version "1.0.2"
@@ -6479,7 +5910,7 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
-base64-js@^1.0.2, base64-js@^1.3.1:
+base64-js@^1.3.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
@@ -6524,23 +5955,11 @@ big.js@^5.2.2:
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328"
   integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==
 
-binary-extensions@^1.0.0:
-  version "1.13.1"
-  resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65"
-  integrity sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==
-
 binary-extensions@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
-bindings@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
-  integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
-  dependencies:
-    file-uri-to-path "1.0.0"
-
 blacklist@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/blacklist/-/blacklist-1.1.4.tgz#b2dd09d6177625b2caa69835a37b28995fa9a2f2"
@@ -6551,16 +5970,6 @@ bluebird@3.7.2, bluebird@^3.5.5:
   resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
-bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9:
-  version "4.12.0"
-  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88"
-  integrity sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==
-
-bn.js@^5.0.0, bn.js@^5.1.1:
-  version "5.2.0"
-  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.0.tgz#358860674396c6997771a9d051fcc1b57d4ae002"
-  integrity sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw==
-
 body-parser@1.20.1:
   version "1.20.1"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668"
@@ -6579,17 +5988,15 @@ body-parser@1.20.1:
     type-is "~1.6.18"
     unpipe "1.0.0"
 
-bonjour@^3.5.0:
-  version "3.5.0"
-  resolved "https://registry.yarnpkg.com/bonjour/-/bonjour-3.5.0.tgz#8e890a183d8ee9a2393b3844c691a42bcf7bc9f5"
-  integrity sha1-jokKGD2O6aI5OzhExpGkK897yfU=
+bonjour-service@^1.0.11:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/bonjour-service/-/bonjour-service-1.1.1.tgz#960948fa0e0153f5d26743ab15baf8e33752c135"
+  integrity sha512-Z/5lQRMOG9k7W+FkeGTNjh7htqn/2LMnfOvBZ8pynNZCM9MwkQkI3zeI4oz09uWdcgmgHugVvBqxGg4VQJ5PCg==
   dependencies:
-    array-flatten "^2.1.0"
-    deep-equal "^1.0.1"
+    array-flatten "^2.1.2"
     dns-equal "^1.0.0"
-    dns-txt "^2.0.2"
-    multicast-dns "^6.0.1"
-    multicast-dns-service-types "^1.1.0"
+    fast-deep-equal "^3.1.3"
+    multicast-dns "^7.2.5"
 
 boolbase@^1.0.0, boolbase@~1.0.0:
   version "1.0.0"
@@ -6604,130 +6011,50 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
-braces@^2.3.1, braces@^2.3.2:
-  version "2.3.2"
-  resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729"
-  integrity sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==
-  dependencies:
-    arr-flatten "^1.1.0"
-    array-unique "^0.3.2"
-    extend-shallow "^2.0.1"
-    fill-range "^4.0.0"
-    isobject "^3.0.1"
-    repeat-element "^1.1.2"
-    snapdragon "^0.8.1"
-    snapdragon-node "^2.0.1"
-    split-string "^3.0.2"
-    to-regex "^3.0.1"
-
-braces@^3.0.1, braces@~3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
-  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
-  dependencies:
-    fill-range "^7.0.1"
-
-brorand@^1.0.1, brorand@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f"
-  integrity sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=
-
-browser-process-hrtime@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626"
-  integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==
-
-browserify-aes@^1.0.0, browserify-aes@^1.0.4:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/browserify-aes/-/browserify-aes-1.2.0.tgz#326734642f403dabc3003209853bb70ad428ef48"
-  integrity sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==
-  dependencies:
-    buffer-xor "^1.0.3"
-    cipher-base "^1.0.0"
-    create-hash "^1.1.0"
-    evp_bytestokey "^1.0.3"
-    inherits "^2.0.1"
-    safe-buffer "^5.0.1"
-
-browserify-cipher@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/browserify-cipher/-/browserify-cipher-1.0.1.tgz#8d6474c1b870bfdabcd3bcfcc1934a10e94f15f0"
-  integrity sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==
-  dependencies:
-    browserify-aes "^1.0.4"
-    browserify-des "^1.0.0"
-    evp_bytestokey "^1.0.0"
-
-browserify-des@^1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/browserify-des/-/browserify-des-1.0.2.tgz#3af4f1f59839403572f1c66204375f7a7f703e9c"
-  integrity sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==
-  dependencies:
-    cipher-base "^1.0.1"
-    des.js "^1.0.0"
-    inherits "^2.0.1"
-    safe-buffer "^5.1.2"
-
-browserify-rsa@^4.0.0, browserify-rsa@^4.0.1:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/browserify-rsa/-/browserify-rsa-4.1.0.tgz#b2fd06b5b75ae297f7ce2dc651f918f5be158c8d"
-  integrity sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==
-  dependencies:
-    bn.js "^5.0.0"
-    randombytes "^2.0.1"
-
-browserify-sign@^4.0.0:
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/browserify-sign/-/browserify-sign-4.2.1.tgz#eaf4add46dd54be3bb3b36c0cf15abbeba7956c3"
-  integrity sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==
-  dependencies:
-    bn.js "^5.1.1"
-    browserify-rsa "^4.0.1"
-    create-hash "^1.2.0"
-    create-hmac "^1.1.7"
-    elliptic "^6.5.3"
-    inherits "^2.0.4"
-    parse-asn1 "^5.1.5"
-    readable-stream "^3.6.0"
-    safe-buffer "^5.2.0"
-
-browserify-zlib@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/browserify-zlib/-/browserify-zlib-0.2.0.tgz#2869459d9aa3be245fe8fe2ca1f46e2e7f54d73f"
-  integrity sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
   dependencies:
-    pako "~1.0.5"
+    balanced-match "^1.0.0"
 
-browserslist@4.14.2:
-  version "4.14.2"
-  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.14.2.tgz#1b3cec458a1ba87588cc5e9be62f19b6d48813ce"
-  integrity sha512-HI4lPveGKUR0x2StIz+2FXfDk9SfVMrxn6PLh1JeGUwcuoDkdKZebWiyLRJ68iIPDpMI4JLVDf7S7XzslgWOhw==
+braces@^2.3.1:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729"
+  integrity sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==
   dependencies:
-    caniuse-lite "^1.0.30001125"
-    electron-to-chromium "^1.3.564"
-    escalade "^3.0.2"
-    node-releases "^1.1.61"
+    arr-flatten "^1.1.0"
+    array-unique "^0.3.2"
+    extend-shallow "^2.0.1"
+    fill-range "^4.0.0"
+    isobject "^3.0.1"
+    repeat-element "^1.1.2"
+    snapdragon "^0.8.1"
+    snapdragon-node "^2.0.1"
+    split-string "^3.0.2"
+    to-regex "^3.0.1"
 
-browserslist@^4.0.0, browserslist@^4.12.0, browserslist@^4.14.5, browserslist@^4.16.6, browserslist@^4.6.2, browserslist@^4.6.4:
-  version "4.16.6"
-  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.6.tgz#d7901277a5a88e554ed305b183ec9b0c08f66fa2"
-  integrity sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ==
+braces@^3.0.2, braces@~3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
+  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
   dependencies:
-    caniuse-lite "^1.0.30001219"
-    colorette "^1.2.2"
-    electron-to-chromium "^1.3.723"
-    escalade "^3.1.1"
-    node-releases "^1.1.71"
+    fill-range "^7.0.1"
+
+browser-process-hrtime@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626"
+  integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==
 
-browserslist@^4.21.3:
-  version "4.21.4"
-  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.4.tgz#e7496bbc67b9e39dd0f98565feccdcb0d4ff6987"
-  integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==
+browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.18.1, browserslist@^4.21.3, browserslist@^4.21.4, browserslist@^4.21.5:
+  version "4.21.9"
+  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.9.tgz#e11bdd3c313d7e2a9e87e8b4b0c7872b13897635"
+  integrity sha512-M0MFoZzbUrRU4KNfCrDLnvyE7gub+peetoTid3TBIqtunaDJyXlwhakT+/VkvSXcfIzFfK/nkCs4nmyTmxdNSg==
   dependencies:
-    caniuse-lite "^1.0.30001400"
-    electron-to-chromium "^1.4.251"
-    node-releases "^2.0.6"
-    update-browserslist-db "^1.0.9"
+    caniuse-lite "^1.0.30001503"
+    electron-to-chromium "^1.4.431"
+    node-releases "^2.0.12"
+    update-browserslist-db "^1.0.11"
 
 bser@2.1.1:
   version "2.1.1"
@@ -6751,25 +6078,6 @@ buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
   integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
 
-buffer-indexof@^1.0.0:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-1.1.1.tgz#52fabcc6a606d1a00302802648ef68f639da268c"
-  integrity sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g==
-
-buffer-xor@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
-  integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=
-
-buffer@^4.3.0:
-  version "4.9.2"
-  resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
-  integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
-  dependencies:
-    base64-js "^1.0.2"
-    ieee754 "^1.1.4"
-    isarray "^1.0.0"
-
 buffer@^5.7.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
@@ -6783,11 +6091,6 @@ builtin-modules@^3.1.0:
   resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.2.0.tgz#45d5db99e7ee5e6bc4f362e008bf917ab5049887"
   integrity sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA==
 
-builtin-status-codes@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8"
-  integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
-
 bytes@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"
@@ -6798,50 +6101,6 @@ bytes@3.1.2:
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
   integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==
 
-cacache@^12.0.2:
-  version "12.0.4"
-  resolved "https://registry.yarnpkg.com/cacache/-/cacache-12.0.4.tgz#668bcbd105aeb5f1d92fe25570ec9525c8faa40c"
-  integrity sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==
-  dependencies:
-    bluebird "^3.5.5"
-    chownr "^1.1.1"
-    figgy-pudding "^3.5.1"
-    glob "^7.1.4"
-    graceful-fs "^4.1.15"
-    infer-owner "^1.0.3"
-    lru-cache "^5.1.1"
-    mississippi "^3.0.0"
-    mkdirp "^0.5.1"
-    move-concurrently "^1.0.1"
-    promise-inflight "^1.0.1"
-    rimraf "^2.6.3"
-    ssri "^6.0.1"
-    unique-filename "^1.1.1"
-    y18n "^4.0.0"
-
-cacache@^15.0.5:
-  version "15.2.0"
-  resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.2.0.tgz#73af75f77c58e72d8c630a7a2858cb18ef523389"
-  integrity sha512-uKoJSHmnrqXgthDFx/IU6ED/5xd+NNGe+Bb+kLZy7Ku4P+BaiWEUflAKPZ7eAzsYGcsAGASJZsybXp+quEcHTw==
-  dependencies:
-    "@npmcli/move-file" "^1.0.1"
-    chownr "^2.0.0"
-    fs-minipass "^2.0.0"
-    glob "^7.1.4"
-    infer-owner "^1.0.4"
-    lru-cache "^6.0.0"
-    minipass "^3.1.1"
-    minipass-collect "^1.0.2"
-    minipass-flush "^1.0.5"
-    minipass-pipeline "^1.2.2"
-    mkdirp "^1.0.3"
-    p-map "^4.0.0"
-    promise-inflight "^1.0.1"
-    rimraf "^3.0.2"
-    ssri "^8.0.1"
-    tar "^6.0.2"
-    unique-filename "^1.1.1"
-
 cache-base@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2"
@@ -6878,25 +6137,6 @@ call-bind@^1.0.0, call-bind@^1.0.2:
     function-bind "^1.1.1"
     get-intrinsic "^1.0.2"
 
-caller-callsite@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/caller-callsite/-/caller-callsite-2.0.0.tgz#847e0fce0a223750a9a027c54b33731ad3154134"
-  integrity sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=
-  dependencies:
-    callsites "^2.0.0"
-
-caller-path@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/caller-path/-/caller-path-2.0.0.tgz#468f83044e369ab2010fac5f06ceee15bb2cb1f4"
-  integrity sha1-Ro+DBE42mrIBD6xfBs7uFbsssfQ=
-  dependencies:
-    caller-callsite "^2.0.0"
-
-callsites@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/callsites/-/callsites-2.0.0.tgz#06eb84f00eea413da86affefacbffb36093b3c50"
-  integrity sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=
-
 callsites@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
@@ -6918,15 +6158,20 @@ camel-case@4.1.2, camel-case@^4.1.1, camel-case@^4.1.2:
     pascal-case "^3.1.2"
     tslib "^2.0.3"
 
-camelcase@5.3.1, camelcase@^5.0.0, camelcase@^5.3.1:
+camelcase-css@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/camelcase-css/-/camelcase-css-2.0.1.tgz#ee978f6947914cc30c6b44741b6ed1df7f043fd5"
+  integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==
+
+camelcase@^5.0.0, camelcase@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
   integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
 
-camelcase@^6.0.0, camelcase@^6.1.0, camelcase@^6.2.0:
-  version "6.2.0"
-  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809"
-  integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==
+camelcase@^6.2.0, camelcase@^6.2.1:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
+  integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
 camelize@^1.0.0:
   version "1.0.0"
@@ -6943,10 +6188,10 @@ caniuse-api@^3.0.0:
     lodash.memoize "^4.1.2"
     lodash.uniq "^4.5.0"
 
-caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001400:
-  version "1.0.30001518"
-  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001518.tgz"
-  integrity sha512-rup09/e3I0BKjncL+FesTayKtPrdwKhUufQFd3riFw1hHg8JmIFoInYfB102cFcY/pPgGmdyl/iy+jgiDi2vdA==
+caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001464, caniuse-lite@^1.0.30001503:
+  version "1.0.30001508"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001508.tgz#4461bbc895c692a96da399639cc1e146e7302a33"
+  integrity sha512-sdQZOJdmt3GJs1UMNpCCCyeuS2IEGLXnHyAo9yIO5JJDjbjoVRij4M1qep6P6gFpptD1PqIYgzM+gwJbOi92mw==
 
 capital-case@^1.0.4:
   version "1.0.4"
@@ -6957,22 +6202,15 @@ capital-case@^1.0.4:
     tslib "^2.0.3"
     upper-case-first "^2.0.2"
 
-capture-exit@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/capture-exit/-/capture-exit-2.0.0.tgz#fb953bfaebeb781f62898239dabb426d08a509a4"
-  integrity sha512-PiT/hQmTonHhl/HFGN+Lx3JJUznrVYJ3+AQsnthneZbvW7x+f08Tk7yLJTLEOUvBTbduLeeBkxEaYXUOUrRq6g==
-  dependencies:
-    rsvp "^4.8.4"
-
 case-anything@^2.1.10:
   version "2.1.10"
   resolved "https://registry.yarnpkg.com/case-anything/-/case-anything-2.1.10.tgz#d18a6ca968d54ec3421df71e3e190f3bced23410"
   integrity sha512-JczJwVrCP0jPKh05McyVsuOg6AYosrB9XWZKbQzXeDAm2ClE/PJE/BcrrQrVyGYH7Jg8V/LDupmyL4kFlVsVFQ==
 
-case-sensitive-paths-webpack-plugin@2.3.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.3.0.tgz#23ac613cc9a856e4f88ff8bb73bbb5e989825cf7"
-  integrity sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ==
+case-sensitive-paths-webpack-plugin@^2.4.0:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4"
+  integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw==
 
 caseless@~0.12.0:
   version "0.12.0"
@@ -6984,15 +6222,6 @@ ccount@^1.0.0:
   resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043"
   integrity sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==
 
-chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2:
-  version "2.4.2"
-  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
-  integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
-  dependencies:
-    ansi-styles "^3.2.1"
-    escape-string-regexp "^1.0.5"
-    supports-color "^5.3.0"
-
 chalk@^1.0.0, chalk@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -7004,6 +6233,15 @@ chalk@^1.0.0, chalk@^1.1.3:
     strip-ansi "^3.0.0"
     supports-color "^2.0.0"
 
+chalk@^2.0.0, chalk@^2.4.1:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
+  integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
+  dependencies:
+    ansi-styles "^3.2.1"
+    escape-string-regexp "^1.0.5"
+    supports-color "^5.3.0"
+
 chalk@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4"
@@ -7012,10 +6250,10 @@ chalk@^3.0.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
-chalk@^4.0.0, chalk@^4.1.0:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.1.tgz#c80b3fab28bf6371e6863325eee67e618b77e6ad"
-  integrity sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==
+chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
+  integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
   dependencies:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
@@ -7075,6 +6313,11 @@ char-regex@^1.0.2:
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
   integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
 
+char-regex@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-2.0.1.tgz#6dafdb25f9d3349914079f010ba8d0e6ff9cd01e"
+  integrity sha512-oSvEeo6ZUD7NepqAat3RqoucZ5SeqLJgOvVIwkafu6IP3V0pO38s/ypdVUmDDK6qIIHNlYHJAKX9E7R7HoKElw==
+
 character-entities-html4@^1.0.0:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-1.1.4.tgz#0e64b0a3753ddbf1fdc044c5fd01d0199a02e125"
@@ -7110,72 +6353,35 @@ check-types@^11.1.1:
   resolved "https://registry.yarnpkg.com/check-types/-/check-types-11.1.2.tgz#86a7c12bf5539f6324eb0e70ca8896c0e38f3e2f"
   integrity sha512-tzWzvgePgLORb9/3a0YenggReLKAIb2owL03H2Xdoe5pKcUyWRSEQ8xfCar8t2SIAuEDwtmx2da1YB52YuHQMQ==
 
-chokidar@^2.1.8:
-  version "2.1.8"
-  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
-  integrity sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==
-  dependencies:
-    anymatch "^2.0.0"
-    async-each "^1.0.1"
-    braces "^2.3.2"
-    glob-parent "^3.1.0"
-    inherits "^2.0.3"
-    is-binary-path "^1.0.0"
-    is-glob "^4.0.0"
-    normalize-path "^3.0.0"
-    path-is-absolute "^1.0.0"
-    readdirp "^2.2.1"
-    upath "^1.1.1"
-  optionalDependencies:
-    fsevents "^1.2.7"
-
-chokidar@^3.4.1, chokidar@^3.4.3:
-  version "3.5.1"
-  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a"
-  integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==
+chokidar@^3.4.2, chokidar@^3.4.3, chokidar@^3.5.3:
+  version "3.5.3"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
+  integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
   dependencies:
-    anymatch "~3.1.1"
+    anymatch "~3.1.2"
     braces "~3.0.2"
-    glob-parent "~5.1.0"
+    glob-parent "~5.1.2"
     is-binary-path "~2.1.0"
     is-glob "~4.0.1"
     normalize-path "~3.0.0"
-    readdirp "~3.5.0"
+    readdirp "~3.6.0"
   optionalDependencies:
-    fsevents "~2.3.1"
-
-chownr@^1.1.1:
-  version "1.1.4"
-  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
-  integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
-
-chownr@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece"
-  integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==
+    fsevents "~2.3.2"
 
 chrome-trace-event@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac"
   integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==
 
-ci-info@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46"
-  integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==
-
-cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de"
-  integrity sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==
-  dependencies:
-    inherits "^2.0.1"
-    safe-buffer "^5.0.1"
+ci-info@^3.2.0:
+  version "3.8.0"
+  resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.8.0.tgz#81408265a5380c929f0bc665d62256628ce9ef91"
+  integrity sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==
 
-cjs-module-lexer@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-0.6.0.tgz#4186fcca0eae175970aee870b9fe2d6cf8d5655f"
-  integrity sha512-uc2Vix1frTfnuzxxu1Hp4ktSvM3QaI4oXl4ZUqL1wjTu/BGki9TrCWoqLTg/drR1KwAEarXuRFCG2Svr1GxPFw==
+cjs-module-lexer@^1.0.0:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz#6c370ab19f8a3394e318fe682686ec0ac684d107"
+  integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==
 
 class-utils@^0.3.5:
   version "0.3.6"
@@ -7187,28 +6393,23 @@ class-utils@^0.3.5:
     isobject "^3.0.0"
     static-extend "^0.1.1"
 
-classnames@*, classnames@2.3.1, classnames@2.x, classnames@^2.2.1, classnames@^2.2.3, classnames@^2.2.5, classnames@^2.2.6:
+classnames@2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.1.tgz#dfcfa3891e306ec1dad105d0e88f4417b8535e8e"
   integrity sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA==
 
-classnames@^2.3.1, classnames@^2.3.2:
+classnames@2.x, classnames@^2.2.1, classnames@^2.2.3, classnames@^2.2.5, classnames@^2.2.6, classnames@^2.3.1, classnames@^2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.2.tgz#351d813bf0137fcc6a76a16b88208d2560a0d924"
   integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==
 
-clean-css@^4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-4.2.3.tgz#507b5de7d97b48ee53d84adb0160ff6216380f78"
-  integrity sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA==
+clean-css@^5.2.2:
+  version "5.3.2"
+  resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-5.3.2.tgz#70ecc7d4d4114921f5d298349ff86a31a9975224"
+  integrity sha512-JVJbM+f3d3Q704rF4bqQ5UUyTtuJ0JRKNbTKVEeujCCBoMdkEi+V+e8oktO9qGQNSvHrFTM6JZRXrUvGR1czww==
   dependencies:
     source-map "~0.6.0"
 
-clean-stack@^2.0.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b"
-  integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==
-
 cli-cursor@^2.0.0, cli-cursor@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5"
@@ -7245,15 +6446,6 @@ clipboard@^2.0.0:
     select "^1.1.2"
     tiny-emitter "^2.0.0"
 
-cliui@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5"
-  integrity sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==
-  dependencies:
-    string-width "^3.1.0"
-    strip-ansi "^5.2.0"
-    wrap-ansi "^5.1.0"
-
 cliui@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/cliui/-/cliui-6.0.0.tgz#511d702c0c4e41ca156d7d0e96021f23e13225b1"
@@ -7272,6 +6464,15 @@ cliui@^7.0.2:
     strip-ansi "^6.0.0"
     wrap-ansi "^7.0.0"
 
+clone-deep@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387"
+  integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==
+  dependencies:
+    is-plain-object "^2.0.4"
+    kind-of "^6.0.2"
+    shallow-clone "^3.0.0"
+
 clone-response@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.2.tgz#d1dc973920314df67fbeb94223b4ee350239e96b"
@@ -7279,11 +6480,6 @@ clone-response@^1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
-clone@^2.1.2:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
-  integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=
-
 clsx@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12"
@@ -7326,7 +6522,7 @@ collection-visit@^1.0.0:
     map-visit "^1.0.0"
     object-visit "^1.0.0"
 
-color-convert@^1.9.0, color-convert@^1.9.1:
+color-convert@^1.9.0:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
   integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@@ -7350,36 +6546,25 @@ color-name@1.1.3:
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
   integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=
 
-color-name@^1.0.0, color-name@~1.1.4:
+color-name@~1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
 
-color-string@^1.5.4:
-  version "1.5.5"
-  resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.5.tgz#65474a8f0e7439625f3d27a6a19d89fc45223014"
-  integrity sha512-jgIoum0OfQfq9Whcfc2z/VhCNcmQjWbey6qBX0vqt7YICflUmBCh9E9CiQD5GSJ+Uehixm3NUwHVhqUAWRivZg==
-  dependencies:
-    color-name "^1.0.0"
-    simple-swizzle "^0.2.2"
-
 color2k@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/color2k/-/color2k-2.0.0.tgz#86992c82e248c29f524023ed0822bc152c4fa670"
   integrity sha512-DWX9eXOC4fbJNiuvdH4QSHvvfLWyFo9TuFp7V9OzdsbPAdrWAuYc8qvFP2bIQ/LKh4LrAVnJ6vhiQYPvAHdtTg==
 
-color@^3.0.0:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/color/-/color-3.1.3.tgz#ca67fb4e7b97d611dcde39eceed422067d91596e"
-  integrity sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ==
-  dependencies:
-    color-convert "^1.9.1"
-    color-string "^1.5.4"
+colord@^2.9.1:
+  version "2.9.3"
+  resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.3.tgz#4f8ce919de456f1d5c1c368c307fe20f3e59fb43"
+  integrity sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==
 
-colorette@^1.2.1, colorette@^1.2.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94"
-  integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==
+colorette@^2.0.10:
+  version "2.0.20"
+  resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a"
+  integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==
 
 combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
   version "1.0.8"
@@ -7398,11 +6583,26 @@ commander@^2.20.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
   integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
 
-commander@^4.1.1:
+commander@^4.0.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068"
   integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==
 
+commander@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
+  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
+
+commander@^8.3.0:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66"
+  integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
+
+common-path-prefix@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/common-path-prefix/-/common-path-prefix-3.0.0.tgz#7d007a7e07c58c4b4d5f433131a19141b29f11e0"
+  integrity sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==
+
 common-tags@1.8.0, common-tags@^1.8.0:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937"
@@ -7423,13 +6623,6 @@ component-emitter@^1.2.1:
   resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0"
   integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==
 
-compose-function@3.0.3:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/compose-function/-/compose-function-3.0.3.tgz#9ed675f13cc54501d30950a486ff6a7ba3ab185f"
-  integrity sha1-ntZ18TzFRQHTCVCkhv9qe6OrGF8=
-  dependencies:
-    arity-n "^1.0.4"
-
 compressible@~2.0.16:
   version "2.0.18"
   resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba"
@@ -7460,30 +6653,15 @@ concat-map@0.0.1:
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
   integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
 
-concat-stream@^1.5.0:
-  version "1.6.2"
-  resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34"
-  integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==
-  dependencies:
-    buffer-from "^1.0.0"
-    inherits "^2.0.3"
-    readable-stream "^2.2.2"
-    typedarray "^0.0.6"
-
-confusing-browser-globals@^1.0.10:
-  version "1.0.10"
-  resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.10.tgz#30d1e7f3d1b882b25ec4933d1d1adac353d20a59"
-  integrity sha512-gNld/3lySHwuhaVluJUKLePYirM3QNCKzVxqAdhJII9/WXKVX5PURzMVJspS1jTslSqjeuG4KMVTSouit5YPHA==
-
-connect-history-api-fallback@^1.6.0:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/connect-history-api-fallback/-/connect-history-api-fallback-1.6.0.tgz#8b32089359308d111115d81cad3fceab888f97bc"
-  integrity sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg==
+confusing-browser-globals@^1.0.10, confusing-browser-globals@^1.0.11:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81"
+  integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==
 
-console-browserify@^1.1.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/console-browserify/-/console-browserify-1.2.0.tgz#67063cef57ceb6cf4993a2ab3a55840ae8c49336"
-  integrity sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==
+connect-history-api-fallback@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz#647264845251a0daf25b97ce87834cace0f5f1c8"
+  integrity sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==
 
 constant-case@^3.0.3, constant-case@^3.0.4:
   version "3.0.4"
@@ -7494,11 +6672,6 @@ constant-case@^3.0.3, constant-case@^3.0.4:
     tslib "^2.0.3"
     upper-case "^2.0.2"
 
-constants-browserify@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/constants-browserify/-/constants-browserify-1.0.0.tgz#c20b96d8c617748aaf1c16021760cd27fcb8cb75"
-  integrity sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=
-
 content-disposition@0.5.4:
   version "0.5.4"
   resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe"
@@ -7511,19 +6684,7 @@ content-type@~1.0.4:
   resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
   integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==
 
-convert-source-map@1.7.0, convert-source-map@^1.4.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442"
-  integrity sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==
-  dependencies:
-    safe-buffer "~5.1.1"
-
-convert-source-map@^0.3.3:
-  version "0.3.5"
-  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-0.3.5.tgz#f1d802950af7dd2631a1febe0596550c86ab3190"
-  integrity sha1-8dgClQr33SYxof6+BZZVDIarMZA=
-
-convert-source-map@^1.5.0:
+convert-source-map@^1.4.0, convert-source-map@^1.5.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f"
   integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==
@@ -7545,76 +6706,46 @@ copy-anything@^2.0.1:
   dependencies:
     is-what "^3.12.0"
 
-copy-concurrently@^1.0.0:
-  version "1.0.5"
-  resolved "https://registry.yarnpkg.com/copy-concurrently/-/copy-concurrently-1.0.5.tgz#92297398cae34937fcafd6ec8139c18051f0b5e0"
-  integrity sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==
-  dependencies:
-    aproba "^1.1.1"
-    fs-write-stream-atomic "^1.0.8"
-    iferr "^0.1.5"
-    mkdirp "^0.5.1"
-    rimraf "^2.5.4"
-    run-queue "^1.0.0"
-
 copy-descriptor@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d"
   integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=
 
-copy-to-clipboard@^3.2.0:
-  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/copy-to-clipboard/-/copy-to-clipboard-3.3.1.tgz#115aa1a9998ffab6196f93076ad6da3b913662ae"
-  integrity sha512-i13qo6kIHTTpCm8/Wup+0b1mVWETvu2kIMzKoK8FpkLkFxlt0znUAHcMzox+T8sPlqtZXq3CulEjQHsYiGFJUw==
-  dependencies:
-    toggle-selection "^1.0.6"
-
-copy-to-clipboard@^3.3.1:
+copy-to-clipboard@^3.2.0, copy-to-clipboard@^3.3.1:
   version "3.3.3"
   resolved "https://registry.yarnpkg.com/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz#55ac43a1db8ae639a4bd99511c148cdd1b83a1b0"
   integrity sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==
   dependencies:
     toggle-selection "^1.0.6"
 
-copy-webpack-plugin@6.4.1:
-  version "6.4.1"
-  resolved "https://registry.yarnpkg.com/copy-webpack-plugin/-/copy-webpack-plugin-6.4.1.tgz#138cd9b436dbca0a6d071720d5414848992ec47e"
-  integrity sha512-MXyPCjdPVx5iiWyl40Va3JGh27bKzOTNY3NjUTrosD2q7dR/cLD0013uqJ3BpFbUjyONINjb6qI7nDIJujrMbA==
+copy-webpack-plugin@^11.0.0:
+  version "11.0.0"
+  resolved "https://registry.yarnpkg.com/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz#96d4dbdb5f73d02dd72d0528d1958721ab72e04a"
+  integrity sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==
   dependencies:
-    cacache "^15.0.5"
-    fast-glob "^3.2.4"
-    find-cache-dir "^3.3.1"
-    glob-parent "^5.1.1"
-    globby "^11.0.1"
-    loader-utils "^2.0.0"
+    fast-glob "^3.2.11"
+    glob-parent "^6.0.1"
+    globby "^13.1.1"
     normalize-path "^3.0.0"
-    p-limit "^3.0.2"
-    schema-utils "^3.0.0"
-    serialize-javascript "^5.0.1"
-    webpack-sources "^1.4.3"
+    schema-utils "^4.0.0"
+    serialize-javascript "^6.0.0"
 
-core-js-compat@^3.6.2, core-js-compat@^3.9.0, core-js-compat@^3.9.1:
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.13.0.tgz#a88f5fa81d8e9b15d7f98abc4447a4dfca2a358f"
-  integrity sha512-jhbI2zpVskgfDC9mGRaDo1gagd0E0i/kYW0+WvibL/rafEHKAHO653hEXIxJHqRlRLITluXtRH3AGTL5qJmifQ==
+core-js-compat@^3.30.1, core-js-compat@^3.30.2:
+  version "3.31.0"
+  resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.31.0.tgz#4030847c0766cc0e803dcdfb30055d7ef2064bf1"
+  integrity sha512-hM7YCu1cU6Opx7MXNu0NuumM0ezNeAeRKadixyiQELWY3vT3De9S4J5ZBMraWV2vZnrE1Cirl0GtFtDtMUXzPw==
   dependencies:
-    browserslist "^4.16.6"
-    semver "7.0.0"
+    browserslist "^4.21.5"
 
-core-js-pure@^3.0.0:
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.13.0.tgz#9d267fb47d1d7046cfbc05e7b67bb235b6735355"
-  integrity sha512-7VTvXbsMxROvzPAVczLgfizR8CyYnvWPrb1eGrtlZAJfjQWEHLofVfCKljLHdpazTfpaziRORwUH/kfGDKvpdA==
+core-js-pure@^3.0.0, core-js-pure@^3.23.3:
+  version "3.31.0"
+  resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.31.0.tgz#052fd9e82fbaaf86457f5db1fadcd06f15966ff2"
+  integrity sha512-/AnE9Y4OsJZicCzIe97JP5XoPKQJfTuEG43aEVLFJGOJpyqELod+pE6LEl63DfG1Mp8wX97LDaDpy1GmLEUxlg==
 
-core-js@^2.4.0:
-  version "2.6.12"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec"
-  integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==
-
-core-js@^3.6.5:
-  version "3.13.0"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.13.0.tgz#58ca436bf01d6903aee3d364089868d0d89fe58d"
-  integrity sha512-iWDbiyha1M5vFwPFmQnvRv+tJzGbFAm6XimJUT0NgHYW3xZEs1SkCAcasWSVFxpI2Xb/V1DDJckq3v90+bQnog==
+core-js@^3.19.2:
+  version "3.31.0"
+  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.31.0.tgz#4471dd33e366c79d8c0977ed2d940821719db344"
+  integrity sha512-NIp2TQSGfR6ba5aalZD+ZQ1fSxGhDo/s1w0nx3RYzf2pnJxt7YynxFlFScP6eV7+GZsKO95NSjGxyJsU3DZgeQ==
 
 core-util-is@1.0.2, core-util-is@~1.0.0:
   version "1.0.2"
@@ -7628,7 +6759,15 @@ cosmiconfig-toml-loader@1.0.0:
   dependencies:
     "@iarna/toml" "^2.2.5"
 
-cosmiconfig@7.0.0, cosmiconfig@^7.0.0:
+cosmiconfig-typescript-loader@^1.0.0:
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-1.0.9.tgz#69c523f7e8c3d9f27f563d02bbeadaf2f27212d3"
+  integrity sha512-tRuMRhxN4m1Y8hP9SNYfz7jRwt8lZdWxdjg/ohg5esKmsndJIn4yT96oJVcf5x0eA11taXl+sIp+ielu529k6g==
+  dependencies:
+    cosmiconfig "^7"
+    ts-node "^10.7.0"
+
+cosmiconfig@7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3"
   integrity sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA==
@@ -7639,16 +6778,6 @@ cosmiconfig@7.0.0, cosmiconfig@^7.0.0:
     path-type "^4.0.0"
     yaml "^1.10.0"
 
-cosmiconfig@^5.0.0:
-  version "5.2.1"
-  resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a"
-  integrity sha512-H65gsXo1SKjf8zmrJ67eJk8aIRKV5ff2D4uKZIBZShbhGSpEmsQOPW/SKMKYhSTrqR7ufy6RP69rPogdaPh/kA==
-  dependencies:
-    import-fresh "^2.0.0"
-    is-directory "^0.3.1"
-    js-yaml "^3.13.1"
-    parse-json "^4.0.0"
-
 cosmiconfig@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982"
@@ -7660,22 +6789,38 @@ cosmiconfig@^6.0.0:
     path-type "^4.0.0"
     yaml "^1.7.2"
 
-craco-antd@^1.19.0:
-  version "1.19.0"
-  resolved "https://registry.yarnpkg.com/craco-antd/-/craco-antd-1.19.0.tgz#ad3b95853a29a20c6de4003874517ae2a995064b"
-  integrity sha512-MpbF2LQxb/POiR003oOkuAhHwpyRx1w5opyg7SA4/2og/FMRR/2oca/eqKYQ7vre2dOpt64gkQ5xWETktHWCQQ==
+cosmiconfig@^7, cosmiconfig@^7.0.0, cosmiconfig@^7.0.1:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz#1443b9afa596b670082ea46cbd8f6a62b84635f6"
+  integrity sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==
+  dependencies:
+    "@types/parse-json" "^4.0.0"
+    import-fresh "^3.2.1"
+    parse-json "^5.0.0"
+    path-type "^4.0.0"
+    yaml "^1.10.0"
+
+craco-antd@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/craco-antd/-/craco-antd-2.0.0.tgz#f38977f4de1714e984ad4f68aae2bcce81bdab79"
+  integrity sha512-qrWBvsDM6ZmR5sBzEpB+rpyN3ZicewL4DTjAnKNR0YxzMzGrM6HqYgRgy+SplGtVvowFzo1a2PPmb8jfolWGrQ==
   dependencies:
-    babel-plugin-import "^1.13.1"
-    craco-less "1.17.0"
-    less-vars-to-js "^1.3.0"
+    babel-plugin-import "1.13.5"
+    craco-less "2.0.0"
+    less-vars-to-js "1.3.0"
+
+craco-babel-loader@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/craco-babel-loader/-/craco-babel-loader-1.0.4.tgz#b887dff41f4ad1e8be84f90c1277af16475ada78"
+  integrity sha512-qoCsRWV/cMcucuIe+m09GRudCwAWtEhZl830MR2N0/weXBK+0VybsL19pUQtH0TwC33v9ll4myu/4TGBxnqZeA==
 
-craco-less@1.17.0:
-  version "1.17.0"
-  resolved "https://registry.yarnpkg.com/craco-less/-/craco-less-1.17.0.tgz#3d92e59966e1fdecef4d173b2389985a75cb6bb9"
-  integrity sha512-G+GPEKPPKiSvYDsnQWuj1C4CIuaY8w+iHvULHkNf5QWLE0LkPfSRf3frhRDJjFxtkThpLPSLjWndD9kx8bCWzw==
+craco-less@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/craco-less/-/craco-less-2.0.0.tgz#a2df18c32e97ebf00f62c3f2ea4cd97035f5f640"
+  integrity sha512-980mQaZVrC4ZsvOwvud6/AgvW7fLY3mW5m5+gR4sw4krxszgHb+qoRyOjqsYPD0F4oUmQoSiZSrlYY/bFGD9kQ==
   dependencies:
-    less "^3.11.1"
-    less-loader "^6.1.0"
+    less "^4.1.1"
+    less-loader "^7.3.0"
 
 create-context-state@^2.0.0:
   version "2.0.0"
@@ -7684,37 +6829,6 @@ create-context-state@^2.0.0:
   dependencies:
     "@babel/runtime" "^7.13.10"
 
-create-ecdh@^4.0.0:
-  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.4.tgz#d6e7f4bffa66736085a0762fd3a632684dabcc4e"
-  integrity sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==
-  dependencies:
-    bn.js "^4.1.0"
-    elliptic "^6.5.3"
-
-create-hash@^1.1.0, create-hash@^1.1.2, create-hash@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/create-hash/-/create-hash-1.2.0.tgz#889078af11a63756bcfb59bd221996be3a9ef196"
-  integrity sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==
-  dependencies:
-    cipher-base "^1.0.1"
-    inherits "^2.0.1"
-    md5.js "^1.3.4"
-    ripemd160 "^2.0.1"
-    sha.js "^2.4.0"
-
-create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
-  version "1.1.7"
-  resolved "https://registry.yarnpkg.com/create-hmac/-/create-hmac-1.1.7.tgz#69170c78b3ab957147b2b8b04572e47ead2243ff"
-  integrity sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==
-  dependencies:
-    cipher-base "^1.0.3"
-    create-hash "^1.1.0"
-    inherits "^2.0.1"
-    ripemd160 "^2.0.0"
-    safe-buffer "^5.0.1"
-    sha.js "^2.4.8"
-
 create-react-class@^15.6.2:
   version "15.7.0"
   resolved "https://registry.yarnpkg.com/create-react-class/-/create-react-class-15.7.0.tgz#7499d7ca2e69bb51d13faf59bd04f0c65a1d6c1e"
@@ -7740,14 +6854,14 @@ cross-fetch@3.0.6:
   dependencies:
     node-fetch "2.6.1"
 
-cross-fetch@3.1.4, cross-fetch@^3.0.6:
+cross-fetch@3.1.4:
   version "3.1.4"
   resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39"
   integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==
   dependencies:
     node-fetch "2.6.1"
 
-cross-fetch@^3.1.5:
+cross-fetch@^3.0.6, cross-fetch@^3.1.5:
   version "3.1.5"
   resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f"
   integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==
@@ -7761,7 +6875,7 @@ cross-inspect@1.0.0:
   dependencies:
     tslib "^2.4.0"
 
-cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
+cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
   integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
@@ -7770,71 +6884,34 @@ cross-spawn@7.0.3, cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
-cross-spawn@^6.0.0:
-  version "6.0.5"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
-  integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==
-  dependencies:
-    nice-try "^1.0.4"
-    path-key "^2.0.1"
-    semver "^5.5.0"
-    shebang-command "^1.2.0"
-    which "^1.2.9"
-
-crypto-browserify@^3.11.0:
-  version "3.12.0"
-  resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec"
-  integrity sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==
-  dependencies:
-    browserify-cipher "^1.0.0"
-    browserify-sign "^4.0.0"
-    create-ecdh "^4.0.0"
-    create-hash "^1.1.0"
-    create-hmac "^1.1.0"
-    diffie-hellman "^5.0.0"
-    inherits "^2.0.1"
-    pbkdf2 "^3.0.3"
-    public-encrypt "^4.0.0"
-    randombytes "^2.0.0"
-    randomfill "^1.0.3"
-
-crypto-random-string@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e"
-  integrity sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=
+crypto-random-string@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
+  integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
 
-css-blank-pseudo@^0.1.4:
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/css-blank-pseudo/-/css-blank-pseudo-0.1.4.tgz#dfdefd3254bf8a82027993674ccf35483bfcb3c5"
-  integrity sha512-LHz35Hr83dnFeipc7oqFDmsjHdljj3TQtxGGiNWSOsTLIAubSm4TEz8qCaKFpk7idaQ1GfWscF4E6mgpBysA1w==
+css-blank-pseudo@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/css-blank-pseudo/-/css-blank-pseudo-3.0.3.tgz#36523b01c12a25d812df343a32c322d2a2324561"
+  integrity sha512-VS90XWtsHGqoM0t4KpH053c4ehxZ2E6HtGI7x68YFV0pTo/QmkV/YFA+NnlvK8guxZVNWGQhVNJGC39Q8XF4OQ==
   dependencies:
-    postcss "^7.0.5"
+    postcss-selector-parser "^6.0.9"
 
 css-color-keywords@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/css-color-keywords/-/css-color-keywords-1.0.0.tgz#fea2616dc676b2962686b3af8dbdbe180b244e05"
   integrity sha1-/qJhbcZ2spYmhrOvjb2+GAskTgU=
 
-css-color-names@0.0.4, css-color-names@^0.0.4:
-  version "0.0.4"
-  resolved "https://registry.yarnpkg.com/css-color-names/-/css-color-names-0.0.4.tgz#808adc2e79cf84738069b646cb20ec27beb629e0"
-  integrity sha1-gIrcLnnPhHOAabZGyyDsJ762KeA=
-
-css-declaration-sorter@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-4.0.1.tgz#c198940f63a76d7e36c1e71018b001721054cb22"
-  integrity sha512-BcxQSKTSEEQUftYpBVnsH4SF05NTuBokb19/sBt6asXGKZ/6VP7PLG1CBCkFDYOnhXhPh0jMhO6xZ71oYHXHBA==
-  dependencies:
-    postcss "^7.0.1"
-    timsort "^0.3.0"
+css-declaration-sorter@^6.3.1:
+  version "6.4.0"
+  resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-6.4.0.tgz#630618adc21724484b3e9505bce812def44000ad"
+  integrity sha512-jDfsatwWMWN0MODAFuHszfjphEXfNw9JUAhmY4pLu3TyTU+ohUpsbVtbU+1MZn4a47D9kqh03i4eyOm+74+zew==
 
-css-has-pseudo@^0.10.0:
-  version "0.10.0"
-  resolved "https://registry.yarnpkg.com/css-has-pseudo/-/css-has-pseudo-0.10.0.tgz#3c642ab34ca242c59c41a125df9105841f6966ee"
-  integrity sha512-Z8hnfsZu4o/kt+AuFzeGpLVhFOGO9mluyHBaA2bA8aCGTwah5sT3WV/fTHH8UNZUytOIImuGPrl/prlb4oX4qQ==
+css-has-pseudo@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/css-has-pseudo/-/css-has-pseudo-3.0.4.tgz#57f6be91ca242d5c9020ee3e51bbb5b89fc7af73"
+  integrity sha512-Vse0xpR1K9MNlp2j5w1pgWIJtm1a8qS0JwS9goFYcImjlHEmywP9VUF05aGBXzGpDJF86QXk4L0ypBmwPhGArw==
   dependencies:
-    postcss "^7.0.6"
-    postcss-selector-parser "^5.0.0-rc.4"
+    postcss-selector-parser "^6.0.9"
 
 css-in-js-utils@^3.1.0:
   version "3.1.0"
@@ -7843,37 +6920,43 @@ css-in-js-utils@^3.1.0:
   dependencies:
     hyphenate-style-name "^1.0.3"
 
-css-loader@4.3.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-4.3.0.tgz#c888af64b2a5b2e85462c72c0f4a85c7e2e0821e"
-  integrity sha512-rdezjCjScIrsL8BSYszgT4s476IcNKt6yX69t0pHjJVnPUTDpn4WfIpDQTN3wCJvUvfsz/mFjuGOekf3PY3NUg==
-  dependencies:
-    camelcase "^6.0.0"
-    cssesc "^3.0.0"
-    icss-utils "^4.1.1"
-    loader-utils "^2.0.0"
-    postcss "^7.0.32"
-    postcss-modules-extract-imports "^2.0.0"
-    postcss-modules-local-by-default "^3.0.3"
-    postcss-modules-scope "^2.2.0"
-    postcss-modules-values "^3.0.0"
-    postcss-value-parser "^4.1.0"
-    schema-utils "^2.7.1"
-    semver "^7.3.2"
+css-loader@^6.5.1:
+  version "6.8.1"
+  resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-6.8.1.tgz#0f8f52699f60f5e679eab4ec0fcd68b8e8a50a88"
+  integrity sha512-xDAXtEVGlD0gJ07iclwWVkLoZOpEvAWaSyf6W18S2pOC//K8+qUDIx8IIT3D+HjnmkJPQeesOPv5aiUaJsCM2g==
+  dependencies:
+    icss-utils "^5.1.0"
+    postcss "^8.4.21"
+    postcss-modules-extract-imports "^3.0.0"
+    postcss-modules-local-by-default "^4.0.3"
+    postcss-modules-scope "^3.0.0"
+    postcss-modules-values "^4.0.0"
+    postcss-value-parser "^4.2.0"
+    semver "^7.3.8"
+
+css-minimizer-webpack-plugin@^3.2.0:
+  version "3.4.1"
+  resolved "https://registry.yarnpkg.com/css-minimizer-webpack-plugin/-/css-minimizer-webpack-plugin-3.4.1.tgz#ab78f781ced9181992fe7b6e4f3422e76429878f"
+  integrity sha512-1u6D71zeIfgngN2XNRJefc/hY7Ybsxd74Jm4qngIXyUEk7fss3VUzuHxLAq/R8NAba4QU9OUSaMZlbpRc7bM4Q==
+  dependencies:
+    cssnano "^5.0.6"
+    jest-worker "^27.0.2"
+    postcss "^8.3.5"
+    schema-utils "^4.0.0"
+    serialize-javascript "^6.0.0"
+    source-map "^0.6.1"
 
-css-prefers-color-scheme@^3.1.1:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/css-prefers-color-scheme/-/css-prefers-color-scheme-3.1.1.tgz#6f830a2714199d4f0d0d0bb8a27916ed65cff1f4"
-  integrity sha512-MTu6+tMs9S3EUqzmqLXEcgNRbNkkD/TGFvowpeoWJn5Vfq7FMgsmRQs9X5NXAURiOBmOxm/lLjsDNXDE6k9bhg==
-  dependencies:
-    postcss "^7.0.5"
+css-prefers-color-scheme@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/css-prefers-color-scheme/-/css-prefers-color-scheme-6.0.3.tgz#ca8a22e5992c10a5b9d315155e7caee625903349"
+  integrity sha512-4BqMbZksRkJQx2zAjrokiGMd07RqOa2IxIrrN10lyBe9xhn9DEvjUK79J6jkeiv9D9hQFXKb6g1jwU62jziJZA==
 
 css-select-base-adapter@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz#3b2ff4972cc362ab88561507a95408a1432135d7"
   integrity sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w==
 
-css-select@^2.0.0, css-select@^2.0.2:
+css-select@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/css-select/-/css-select-2.1.0.tgz#6a34653356635934a81baca68d0255432105dbef"
   integrity sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==
@@ -7883,6 +6966,17 @@ css-select@^2.0.0, css-select@^2.0.2:
     domutils "^1.7.0"
     nth-check "^1.0.2"
 
+css-select@^4.1.3:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.3.0.tgz#db7129b2846662fd8628cfc496abb2b59e41529b"
+  integrity sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==
+  dependencies:
+    boolbase "^1.0.0"
+    css-what "^6.0.1"
+    domhandler "^4.3.1"
+    domutils "^2.8.0"
+    nth-check "^2.0.1"
+
 css-to-react-native@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/css-to-react-native/-/css-to-react-native-3.0.0.tgz#62dbe678072a824a689bcfee011fc96e02a7d756"
@@ -7900,7 +6994,7 @@ css-tree@1.0.0-alpha.37:
     mdn-data "2.0.4"
     source-map "^0.6.1"
 
-css-tree@^1.1.2:
+css-tree@^1.1.2, css-tree@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d"
   integrity sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==
@@ -7913,21 +7007,16 @@ css-what@^3.2.1:
   resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.2.tgz#ea7026fcb01777edbde52124e21f327e7ae950e4"
   integrity sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==
 
+css-what@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4"
+  integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==
+
 css.escape@^1.5.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/css.escape/-/css.escape-1.5.1.tgz#42e27d4fa04ae32f931a4b4d4191fa9cddee97cb"
   integrity sha1-QuJ9T6BK4y+TGktNQZH6nN3ul8s=
 
-css@^2.0.0:
-  version "2.2.4"
-  resolved "https://registry.yarnpkg.com/css/-/css-2.2.4.tgz#c646755c73971f2bba6a601e2cf2fd71b1298929"
-  integrity sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==
-  dependencies:
-    inherits "^2.0.3"
-    source-map "^0.6.1"
-    source-map-resolve "^0.5.2"
-    urix "^0.1.0"
-
 css@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/css/-/css-3.0.0.tgz#4447a4d58fdd03367c516ca9f64ae365cee4aa5d"
@@ -7937,90 +7026,66 @@ css@^3.0.0:
     source-map "^0.6.1"
     source-map-resolve "^0.6.0"
 
-cssdb@^4.4.0:
-  version "4.4.0"
-  resolved "https://registry.yarnpkg.com/cssdb/-/cssdb-4.4.0.tgz#3bf2f2a68c10f5c6a08abd92378331ee803cddb0"
-  integrity sha512-LsTAR1JPEM9TpGhl/0p3nQecC2LJ0kD8X5YARu1hk/9I1gril5vDtMZyNxcEpxxDj34YNck/ucjuoUd66K03oQ==
-
-cssesc@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-2.0.0.tgz#3b13bd1bb1cb36e1bcb5a4dcd27f54c5dcb35703"
-  integrity sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg==
+cssdb@^7.1.0:
+  version "7.6.0"
+  resolved "https://registry.yarnpkg.com/cssdb/-/cssdb-7.6.0.tgz#beac8f7a5f676db62d3c33da517ef4c9eb008f8b"
+  integrity sha512-Nna7rph8V0jC6+JBY4Vk4ndErUmfJfV6NJCaZdurL0omggabiy+QB2HCQtu5c/ACLZ0I7REv7A4QyPIoYzZx0w==
 
 cssesc@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee"
   integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==
 
-cssnano-preset-default@^4.0.8:
-  version "4.0.8"
-  resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-4.0.8.tgz#920622b1fc1e95a34e8838203f1397a504f2d3ff"
-  integrity sha512-LdAyHuq+VRyeVREFmuxUZR1TXjQm8QQU/ktoo/x7bz+SdOge1YKc5eMN6pRW7YWBmyq59CqYba1dJ5cUukEjLQ==
-  dependencies:
-    css-declaration-sorter "^4.0.1"
-    cssnano-util-raw-cache "^4.0.1"
-    postcss "^7.0.0"
-    postcss-calc "^7.0.1"
-    postcss-colormin "^4.0.3"
-    postcss-convert-values "^4.0.1"
-    postcss-discard-comments "^4.0.2"
-    postcss-discard-duplicates "^4.0.2"
-    postcss-discard-empty "^4.0.1"
-    postcss-discard-overridden "^4.0.1"
-    postcss-merge-longhand "^4.0.11"
-    postcss-merge-rules "^4.0.3"
-    postcss-minify-font-values "^4.0.2"
-    postcss-minify-gradients "^4.0.2"
-    postcss-minify-params "^4.0.2"
-    postcss-minify-selectors "^4.0.2"
-    postcss-normalize-charset "^4.0.1"
-    postcss-normalize-display-values "^4.0.2"
-    postcss-normalize-positions "^4.0.2"
-    postcss-normalize-repeat-style "^4.0.2"
-    postcss-normalize-string "^4.0.2"
-    postcss-normalize-timing-functions "^4.0.2"
-    postcss-normalize-unicode "^4.0.1"
-    postcss-normalize-url "^4.0.1"
-    postcss-normalize-whitespace "^4.0.2"
-    postcss-ordered-values "^4.1.2"
-    postcss-reduce-initial "^4.0.3"
-    postcss-reduce-transforms "^4.0.2"
-    postcss-svgo "^4.0.3"
-    postcss-unique-selectors "^4.0.1"
-
-cssnano-util-get-arguments@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/cssnano-util-get-arguments/-/cssnano-util-get-arguments-4.0.0.tgz#ed3a08299f21d75741b20f3b81f194ed49cc150f"
-  integrity sha1-7ToIKZ8h11dBsg87gfGU7UnMFQ8=
-
-cssnano-util-get-match@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/cssnano-util-get-match/-/cssnano-util-get-match-4.0.0.tgz#c0e4ca07f5386bb17ec5e52250b4f5961365156d"
-  integrity sha1-wOTKB/U4a7F+xeUiULT1lhNlFW0=
-
-cssnano-util-raw-cache@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/cssnano-util-raw-cache/-/cssnano-util-raw-cache-4.0.1.tgz#b26d5fd5f72a11dfe7a7846fb4c67260f96bf282"
-  integrity sha512-qLuYtWK2b2Dy55I8ZX3ky1Z16WYsx544Q0UWViebptpwn/xDBmog2TLg4f+DBMg1rJ6JDWtn96WHbOKDWt1WQA==
-  dependencies:
-    postcss "^7.0.0"
-
-cssnano-util-same-parent@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/cssnano-util-same-parent/-/cssnano-util-same-parent-4.0.1.tgz#574082fb2859d2db433855835d9a8456ea18bbf3"
-  integrity sha512-WcKx5OY+KoSIAxBW6UBBRay1U6vkYheCdjyVNDm85zt5K9mHoGOfsOsqIszfAqrQQFIIKgjh2+FDgIj/zsl21Q==
+cssnano-preset-default@^5.2.14:
+  version "5.2.14"
+  resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-5.2.14.tgz#309def4f7b7e16d71ab2438052093330d9ab45d8"
+  integrity sha512-t0SFesj/ZV2OTylqQVOrFgEh5uanxbO6ZAdeCrNsUQ6fVuXwYTxJPNAGvGTxHbD68ldIJNec7PyYZDBrfDQ+6A==
+  dependencies:
+    css-declaration-sorter "^6.3.1"
+    cssnano-utils "^3.1.0"
+    postcss-calc "^8.2.3"
+    postcss-colormin "^5.3.1"
+    postcss-convert-values "^5.1.3"
+    postcss-discard-comments "^5.1.2"
+    postcss-discard-duplicates "^5.1.0"
+    postcss-discard-empty "^5.1.1"
+    postcss-discard-overridden "^5.1.0"
+    postcss-merge-longhand "^5.1.7"
+    postcss-merge-rules "^5.1.4"
+    postcss-minify-font-values "^5.1.0"
+    postcss-minify-gradients "^5.1.1"
+    postcss-minify-params "^5.1.4"
+    postcss-minify-selectors "^5.2.1"
+    postcss-normalize-charset "^5.1.0"
+    postcss-normalize-display-values "^5.1.0"
+    postcss-normalize-positions "^5.1.1"
+    postcss-normalize-repeat-style "^5.1.1"
+    postcss-normalize-string "^5.1.0"
+    postcss-normalize-timing-functions "^5.1.0"
+    postcss-normalize-unicode "^5.1.1"
+    postcss-normalize-url "^5.1.0"
+    postcss-normalize-whitespace "^5.1.1"
+    postcss-ordered-values "^5.1.3"
+    postcss-reduce-initial "^5.1.2"
+    postcss-reduce-transforms "^5.1.0"
+    postcss-svgo "^5.1.0"
+    postcss-unique-selectors "^5.1.1"
+
+cssnano-utils@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/cssnano-utils/-/cssnano-utils-3.1.0.tgz#95684d08c91511edfc70d2636338ca37ef3a6861"
+  integrity sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA==
 
-cssnano@^4.1.10:
-  version "4.1.11"
-  resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-4.1.11.tgz#c7b5f5b81da269cb1fd982cb960c1200910c9a99"
-  integrity sha512-6gZm2htn7xIPJOHY824ERgj8cNPgPxyCSnkXc4v7YvNW+TdVfzgngHcEhy/8D11kUWRUMbke+tC+AUcUsnMz2g==
+cssnano@^5.0.6:
+  version "5.1.15"
+  resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.1.15.tgz#ded66b5480d5127fcb44dac12ea5a983755136bf"
+  integrity sha512-j+BKgDcLDQA+eDifLx0EO4XSA56b7uut3BQFH+wbSaSTuGLuiyTa/wbRYthUXX8LC9mLg+WWKe8h+qJuwTAbHw==
   dependencies:
-    cosmiconfig "^5.0.0"
-    cssnano-preset-default "^4.0.8"
-    is-resolvable "^1.0.0"
-    postcss "^7.0.0"
+    cssnano-preset-default "^5.2.14"
+    lilconfig "^2.0.3"
+    yaml "^1.10.2"
 
-csso@^4.0.2:
+csso@^4.0.2, csso@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529"
   integrity sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==
@@ -8041,107 +7106,75 @@ cssstyle@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852"
   integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==
-  dependencies:
-    cssom "~0.3.6"
-
-csstype@^3.0.2:
-  version "3.0.8"
-  resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.8.tgz#d2266a792729fb227cd216fb572f43728e1ad340"
-  integrity sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==
-
-csstype@^3.0.6, csstype@^3.0.7, csstype@^3.1.0, csstype@^3.1.1:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9"
-  integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==
-
-cyclist@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
-  integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=
-
-d3-array@2, d3-array@^2.3.0:
-  version "2.12.1"
-  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.12.1.tgz#e20b41aafcdffdf5d50928004ececf815a465e81"
-  integrity sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==
-  dependencies:
-    internmap "^1.0.0"
-
-d3-array@^1.2.0, d3-array@^1.2.1:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"
-  integrity sha512-KHW6M86R+FUPYGb3R5XiYjXPq7VzwxZ22buHhAEVG5ztoEcZZMLov530mmccaqA1GghZArjQV46fuc8kUqhhHw==
+  dependencies:
+    cssom "~0.3.6"
 
-d3-collection@1:
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/d3-collection/-/d3-collection-1.0.7.tgz#349bd2aa9977db071091c13144d5e4f16b5b310e"
-  integrity sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==
+csstype@^3.0.2, csstype@^3.0.6, csstype@^3.0.7, csstype@^3.1.0, csstype@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9"
+  integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==
 
-d3-color@1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-1.4.1.tgz#c52002bf8846ada4424d55d97982fef26eb3bc8a"
-  integrity sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==
+"d3-array@2 - 3", "d3-array@2.10.0 - 3":
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5"
+  integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==
+  dependencies:
+    internmap "1 - 2"
 
-"d3-color@1 - 2":
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-2.0.0.tgz#8d625cab42ed9b8f601a1760a389f7ea9189d62e"
-  integrity sha512-SPXi0TSKPD4g9tw0NMZFnR95XVgUZiBH+uUTqQuDu1OsE2zomHU7ho0FISciaPvosimixwHFl3WHLGabv6dDgQ==
+d3-array@3.2.1:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.1.tgz#39331ea706f5709417d31bbb6ec152e0328b39b3"
+  integrity sha512-gUY/qeHq/yNqqoCKNq4vtpFLdoCdvyNpWoC/KNjhGbhDuQpAM9sIQQKkXSNpXa9h5KySs/gzm7R88WkUutgwWQ==
+  dependencies:
+    internmap "1 - 2"
 
-d3-format@1:
-  version "1.4.5"
-  resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-1.4.5.tgz#374f2ba1320e3717eb74a9356c67daee17a7edb4"
-  integrity sha512-J0piedu6Z8iB6TbIGfZgDzfXxUFN3qQRMofy2oPdXzQibYGqPB/9iMcxr/TGalU+2RsyDO+U4f33id8tbnSRMQ==
+"d3-color@1 - 3", d3-color@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2"
+  integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==
 
-"d3-format@1 - 2":
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-2.0.0.tgz#a10bcc0f986c372b729ba447382413aabf5b0767"
-  integrity sha512-Ab3S6XuE/Q+flY96HXT0jOXcM4EAClYFnRGY5zsjRGNy6qCYrQsMffs7cV5Q9xejb35zxW5hf/guKw34kvIKsA==
+"d3-format@1 - 3", d3-format@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.0.tgz#9260e23a28ea5cb109e93b21a06e24e2ebd55641"
+  integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==
 
 d3-hierarchy@^1.1.4:
   version "1.1.9"
   resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-1.1.9.tgz#2f6bee24caaea43f8dc37545fa01628559647a83"
   integrity sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==
 
-d3-interpolate@1, d3-interpolate@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-1.4.0.tgz#526e79e2d80daa383f9e0c1c1c7dcc0f0583e987"
-  integrity sha512-V9znK0zc3jOPV4VD2zZn0sDhZU3WAE2bmlxdIwwQPPzPjvyLkd8B3JUVdS1IDUFDkWZ72c9qnv1GK2ZagTZ8EA==
-  dependencies:
-    d3-color "1"
+d3-interpolate-path@2.2.1:
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/d3-interpolate-path/-/d3-interpolate-path-2.2.1.tgz#fd8ff20a90aff3f380bcd1c15305e7b531e55d07"
+  integrity sha512-6qLLh/KJVzls0XtMsMpcxhqMhgVEN7VIbR/6YGZe2qlS8KDgyyVB20XcmGnDyB051HcefQXM/Tppa9vcANEA4Q==
 
-"d3-interpolate@1.2.0 - 2":
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-2.0.1.tgz#98be499cfb8a3b94d4ff616900501a64abc91163"
-  integrity sha512-c5UhwwTs/yybcmTpAVqwSFl6vrQ8JZJoT5F7xNFK9pymv5C0Ymcc9/LIJHtYIggg/yS9YHw8i8O8tgb9pupjeQ==
+"d3-interpolate@1.2.0 - 3", d3-interpolate@3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d"
+  integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==
   dependencies:
-    d3-color "1 - 2"
+    d3-color "1 - 3"
 
 d3-path@1, d3-path@^1.0.5:
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-1.0.9.tgz#48c050bb1fe8c262493a8caf5524e3e9591701cf"
   integrity sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==
 
-d3-scale@^2.0.0:
-  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-2.2.2.tgz#4e880e0b2745acaaddd3ede26a9e908a9e17b81f"
-  integrity sha512-LbeEvGgIb8UMcAa0EATLNX0lelKWGYDQiPdHj+gLblGVhGLyNbaCn3EvrJf0A3Y/uOOU5aD6MTh5ZFCdEwGiCw==
-  dependencies:
-    d3-array "^1.2.0"
-    d3-collection "1"
-    d3-format "1"
-    d3-interpolate "1"
-    d3-time "1"
-    d3-time-format "2"
+"d3-path@1 - 2":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-2.0.0.tgz#55d86ac131a0548adae241eebfb56b4582dd09d8"
+  integrity sha512-ZwZQxKhBnv9yHaiWd6ZU4x5BtCQ7pXszEV9CU6kRgwIQVQGLMv1oiL4M+MK/n79sYzsj+gcgpPQSctJUsLN7fA==
 
-d3-scale@^3.0.1, d3-scale@^3.3.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-3.3.0.tgz#28c600b29f47e5b9cd2df9749c206727966203f3"
-  integrity sha512-1JGp44NQCt5d1g+Yy+GeOnZP7xHo0ii8zsQp6PGzd+C1/dl0KGsp9A7Mxwp+1D1o4unbTTxVdU/ZOIEBoeZPbQ==
+d3-scale@4.0.2, d3-scale@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.2.tgz#82b38e8e8ff7080764f8dcec77bd4be393689396"
+  integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==
   dependencies:
-    d3-array "^2.3.0"
-    d3-format "1 - 2"
-    d3-interpolate "1.2.0 - 2"
-    d3-time "^2.1.1"
-    d3-time-format "2 - 3"
+    d3-array "2.10.0 - 3"
+    d3-format "1 - 3"
+    d3-interpolate "1.2.0 - 3"
+    d3-time "2.1.1 - 3"
+    d3-time-format "2 - 4"
 
 d3-shape@^1.0.6, d3-shape@^1.2.0:
   version "1.3.7"
@@ -8150,49 +7183,36 @@ d3-shape@^1.0.6, d3-shape@^1.2.0:
   dependencies:
     d3-path "1"
 
-d3-time-format@2:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.3.0.tgz#107bdc028667788a8924ba040faf1fbccd5a7850"
-  integrity sha512-guv6b2H37s2Uq/GefleCDtbe0XZAuy7Wa49VGkPVPMfLL9qObgBST3lEHJBMUp8S7NdLQAGIvr2KXk8Hc98iKQ==
+d3-shape@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-2.1.0.tgz#3b6a82ccafbc45de55b57fcf956c584ded3b666f"
+  integrity sha512-PnjUqfM2PpskbSLTJvAzp2Wv4CZsnAgTfcVRTwW03QR3MkXF8Uo7B1y/lWkAsmbKwuecto++4NlsYcvYpXpTHA==
   dependencies:
-    d3-time "1"
+    d3-path "1 - 2"
 
-"d3-time-format@2 - 3":
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-3.0.0.tgz#df8056c83659e01f20ac5da5fdeae7c08d5f1bb6"
-  integrity sha512-UXJh6EKsHBTjopVqZBhFysQcoXSv/5yLONZvkQ5Kk3qbwiUYkdX17Xa1PT6U1ZWXGGfB1ey5L8dKMlFq2DO0Ag==
+"d3-time-format@2 - 4", d3-time-format@4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a"
+  integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==
   dependencies:
-    d3-time "1 - 2"
-
-d3-time@1, d3-time@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.1.0.tgz#b1e19d307dae9c900b7e5b25ffc5dcc249a8a0f1"
-  integrity sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA==
+    d3-time "1 - 3"
 
-"d3-time@1 - 2", d3-time@^2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-2.1.1.tgz#e9d8a8a88691f4548e68ca085e5ff956724a6682"
-  integrity sha512-/eIQe/eR4kCQwq7yxi7z4c6qEXf2IYGcjoWB5OOQy4Tq9Uv39/947qlDcN2TLkiTzQWzvnsuYPB9TrWaNfipKQ==
+"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7"
+  integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==
   dependencies:
-    d3-array "2"
+    d3-array "2 - 3"
 
 d3-voronoi@^1.1.2:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/d3-voronoi/-/d3-voronoi-1.1.4.tgz#dd3c78d7653d2bb359284ae478645d95944c8297"
   integrity sha512-dArJ32hchFsrQ8uMiTBLq256MpnZjeuBtdHpaDlYuQyjU0CVzCJl/BVW+SkszaAeH95D/8gxqAhgx0ouAWAfRg==
 
-d@1, d@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a"
-  integrity sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==
-  dependencies:
-    es5-ext "^0.10.50"
-    type "^1.0.1"
-
-damerau-levenshtein@^1.0.6:
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.7.tgz#64368003512a1a6992593741a09a9d31a836f55d"
-  integrity sha512-VvdQIPGdWP0SqFXghj79Wf/5LArmreyMsGLa6FG6iC4t3j7j5s71TrwWmT/4akbDQIqjfACkLZmjXhA7g2oUZw==
+damerau-levenshtein@^1.0.8:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz#b43d286ccbd36bc5b2f7ed41caf2d0aba1f8a6e7"
+  integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==
 
 dash-get@^1.0.2:
   version "1.0.2"
@@ -8230,32 +7250,27 @@ date-fns@^1.27.2:
   resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c"
   integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==
 
-dayjs@1.x:
-  version "1.11.6"
-  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.6.tgz#2e79a226314ec3ec904e3ee1dd5a4f5e5b1c7afb"
-  integrity sha512-zZbY5giJAinCG+7AGaw0wIhNZ6J8AhWuSXKvuc1KAyMiRsvGQWqh4L+MomvhdAYjN+lqvVCMq1I41e3YHvXkyQ==
-
-dayjs@^1.11.7:
+dayjs@1.x, dayjs@^1.11.7:
   version "1.11.7"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2"
   integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==
 
-debounce@^1.2.0:
+debounce@^1.2.0, debounce@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.1.tgz#38881d8f4166a5c5848020c11827b834bcb3e0a5"
   integrity sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug==
 
-debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9:
+debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
   integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==
   dependencies:
     ms "2.0.0"
 
-debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1:
-  version "4.3.2"
-  resolved "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
-  integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
+debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
+  version "4.3.4"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
+  integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
   dependencies:
     ms "2.1.2"
 
@@ -8273,7 +7288,7 @@ debug@4.3.1:
   dependencies:
     ms "2.1.2"
 
-debug@^3.1.1, debug@^3.2.6, debug@^3.2.7:
+debug@^3.2.6, debug@^3.2.7:
   version "3.2.7"
   resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a"
   integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==
@@ -8307,18 +7322,6 @@ dedent@^0.7.0:
   resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c"
   integrity sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=
 
-deep-equal@^1.0.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.1.1.tgz#b5c98c942ceffaf7cb051e24e1434a25a2e6076a"
-  integrity sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==
-  dependencies:
-    is-arguments "^1.0.4"
-    is-date-object "^1.0.1"
-    is-regex "^1.0.4"
-    object-is "^1.0.1"
-    object-keys "^1.1.1"
-    regexp.prototype.flags "^1.2.0"
-
 deep-extend@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
@@ -8334,27 +7337,24 @@ deepmerge@^4.2.2:
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
   integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
 
-default-gateway@^4.2.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/default-gateway/-/default-gateway-4.2.0.tgz#167104c7500c2115f6dd69b0a536bb8ed720552b"
-  integrity sha512-h6sMrVB1VMWVrW13mSc6ia/DwYYw5MN6+exNu1OaJeFac5aSAvwM7lZ0NVfTABuSkQelr4h5oebg3KB1XPdjgA==
+default-gateway@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/default-gateway/-/default-gateway-6.0.3.tgz#819494c888053bdb743edbf343d6cdf7f2943a71"
+  integrity sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==
   dependencies:
-    execa "^1.0.0"
-    ip-regex "^2.1.0"
+    execa "^5.0.0"
 
 defer-to-connect@^1.0.1:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591"
   integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==
 
-define-properties@^1.1.2, define-properties@^1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1"
-  integrity sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==
-  dependencies:
-    object-keys "^1.0.12"
+define-lazy-prop@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f"
+  integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==
 
-define-properties@^1.1.4:
+define-properties@^1.1.3, define-properties@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1"
   integrity sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==
@@ -8384,19 +7384,6 @@ define-property@^2.0.2:
     is-descriptor "^1.0.2"
     isobject "^3.0.1"
 
-del@^4.1.1:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/del/-/del-4.1.1.tgz#9e8f117222ea44a31ff3a156c049b99052a9f0b4"
-  integrity sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==
-  dependencies:
-    "@types/glob" "^7.1.1"
-    globby "^6.1.0"
-    is-path-cwd "^2.0.0"
-    is-path-in-cwd "^2.0.0"
-    p-map "^2.0.0"
-    pify "^4.0.1"
-    rimraf "^2.6.3"
-
 delayed-stream@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
@@ -8427,13 +7414,10 @@ dependency-graph@^0.9.0:
   resolved "https://registry.yarnpkg.com/dependency-graph/-/dependency-graph-0.9.0.tgz#11aed7e203bc8b00f48356d92db27b265c445318"
   integrity sha512-9YLIBURXj4DJMFALxXw9K3Y3rwb5Fk0X5/8ipCzaN84+gKxoHK43tVKRNakCQbiEx07E8Uwhuq21BpUagFhZ8w==
 
-des.js@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/des.js/-/des.js-1.0.1.tgz#5382142e1bdc53f85d86d53e5f4aa7deb91e0843"
-  integrity sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==
-  dependencies:
-    inherits "^2.0.1"
-    minimalistic-assert "^1.0.0"
+dequal@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be"
+  integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==
 
 destroy@1.2.0:
   version "1.2.0"
@@ -8460,7 +7444,7 @@ detect-node@^2.0.4:
   resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1"
   integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==
 
-detect-port-alt@1.1.6:
+detect-port-alt@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/detect-port-alt/-/detect-port-alt-1.1.6.tgz#24707deabe932d4a3cf621302027c2b266568275"
   integrity sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q==
@@ -8468,11 +7452,21 @@ detect-port-alt@1.1.6:
     address "^1.0.1"
     debug "^2.6.0"
 
+didyoumean@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/didyoumean/-/didyoumean-1.2.2.tgz#989346ffe9e839b4555ecf5666edea0d3e8ad037"
+  integrity sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==
+
 diff-sequences@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.6.2.tgz#48ba99157de1923412eed41db6b6d4aa9ca7c0b1"
   integrity sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q==
 
+diff-sequences@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327"
+  integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==
+
 diff@^4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
@@ -8483,15 +7477,6 @@ diff@^5.0.0:
   resolved "https://registry.yarnpkg.com/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b"
   integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==
 
-diffie-hellman@^5.0.0:
-  version "5.0.3"
-  resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"
-  integrity sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==
-  dependencies:
-    bn.js "^4.1.0"
-    miller-rabin "^4.0.0"
-    randombytes "^2.0.0"
-
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
@@ -8514,20 +7499,12 @@ dns-equal@^1.0.0:
   resolved "https://registry.yarnpkg.com/dns-equal/-/dns-equal-1.0.0.tgz#b39e7f1da6eb0a75ba9c17324b34753c47e0654d"
   integrity sha1-s55/HabrCnW6nBcySzR1PEfgZU0=
 
-dns-packet@^1.3.1:
-  version "1.3.4"
-  resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-1.3.4.tgz#e3455065824a2507ba886c55a89963bb107dec6f"
-  integrity sha512-BQ6F4vycLXBvdrJZ6S3gZewt6rcrks9KBgM9vrhW+knGRqc8uEdT7fuCwloc7nny5xNoMJ17HGH0R/6fpo8ECA==
-  dependencies:
-    ip "^1.1.0"
-    safe-buffer "^5.0.1"
-
-dns-txt@^2.0.2:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/dns-txt/-/dns-txt-2.0.2.tgz#b91d806f5d27188e4ab3e7d107d881a1cc4642b6"
-  integrity sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=
+dns-packet@^5.2.2:
+  version "5.6.0"
+  resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-5.6.0.tgz#2202c947845c7a63c23ece58f2f70ff6ab4c2f7d"
+  integrity sha512-rza3UH1LwdHh9qyPXp8lkwpjSNk/AMD3dPytUoRoqnypDUhY0xvbdmVhWOfxO68frEfV9BU8V12Ez7ZsHGZpCQ==
   dependencies:
-    buffer-indexof "^1.0.0"
+    "@leichtgewicht/ip-codec" "^2.0.1"
 
 doctrine@^2.1.0:
   version "2.1.0"
@@ -8553,7 +7530,7 @@ dom-align@^1.7.0:
   resolved "https://registry.yarnpkg.com/dom-align/-/dom-align-1.12.2.tgz#0f8164ebd0c9c21b0c790310493cd855892acd4b"
   integrity sha512-pHuazgqrsTFrGU2WLDdXxCFabkdQDx72ddkraZNih1KsMcN5qsRSTR9O4VJRlwTPCPb5COYg3LOfiMHHcPInHg==
 
-dom-converter@^0.2:
+dom-converter@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/dom-converter/-/dom-converter-0.2.0.tgz#6721a9daee2e293682955b6afe416771627bb768"
   integrity sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==
@@ -8576,25 +7553,29 @@ dom-serializer@0:
     domelementtype "^2.0.1"
     entities "^2.0.0"
 
+dom-serializer@^1.0.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.4.1.tgz#de5d41b1aea290215dc45a6dae8adcf1d32e2d30"
+  integrity sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.2.0"
+    entities "^2.0.0"
+
 dom-walk@^0.1.0:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84"
   integrity sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==
 
-domain-browser@^1.1.1:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda"
-  integrity sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==
-
-domelementtype@1, domelementtype@^1.3.1:
+domelementtype@1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
   integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
 
-domelementtype@^2.0.1:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57"
-  integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==
+domelementtype@^2.0.1, domelementtype@^2.2.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d"
+  integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==
 
 domexception@^2.0.1:
   version "2.0.1"
@@ -8603,12 +7584,12 @@ domexception@^2.0.1:
   dependencies:
     webidl-conversions "^5.0.0"
 
-domhandler@^2.3.0:
-  version "2.4.2"
-  resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803"
-  integrity sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==
+domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.3.1:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.3.1.tgz#8d792033416f59d68bc03a5aa7b018c1ca89279c"
+  integrity sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==
   dependencies:
-    domelementtype "1"
+    domelementtype "^2.2.0"
 
 domino@^2.1.6:
   version "2.1.6"
@@ -8620,7 +7601,7 @@ dompurify@^2.3.8:
   resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f"
   integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw==
 
-domutils@^1.5.1, domutils@^1.7.0:
+domutils@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
   integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==
@@ -8628,6 +7609,15 @@ domutils@^1.5.1, domutils@^1.7.0:
     dom-serializer "0"
     domelementtype "1"
 
+domutils@^2.5.2, domutils@^2.8.0:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135"
+  integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==
+  dependencies:
+    dom-serializer "^1.0.1"
+    domelementtype "^2.2.0"
+    domhandler "^4.2.0"
+
 dot-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-3.0.4.tgz#9b2b670d00a431667a8a75ba29cd1b98809ce751"
@@ -8636,22 +7626,15 @@ dot-case@^3.0.4:
     no-case "^3.0.4"
     tslib "^2.0.3"
 
-dot-prop@^5.2.0:
-  version "5.3.0"
-  resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88"
-  integrity sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==
-  dependencies:
-    is-obj "^2.0.0"
-
-dotenv-expand@5.1.0:
+dotenv-expand@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0"
   integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==
 
-dotenv@8.2.0:
-  version "8.2.0"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
-  integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
+dotenv@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81"
+  integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==
 
 dotenv@^8.2.0:
   version "8.6.0"
@@ -8668,21 +7651,11 @@ duplexer3@^0.1.4:
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
   integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=
 
-duplexer@^0.1.1, duplexer@^0.1.2, duplexer@~0.1.1:
+duplexer@^0.1.2, duplexer@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6"
   integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==
 
-duplexify@^3.4.2, duplexify@^3.6.0:
-  version "3.7.1"
-  resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309"
-  integrity sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==
-  dependencies:
-    end-of-stream "^1.0.0"
-    inherits "^2.0.1"
-    readable-stream "^2.0.0"
-    stream-shift "^1.0.0"
-
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -8703,62 +7676,39 @@ ee-first@1.1.1:
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
   integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==
 
-ejs@^2.6.1:
-  version "2.7.4"
-  resolved "https://registry.yarnpkg.com/ejs/-/ejs-2.7.4.tgz#48661287573dcc53e366c7a1ae52c3a120eec9ba"
-  integrity sha512-7vmuyh5+kuUyJKePhQfRQBhXV5Ce+RnaeeQArKu1EAMpL3WbgMt5WG6uQZpEVvYSSsxMXRKOewtDk9RaTKXRlA==
-
-ejs@^3.1.5:
-  version "3.1.6"
-  resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.6.tgz#5bfd0a0689743bb5268b3550cceeebbc1702822a"
-  integrity sha512-9lt9Zse4hPucPkoP7FHDF0LQAlGyF9JVpnClFLFH3aSSbxmyoqINRpp/9wePWJTUl4KOQwRL72Iw3InHPDkoGw==
+ejs@^3.1.5, ejs@^3.1.6:
+  version "3.1.9"
+  resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361"
+  integrity sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==
   dependencies:
-    jake "^10.6.1"
-
-electron-to-chromium@^1.3.564, electron-to-chromium@^1.3.723:
-  version "1.3.739"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.739.tgz#f07756aa92cabd5a6eec6f491525a64fe62f98b9"
-  integrity sha512-+LPJVRsN7hGZ9EIUUiWCpO7l4E3qBYHNadazlucBfsXBbccDFNKUBAgzE68FnkWGJPwD/AfKhSzL+G+Iqb8A4A==
+    jake "^10.8.5"
 
-electron-to-chromium@^1.4.251:
-  version "1.4.284"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592"
-  integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==
+electron-to-chromium@^1.4.431:
+  version "1.4.441"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.441.tgz#94dd9c1cbf081d83f032a4f1cd9f787e21fc24ce"
+  integrity sha512-LlCgQ8zgYZPymf5H4aE9itwiIWH4YlCiv1HFLmmcBeFYi5E+3eaIFnjHzYtcFQbaKfAW+CqZ9pgxo33DZuoqPg==
 
 elegant-spinner@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz#db043521c95d7e303fd8f345bedc3349cfb0729e"
   integrity sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4=
 
-elliptic@^6.5.3:
-  version "6.5.4"
-  resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb"
-  integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==
-  dependencies:
-    bn.js "^4.11.9"
-    brorand "^1.1.0"
-    hash.js "^1.0.0"
-    hmac-drbg "^1.0.1"
-    inherits "^2.0.4"
-    minimalistic-assert "^1.0.1"
-    minimalistic-crypto-utils "^1.0.1"
+emittery@^0.10.2:
+  version "0.10.2"
+  resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.10.2.tgz#902eec8aedb8c41938c46e9385e9db7e03182933"
+  integrity sha512-aITqOwnLanpHLNXZJENbOgjUBeHocD+xsSJmNrjovKBW5HbSpW3d1pEls7GFQPUWXiwG9+0P4GtHfEqC/4M0Iw==
 
-emittery@^0.7.1:
-  version "0.7.2"
-  resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.7.2.tgz#25595908e13af0f5674ab419396e2fb394cdfa82"
-  integrity sha512-A8OG5SR/ij3SsJdWDJdkkSYUjQdCUx6APQXem0SaEePBSRg4eymGYwBkKo1Y6DU+af/Jn2dBQqDBvjnr9Vi8nQ==
-
-emoji-regex@^7.0.1:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156"
-  integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==
+emittery@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.8.1.tgz#bb23cc86d03b30aa75a7f734819dee2e1ba70860"
+  integrity sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg==
 
 emoji-regex@^8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
-emoji-regex@^9.0.0:
+emoji-regex@^9.2.2:
   version "9.2.2"
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72"
   integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==
@@ -8788,11 +7738,6 @@ emojibase@^6.0.0:
   resolved "https://registry.yarnpkg.com/emojibase/-/emojibase-6.1.0.tgz#c3bc281e998a0e06398416090c23bac8c5ed3ee8"
   integrity sha512-1GkKJPXP6tVkYJHOBSJHoGOr/6uaDxZ9xJ6H7m6PfdGXTmQgbALHLWaVRY4Gi/qf5x/gT/NUXLPuSHYLqtLtrQ==
 
-emojis-list@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389"
-  integrity sha1-TapNnbAPmBmIDHn6RXrlsJof04k=
-
 emojis-list@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78"
@@ -8803,40 +7748,27 @@ encodeurl@~1.0.2:
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
   integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==
 
-end-of-stream@^1.0.0, end-of-stream@^1.1.0:
+end-of-stream@^1.1.0:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
   integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
   dependencies:
     once "^1.4.0"
 
-enhanced-resolve@^4.3.0:
-  version "4.5.0"
-  resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz#2f3cfd84dbe3b487f18f2db2ef1e064a571ca5ec"
-  integrity sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg==
-  dependencies:
-    graceful-fs "^4.1.2"
-    memory-fs "^0.5.0"
-    tapable "^1.0.0"
-
-enquirer@^2.3.5:
-  version "2.3.6"
-  resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d"
-  integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==
+enhanced-resolve@^5.15.0:
+  version "5.15.0"
+  resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz#1af946c7d93603eb88e9896cee4904dc012e9c35"
+  integrity sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==
   dependencies:
-    ansi-colors "^4.1.1"
-
-entities@^1.1.1:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
-  integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
+    graceful-fs "^4.2.4"
+    tapable "^2.2.0"
 
 entities@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
   integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
 
-errno@^0.1.1, errno@^0.1.3, errno@~0.1.7:
+errno@^0.1.1:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f"
   integrity sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==
@@ -8857,29 +7789,7 @@ error-stack-parser@^2.0.6:
   dependencies:
     stackframe "^1.1.1"
 
-es-abstract@^1.17.2, es-abstract@^1.18.0-next.1, es-abstract@^1.18.0-next.2, es-abstract@^1.18.2:
-  version "1.18.2"
-  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.2.tgz#6eb518b640262e8ddcbd48e0bc8549f82efd48a7"
-  integrity sha512-byRiNIQXE6HWNySaU6JohoNXzYgbBjztwFnBLUTiJmWXjaU9bSq3urQLUlNLQ292tc+gc07zYZXNZjaOoAX3sw==
-  dependencies:
-    call-bind "^1.0.2"
-    es-to-primitive "^1.2.1"
-    function-bind "^1.1.1"
-    get-intrinsic "^1.1.1"
-    has "^1.0.3"
-    has-symbols "^1.0.2"
-    is-callable "^1.2.3"
-    is-negative-zero "^2.0.1"
-    is-regex "^1.1.3"
-    is-string "^1.0.6"
-    object-inspect "^1.10.3"
-    object-keys "^1.1.1"
-    object.assign "^4.1.2"
-    string.prototype.trimend "^1.0.4"
-    string.prototype.trimstart "^1.0.4"
-    unbox-primitive "^1.0.1"
-
-es-abstract@^1.19.0, es-abstract@^1.20.4:
+es-abstract@^1.17.2, es-abstract@^1.18.0-next.2, es-abstract@^1.19.0, es-abstract@^1.20.4:
   version "1.20.4"
   resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.20.4.tgz#1d103f9f8d78d4cf0713edcd6d0ed1a46eed5861"
   integrity sha512-0UtvRN79eMe2L+UNEF1BwRe364sj/DXhQ/k5FmivgoSdpM90b8Jc0mDzKMGo7QS0BVbOP/bTwBKNnDc9rNzaPA==
@@ -8909,6 +7819,18 @@ es-abstract@^1.19.0, es-abstract@^1.20.4:
     string.prototype.trimstart "^1.0.5"
     unbox-primitive "^1.0.2"
 
+es-module-lexer@^1.2.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.3.0.tgz#6be9c9e0b4543a60cd166ff6f8b4e9dae0b0c16f"
+  integrity sha512-vZK7T0N2CBmBOixhmjdqx2gWVbFZ4DXZ/NyRMZVlJXPa7CyFS+/a4QQsDGDQy9ZfEzxFuNEsMLeQJnKP2p5/JA==
+
+es-shim-unscopables@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz#702e632193201e3edf8713635d083d378e510241"
+  integrity sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w==
+  dependencies:
+    has "^1.0.3"
+
 es-to-primitive@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a"
@@ -8918,33 +7840,7 @@ es-to-primitive@^1.2.1:
     is-date-object "^1.0.1"
     is-symbol "^1.0.2"
 
-es5-ext@^0.10.35, es5-ext@^0.10.50:
-  version "0.10.53"
-  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.53.tgz#93c5a3acfdbef275220ad72644ad02ee18368de1"
-  integrity sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==
-  dependencies:
-    es6-iterator "~2.0.3"
-    es6-symbol "~3.1.3"
-    next-tick "~1.0.0"
-
-es6-iterator@2.0.3, es6-iterator@~2.0.3:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7"
-  integrity sha1-p96IkUGgWpSwhUQDstCg+/qY87c=
-  dependencies:
-    d "1"
-    es5-ext "^0.10.35"
-    es6-symbol "^3.1.1"
-
-es6-symbol@^3.1.1, es6-symbol@~3.1.3:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.3.tgz#bad5d3c1bcdac28269f4cb331e431c78ac705d18"
-  integrity sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==
-  dependencies:
-    d "^1.0.1"
-    ext "^1.1.2"
-
-escalade@^3.0.2, escalade@^3.1.1:
+escalade@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
   integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==
@@ -8954,16 +7850,16 @@ escape-html@^1.0.3, escape-html@~1.0.3:
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
   integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==
 
-escape-string-regexp@2.0.0, escape-string-regexp@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344"
-  integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==
-
 escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
   integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=
 
+escape-string-regexp@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344"
+  integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==
+
 escape-string-regexp@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34"
@@ -8981,160 +7877,165 @@ escodegen@^2.0.0:
   optionalDependencies:
     source-map "~0.6.1"
 
-eslint-config-airbnb-base@^14.2.0, eslint-config-airbnb-base@^14.2.1:
-  version "14.2.1"
-  resolved "https://registry.yarnpkg.com/eslint-config-airbnb-base/-/eslint-config-airbnb-base-14.2.1.tgz#8a2eb38455dc5a312550193b319cdaeef042cd1e"
-  integrity sha512-GOrQyDtVEc1Xy20U7vsB2yAoB4nBlfH5HZJeatRXHleO+OS5Ot+MWij4Dpltw4/DyIkqUfqz1epfhVR5XWWQPA==
+eslint-config-airbnb-base@^15.0.0:
+  version "15.0.0"
+  resolved "https://registry.yarnpkg.com/eslint-config-airbnb-base/-/eslint-config-airbnb-base-15.0.0.tgz#6b09add90ac79c2f8d723a2580e07f3925afd236"
+  integrity sha512-xaX3z4ZZIcFLvh2oUNvcX5oEofXda7giYmuplVxoOg5A7EXJMrUyqRgR+mhDhPK8LZ4PttFOBvCYDbX3sUoUig==
   dependencies:
     confusing-browser-globals "^1.0.10"
     object.assign "^4.1.2"
-    object.entries "^1.1.2"
+    object.entries "^1.1.5"
+    semver "^6.3.0"
 
-eslint-config-airbnb-typescript@^12.3.1:
-  version "12.3.1"
-  resolved "https://registry.yarnpkg.com/eslint-config-airbnb-typescript/-/eslint-config-airbnb-typescript-12.3.1.tgz#83ab40d76402c208eb08516260d1d6fac8f8acbc"
-  integrity sha512-ql/Pe6/hppYuRp4m3iPaHJqkBB7dgeEmGPQ6X0UNmrQOfTF+dXw29/ZjU2kQ6RDoLxaxOA+Xqv07Vbef6oVTWw==
+eslint-config-airbnb-typescript@^17.0.0:
+  version "17.0.0"
+  resolved "https://registry.yarnpkg.com/eslint-config-airbnb-typescript/-/eslint-config-airbnb-typescript-17.0.0.tgz#360dbcf810b26bbcf2ff716198465775f1c49a07"
+  integrity sha512-elNiuzD0kPAPTXjFWg+lE24nMdHMtuxgYoD30OyMD6yrW1AhFZPAg27VX7d3tzOErw+dgJTNWfRSDqEcXb4V0g==
   dependencies:
-    "@typescript-eslint/parser" "^4.4.1"
-    eslint-config-airbnb "^18.2.0"
-    eslint-config-airbnb-base "^14.2.0"
+    eslint-config-airbnb-base "^15.0.0"
 
-eslint-config-airbnb@^18.2.0:
-  version "18.2.1"
-  resolved "https://registry.yarnpkg.com/eslint-config-airbnb/-/eslint-config-airbnb-18.2.1.tgz#b7fe2b42f9f8173e825b73c8014b592e449c98d9"
-  integrity sha512-glZNDEZ36VdlZWoxn/bUR1r/sdFKPd1mHPbqUtkctgNG4yT2DLLtJ3D+yCV+jzZCc2V1nBVkmdknOJBZ5Hc0fg==
+eslint-config-airbnb@19.0.4:
+  version "19.0.4"
+  resolved "https://registry.yarnpkg.com/eslint-config-airbnb/-/eslint-config-airbnb-19.0.4.tgz#84d4c3490ad70a0ffa571138ebcdea6ab085fdc3"
+  integrity sha512-T75QYQVQX57jiNgpF9r1KegMICE94VYwoFQyMGhrvc+lB8YF2E/M/PYDaQe1AJcWaEgqLE+ErXV1Og/+6Vyzew==
   dependencies:
-    eslint-config-airbnb-base "^14.2.1"
+    eslint-config-airbnb-base "^15.0.0"
     object.assign "^4.1.2"
-    object.entries "^1.1.2"
-
-eslint-config-prettier@^8.3.0:
-  version "8.3.0"
-  resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz#f7471b20b6fe8a9a9254cc684454202886a2dd7a"
-  integrity sha512-BgZuLUSeKzvlL/VUjx/Yb787VQ26RU3gGjA3iiFvdsp/2bMfVIWUVP7tjxtjS0e+HP409cPlPvNkQloz8C91ew==
+    object.entries "^1.1.5"
 
-eslint-config-react-app@^6.0.0:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/eslint-config-react-app/-/eslint-config-react-app-6.0.0.tgz#ccff9fc8e36b322902844cbd79197982be355a0e"
-  integrity sha512-bpoAAC+YRfzq0dsTk+6v9aHm/uqnDwayNAXleMypGl6CpxI9oXXscVHo4fk3eJPIn+rsbtNetB4r/ZIidFIE8A==
-  dependencies:
-    confusing-browser-globals "^1.0.10"
+eslint-config-prettier@^8.8.0:
+  version "8.8.0"
+  resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-8.8.0.tgz#bfda738d412adc917fd7b038857110efe98c9348"
+  integrity sha512-wLbQiFre3tdGgpDv67NQKnJuTlcUVYHas3k+DZCc2U2BadthoEY4B7hLPvAxaqdyOGCzuLfii2fqGph10va7oA==
 
-eslint-import-resolver-node@^0.3.4:
-  version "0.3.4"
-  resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz#85ffa81942c25012d8231096ddf679c03042c717"
-  integrity sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==
+eslint-config-react-app@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/eslint-config-react-app/-/eslint-config-react-app-7.0.1.tgz#73ba3929978001c5c86274c017ea57eb5fa644b4"
+  integrity sha512-K6rNzvkIeHaTd8m/QEh1Zko0KI7BACWkkneSs6s9cKZC/J27X3eZR6Upt1jkmZ/4FK+XUOPPxMEN7+lbUXfSlA==
+  dependencies:
+    "@babel/core" "^7.16.0"
+    "@babel/eslint-parser" "^7.16.3"
+    "@rushstack/eslint-patch" "^1.1.0"
+    "@typescript-eslint/eslint-plugin" "^5.5.0"
+    "@typescript-eslint/parser" "^5.5.0"
+    babel-preset-react-app "^10.0.1"
+    confusing-browser-globals "^1.0.11"
+    eslint-plugin-flowtype "^8.0.3"
+    eslint-plugin-import "^2.25.3"
+    eslint-plugin-jest "^25.3.0"
+    eslint-plugin-jsx-a11y "^6.5.1"
+    eslint-plugin-react "^7.27.1"
+    eslint-plugin-react-hooks "^4.3.0"
+    eslint-plugin-testing-library "^5.0.1"
+
+eslint-import-resolver-node@^0.3.7:
+  version "0.3.7"
+  resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.7.tgz#83b375187d412324a1963d84fa664377a23eb4d7"
+  integrity sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==
   dependencies:
-    debug "^2.6.9"
-    resolve "^1.13.1"
+    debug "^3.2.7"
+    is-core-module "^2.11.0"
+    resolve "^1.22.1"
 
-eslint-module-utils@^2.6.1:
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.6.1.tgz#b51be1e473dd0de1c5ea638e22429c2490ea8233"
-  integrity sha512-ZXI9B8cxAJIH4nfkhTwcRTEAnrVfobYqwjWy/QMCZ8rHkZHFjf9yO4BzpiF9kCSfNlMG54eKigISHpX0+AaT4A==
+eslint-module-utils@^2.7.4:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz#e439fee65fc33f6bba630ff621efc38ec0375c49"
+  integrity sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==
   dependencies:
     debug "^3.2.7"
-    pkg-dir "^2.0.0"
 
-eslint-plugin-flowtype@^5.2.0:
-  version "5.7.2"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-flowtype/-/eslint-plugin-flowtype-5.7.2.tgz#482a42fe5d15ee614652ed256d37543d584d7bc0"
-  integrity sha512-7Oq/N0+3nijBnYWQYzz/Mp/7ZCpwxYvClRyW/PLAmimY9uLCBvoXsNsERcJdkKceyOjgRbFhhxs058KTrne9Mg==
+eslint-plugin-flowtype@^8.0.3:
+  version "8.0.3"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-flowtype/-/eslint-plugin-flowtype-8.0.3.tgz#e1557e37118f24734aa3122e7536a038d34a4912"
+  integrity sha512-dX8l6qUL6O+fYPtpNRideCFSpmWOUVx5QcaGLVqe/vlDiBSe4vYljDWDETwnyFzpl7By/WVIu6rcrniCgH9BqQ==
   dependencies:
-    lodash "^4.17.15"
+    lodash "^4.17.21"
     string-natural-compare "^3.0.1"
 
-eslint-plugin-import@^2.22.1:
-  version "2.23.3"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.23.3.tgz#8a1b073289fff03c4af0f04b6df956b7d463e191"
-  integrity sha512-wDxdYbSB55F7T5CC7ucDjY641VvKmlRwT0Vxh7PkY1mI4rclVRFWYfsrjDgZvwYYDZ5ee0ZtfFKXowWjqvEoRQ==
+eslint-plugin-import@^2.25.3:
+  version "2.27.5"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz#876a6d03f52608a3e5bb439c2550588e51dd6c65"
+  integrity sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==
   dependencies:
-    array-includes "^3.1.3"
-    array.prototype.flat "^1.2.4"
-    debug "^2.6.9"
+    array-includes "^3.1.6"
+    array.prototype.flat "^1.3.1"
+    array.prototype.flatmap "^1.3.1"
+    debug "^3.2.7"
     doctrine "^2.1.0"
-    eslint-import-resolver-node "^0.3.4"
-    eslint-module-utils "^2.6.1"
-    find-up "^2.0.0"
+    eslint-import-resolver-node "^0.3.7"
+    eslint-module-utils "^2.7.4"
     has "^1.0.3"
-    is-core-module "^2.4.0"
-    minimatch "^3.0.4"
-    object.values "^1.1.3"
-    pkg-up "^2.0.0"
-    read-pkg-up "^3.0.0"
-    resolve "^1.20.0"
-    tsconfig-paths "^3.9.0"
+    is-core-module "^2.11.0"
+    is-glob "^4.0.3"
+    minimatch "^3.1.2"
+    object.values "^1.1.6"
+    resolve "^1.22.1"
+    semver "^6.3.0"
+    tsconfig-paths "^3.14.1"
 
-eslint-plugin-jest@^24.1.0:
-  version "24.3.6"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-24.3.6.tgz#5f0ca019183c3188c5ad3af8e80b41de6c8e9173"
-  integrity sha512-WOVH4TIaBLIeCX576rLcOgjNXqP+jNlCiEmRgFTfQtJ52DpwnIQKAVGlGPAN7CZ33bW6eNfHD6s8ZbEUTQubJg==
+eslint-plugin-jest@^25.3.0:
+  version "25.7.0"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-25.7.0.tgz#ff4ac97520b53a96187bad9c9814e7d00de09a6a"
+  integrity sha512-PWLUEXeeF7C9QGKqvdSbzLOiLTx+bno7/HC9eefePfEb257QFHg7ye3dh80AZVkaa/RQsBB1Q/ORQvg2X7F0NQ==
   dependencies:
-    "@typescript-eslint/experimental-utils" "^4.0.1"
+    "@typescript-eslint/experimental-utils" "^5.0.0"
 
-eslint-plugin-jsx-a11y@^6.3.1:
-  version "6.4.1"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.4.1.tgz#a2d84caa49756942f42f1ffab9002436391718fd"
-  integrity sha512-0rGPJBbwHoGNPU73/QCLP/vveMlM1b1Z9PponxO87jfr6tuH5ligXbDT6nHSSzBC8ovX2Z+BQu7Bk5D/Xgq9zg==
+eslint-plugin-jsx-a11y@^6.5.1:
+  version "6.7.1"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.7.1.tgz#fca5e02d115f48c9a597a6894d5bcec2f7a76976"
+  integrity sha512-63Bog4iIethyo8smBklORknVjB0T2dwB8Mr/hIC+fBS0uyHdYYpzM/Ed+YC8VxTjlXHEWFOdmgwcDn1U2L9VCA==
   dependencies:
-    "@babel/runtime" "^7.11.2"
-    aria-query "^4.2.2"
-    array-includes "^3.1.1"
+    "@babel/runtime" "^7.20.7"
+    aria-query "^5.1.3"
+    array-includes "^3.1.6"
+    array.prototype.flatmap "^1.3.1"
     ast-types-flow "^0.0.7"
-    axe-core "^4.0.2"
-    axobject-query "^2.2.0"
-    damerau-levenshtein "^1.0.6"
-    emoji-regex "^9.0.0"
+    axe-core "^4.6.2"
+    axobject-query "^3.1.1"
+    damerau-levenshtein "^1.0.8"
+    emoji-regex "^9.2.2"
     has "^1.0.3"
-    jsx-ast-utils "^3.1.0"
-    language-tags "^1.0.5"
-
-eslint-plugin-prettier@^3.4.0:
-  version "3.4.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz#cdbad3bf1dbd2b177e9825737fe63b476a08f0c7"
-  integrity sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==
-  dependencies:
-    prettier-linter-helpers "^1.0.0"
+    jsx-ast-utils "^3.3.3"
+    language-tags "=1.0.5"
+    minimatch "^3.1.2"
+    object.entries "^1.1.6"
+    object.fromentries "^2.0.6"
+    semver "^6.3.0"
 
-eslint-plugin-react-hooks@^4.2.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.2.0.tgz#8c229c268d468956334c943bb45fc860280f5556"
-  integrity sha512-623WEiZJqxR7VdxFCKLI6d6LLpwJkGPYKODnkH3D7WpOG5KM8yWueBd8TLsNAetEJNF5iJmolaAKO3F8yzyVBQ==
+eslint-plugin-react-hooks@^4.3.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz#4c3e697ad95b77e93f8646aaa1630c1ba607edd3"
+  integrity sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==
 
-eslint-plugin-react@^7.21.5, eslint-plugin-react@^7.23.2:
-  version "7.23.2"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.23.2.tgz#2d2291b0f95c03728b55869f01102290e792d494"
-  integrity sha512-AfjgFQB+nYszudkxRkTFu0UR1zEQig0ArVMPloKhxwlwkzaw/fBiH0QWcBBhZONlXqQC51+nfqFrkn4EzHcGBw==
+eslint-plugin-react@^7.27.1, eslint-plugin-react@^7.28.0:
+  version "7.32.2"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.32.2.tgz#e71f21c7c265ebce01bcbc9d0955170c55571f10"
+  integrity sha512-t2fBMa+XzonrrNkyVirzKlvn5RXzzPwRHtMvLAtVZrt8oxgnTQaYbU6SXTOO1mwQgp1y5+toMSKInnzGr0Knqg==
   dependencies:
-    array-includes "^3.1.3"
-    array.prototype.flatmap "^1.2.4"
+    array-includes "^3.1.6"
+    array.prototype.flatmap "^1.3.1"
+    array.prototype.tosorted "^1.1.1"
     doctrine "^2.1.0"
-    has "^1.0.3"
+    estraverse "^5.3.0"
     jsx-ast-utils "^2.4.1 || ^3.0.0"
-    minimatch "^3.0.4"
-    object.entries "^1.1.3"
-    object.fromentries "^2.0.4"
-    object.values "^1.1.3"
-    prop-types "^15.7.2"
-    resolve "^2.0.0-next.3"
-    string.prototype.matchall "^4.0.4"
-
-eslint-plugin-testing-library@^3.9.2:
-  version "3.10.2"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-testing-library/-/eslint-plugin-testing-library-3.10.2.tgz#609ec2b0369da7cf2e6d9edff5da153cc31d87bd"
-  integrity sha512-WAmOCt7EbF1XM8XfbCKAEzAPnShkNSwcIsAD2jHdsMUT9mZJPjLCG7pMzbcC8kK366NOuGip8HKLDC+Xk4yIdA==
-  dependencies:
-    "@typescript-eslint/experimental-utils" "^3.10.1"
+    minimatch "^3.1.2"
+    object.entries "^1.1.6"
+    object.fromentries "^2.0.6"
+    object.hasown "^1.1.2"
+    object.values "^1.1.6"
+    prop-types "^15.8.1"
+    resolve "^2.0.0-next.4"
+    semver "^6.3.0"
+    string.prototype.matchall "^4.0.8"
 
-eslint-scope@^4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-4.0.3.tgz#ca03833310f6889a3264781aa82e63eb9cfe7848"
-  integrity sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==
+eslint-plugin-testing-library@^5.0.1:
+  version "5.11.0"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-testing-library/-/eslint-plugin-testing-library-5.11.0.tgz#0bad7668e216e20dd12f8c3652ca353009163121"
+  integrity sha512-ELY7Gefo+61OfXKlQeXNIDVVLPcvKTeiQOoMZG9TeuWa7Ln4dUNRv8JdRWBQI9Mbb427XGlVB1aa1QPZxBJM8Q==
   dependencies:
-    esrecurse "^4.1.0"
-    estraverse "^4.1.1"
+    "@typescript-eslint/utils" "^5.58.0"
 
-eslint-scope@^5.0.0, eslint-scope@^5.1.1:
+eslint-scope@5.1.1, eslint-scope@^5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c"
   integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==
@@ -9142,102 +8043,102 @@ eslint-scope@^5.0.0, eslint-scope@^5.1.1:
     esrecurse "^4.3.0"
     estraverse "^4.1.1"
 
-eslint-utils@^2.0.0, eslint-utils@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.1.0.tgz#d2de5e03424e707dc10c74068ddedae708741b27"
-  integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==
+eslint-scope@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.0.tgz#f21ebdafda02352f103634b96dd47d9f81ca117b"
+  integrity sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==
   dependencies:
-    eslint-visitor-keys "^1.1.0"
-
-eslint-visitor-keys@^1.0.0, eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e"
-  integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==
+    esrecurse "^4.3.0"
+    estraverse "^5.2.0"
 
-eslint-visitor-keys@^2.0.0:
+eslint-visitor-keys@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303"
   integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==
 
-eslint-webpack-plugin@^2.5.2:
-  version "2.5.4"
-  resolved "https://registry.yarnpkg.com/eslint-webpack-plugin/-/eslint-webpack-plugin-2.5.4.tgz#473b84932f1a8e2c2b8e66a402d0497bf440b986"
-  integrity sha512-7rYh0m76KyKSDE+B+2PUQrlNS4HJ51t3WKpkJg6vo2jFMbEPTG99cBV0Dm7LXSHucN4WGCG65wQcRiTFrj7iWw==
-  dependencies:
-    "@types/eslint" "^7.2.6"
-    arrify "^2.0.1"
-    jest-worker "^26.6.2"
-    micromatch "^4.0.2"
-    normalize-path "^3.0.0"
-    schema-utils "^3.0.0"
+eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1:
+  version "3.4.1"
+  resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz#c22c48f48942d08ca824cc526211ae400478a994"
+  integrity sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==
 
-eslint@^7.11.0, eslint@^7.27.0:
-  version "7.27.0"
-  resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.27.0.tgz#665a1506d8f95655c9274d84bd78f7166b07e9c7"
-  integrity sha512-JZuR6La2ZF0UD384lcbnd0Cgg6QJjiCwhMD6eU4h/VGPcVGwawNNzKU41tgokGXnfjOOyI6QIffthhJTPzzuRA==
+eslint-webpack-plugin@^3.1.1:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/eslint-webpack-plugin/-/eslint-webpack-plugin-3.2.0.tgz#1978cdb9edc461e4b0195a20da950cf57988347c"
+  integrity sha512-avrKcGncpPbPSUHX6B3stNGzkKFto3eL+DKM4+VyMrVnhPc3vRczVlCq3uhuFOdRvDHTVXuzwk1ZKUrqDQHQ9w==
   dependencies:
-    "@babel/code-frame" "7.12.11"
-    "@eslint/eslintrc" "^0.4.1"
+    "@types/eslint" "^7.29.0 || ^8.4.1"
+    jest-worker "^28.0.2"
+    micromatch "^4.0.5"
+    normalize-path "^3.0.0"
+    schema-utils "^4.0.0"
+
+eslint@^8.2.0, eslint@^8.3.0:
+  version "8.43.0"
+  resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.43.0.tgz#3e8c6066a57097adfd9d390b8fc93075f257a094"
+  integrity sha512-aaCpf2JqqKesMFGgmRPessmVKjcGXqdlAYLLC3THM8t5nBRZRQ+st5WM/hoJXkdioEXLLbXgclUpM0TXo5HX5Q==
+  dependencies:
+    "@eslint-community/eslint-utils" "^4.2.0"
+    "@eslint-community/regexpp" "^4.4.0"
+    "@eslint/eslintrc" "^2.0.3"
+    "@eslint/js" "8.43.0"
+    "@humanwhocodes/config-array" "^0.11.10"
+    "@humanwhocodes/module-importer" "^1.0.1"
+    "@nodelib/fs.walk" "^1.2.8"
     ajv "^6.10.0"
     chalk "^4.0.0"
     cross-spawn "^7.0.2"
-    debug "^4.0.1"
+    debug "^4.3.2"
     doctrine "^3.0.0"
-    enquirer "^2.3.5"
     escape-string-regexp "^4.0.0"
-    eslint-scope "^5.1.1"
-    eslint-utils "^2.1.0"
-    eslint-visitor-keys "^2.0.0"
-    espree "^7.3.1"
-    esquery "^1.4.0"
+    eslint-scope "^7.2.0"
+    eslint-visitor-keys "^3.4.1"
+    espree "^9.5.2"
+    esquery "^1.4.2"
     esutils "^2.0.2"
     fast-deep-equal "^3.1.3"
     file-entry-cache "^6.0.1"
-    functional-red-black-tree "^1.0.1"
-    glob-parent "^5.0.0"
-    globals "^13.6.0"
-    ignore "^4.0.6"
+    find-up "^5.0.0"
+    glob-parent "^6.0.2"
+    globals "^13.19.0"
+    graphemer "^1.4.0"
+    ignore "^5.2.0"
     import-fresh "^3.0.0"
     imurmurhash "^0.1.4"
     is-glob "^4.0.0"
-    js-yaml "^3.13.1"
+    is-path-inside "^3.0.3"
+    js-yaml "^4.1.0"
     json-stable-stringify-without-jsonify "^1.0.1"
     levn "^0.4.1"
     lodash.merge "^4.6.2"
-    minimatch "^3.0.4"
+    minimatch "^3.1.2"
     natural-compare "^1.4.0"
     optionator "^0.9.1"
-    progress "^2.0.0"
-    regexpp "^3.1.0"
-    semver "^7.2.1"
-    strip-ansi "^6.0.0"
+    strip-ansi "^6.0.1"
     strip-json-comments "^3.1.0"
-    table "^6.0.9"
     text-table "^0.2.0"
-    v8-compile-cache "^2.0.3"
 
-espree@^7.3.0, espree@^7.3.1:
-  version "7.3.1"
-  resolved "https://registry.yarnpkg.com/espree/-/espree-7.3.1.tgz#f2df330b752c6f55019f8bd89b7660039c1bbbb6"
-  integrity sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==
+espree@^9.5.2:
+  version "9.5.2"
+  resolved "https://registry.yarnpkg.com/espree/-/espree-9.5.2.tgz#e994e7dc33a082a7a82dceaf12883a829353215b"
+  integrity sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==
   dependencies:
-    acorn "^7.4.0"
-    acorn-jsx "^5.3.1"
-    eslint-visitor-keys "^1.3.0"
+    acorn "^8.8.0"
+    acorn-jsx "^5.3.2"
+    eslint-visitor-keys "^3.4.1"
 
 esprima@^4.0.0, esprima@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
   integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
 
-esquery@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5"
-  integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==
+esquery@^1.4.2:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.5.0.tgz#6ce17738de8577694edd7361c57182ac8cb0db0b"
+  integrity sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==
   dependencies:
     estraverse "^5.1.0"
 
-esrecurse@^4.1.0, esrecurse@^4.3.0:
+esrecurse@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921"
   integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==
@@ -9249,15 +8150,10 @@ estraverse@^4.1.1:
   resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d"
   integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==
 
-estraverse@^5.1.0, estraverse@^5.2.0:
-  version "5.2.0"
-  resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880"
-  integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==
-
-estree-walker@^0.6.1:
-  version "0.6.1"
-  resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362"
-  integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==
+estraverse@^5.1.0, estraverse@^5.2.0, estraverse@^5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123"
+  integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==
 
 estree-walker@^1.0.1:
   version "1.0.1"
@@ -9302,31 +8198,11 @@ eventemitter3@^4.0.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
   integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
 
-events@^3.0.0:
+events@^3.2.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
   integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==
 
-eventsource@^1.0.7:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/eventsource/-/eventsource-1.1.1.tgz#4544a35a57d7120fba4fa4c86cb4023b2c09df2f"
-  integrity sha512-qV5ZC0h7jYIAOhArFJgSfdyz6rALJyb270714o7ZtNnw2WSJ+eexhKtE0O8LYPRsHZHf2osHKZBxGPvm3kPkCA==
-  dependencies:
-    original "^1.0.0"
-
-evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02"
-  integrity sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==
-  dependencies:
-    md5.js "^1.3.4"
-    safe-buffer "^5.1.1"
-
-exec-sh@^0.3.2:
-  version "0.3.6"
-  resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.6.tgz#ff264f9e325519a60cb5e273692943483cca63bc"
-  integrity sha512-nQn+hI3yp+oD0huYhKwvYI32+JFeq+XkNcD1GAo3Y/MjxsfVGmrrzrnzjWiNY6f+pUCP440fThsFh5gZrRAU/w==
-
 execa@5.0.0:
   version "5.0.0"
   resolved "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz#4029b0007998a841fbd1032e5f4de86a3c1e3376"
@@ -9342,32 +8218,19 @@ execa@5.0.0:
     signal-exit "^3.0.3"
     strip-final-newline "^2.0.0"
 
-execa@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
-  integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==
-  dependencies:
-    cross-spawn "^6.0.0"
-    get-stream "^4.0.0"
-    is-stream "^1.1.0"
-    npm-run-path "^2.0.0"
-    p-finally "^1.0.0"
-    signal-exit "^3.0.0"
-    strip-eof "^1.0.0"
-
-execa@^4.0.0:
-  version "4.1.0"
-  resolved "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a"
-  integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==
+execa@^5.0.0:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd"
+  integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==
   dependencies:
-    cross-spawn "^7.0.0"
-    get-stream "^5.0.0"
-    human-signals "^1.1.1"
+    cross-spawn "^7.0.3"
+    get-stream "^6.0.0"
+    human-signals "^2.1.0"
     is-stream "^2.0.0"
     merge-stream "^2.0.0"
-    npm-run-path "^4.0.0"
-    onetime "^5.1.0"
-    signal-exit "^3.0.2"
+    npm-run-path "^4.0.1"
+    onetime "^5.1.2"
+    signal-exit "^3.0.3"
     strip-final-newline "^2.0.0"
 
 exit@^0.1.2:
@@ -9388,19 +8251,17 @@ expand-brackets@^2.1.4:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-expect@^26.6.0, expect@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/expect/-/expect-26.6.2.tgz#c6b996bf26bf3fe18b67b2d0f51fc981ba934417"
-  integrity sha512-9/hlOBkQl2l/PLHJx6JjoDF6xPKcJEsUlWKb23rKE7KzeDqUZKXKNMW27KIue5JMdBV9HgmoJPcc8HtO85t9IA==
+expect@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/expect/-/expect-27.5.1.tgz#83ce59f1e5bdf5f9d2b94b61d2050db48f3fef74"
+  integrity sha512-E1q5hSUG2AmYQwQJ041nvgpkODHQvB+RKlB4IYdru6uJsyFTRyZAP463M+1lINorwbqAmUggi6+WwkD8lCS/Dw==
   dependencies:
-    "@jest/types" "^26.6.2"
-    ansi-styles "^4.0.0"
-    jest-get-type "^26.3.0"
-    jest-matcher-utils "^26.6.2"
-    jest-message-util "^26.6.2"
-    jest-regex-util "^26.0.0"
+    "@jest/types" "^27.5.1"
+    jest-get-type "^27.5.1"
+    jest-matcher-utils "^27.5.1"
+    jest-message-util "^27.5.1"
 
-express@^4.17.1:
+express@^4.17.3:
   version "4.18.2"
   resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59"
   integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==
@@ -9437,13 +8298,6 @@ express@^4.17.1:
     utils-merge "1.0.1"
     vary "~1.1.2"
 
-ext@^1.1.2:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/ext/-/ext-1.4.0.tgz#89ae7a07158f79d35517882904324077e4379244"
-  integrity sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==
-  dependencies:
-    type "^2.0.0"
-
 extend-shallow@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f"
@@ -9522,27 +8376,21 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3:
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
   integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==
 
-fast-diff@^1.1.2:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03"
-  integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==
-
-fast-glob@^3.1.1:
-  version "3.2.5"
-  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.5.tgz#7939af2a656de79a4f1901903ee8adcaa7cb9661"
-  integrity sha512-2DtFcgT68wiTTiwZ2hNdJfcHNke9XOfnwmBRWXhmeKM8rF0TGwmC/Qto3S7RoZKp5cilZbxzO5iTNTQsJ+EeDg==
+fast-glob@^3.1.1, fast-glob@^3.2.12, fast-glob@^3.2.9:
+  version "3.2.12"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.12.tgz#7f39ec99c2e6ab030337142da9e0c18f37afae80"
+  integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==
   dependencies:
     "@nodelib/fs.stat" "^2.0.2"
     "@nodelib/fs.walk" "^1.2.3"
-    glob-parent "^5.1.0"
+    glob-parent "^5.1.2"
     merge2 "^1.3.0"
-    micromatch "^4.0.2"
-    picomatch "^2.2.1"
+    micromatch "^4.0.4"
 
-fast-glob@^3.2.4:
-  version "3.2.11"
-  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9"
-  integrity sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==
+fast-glob@^3.2.11:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.0.tgz#7c40cb491e1e2ed5664749e87bfb516dbe8727c0"
+  integrity sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA==
   dependencies:
     "@nodelib/fs.stat" "^2.0.2"
     "@nodelib/fs.walk" "^1.2.3"
@@ -9621,11 +8469,6 @@ fbjs@^3.0.0:
     setimmediate "^1.0.5"
     ua-parser-js "^0.7.30"
 
-figgy-pudding@^3.5.1:
-  version "3.5.2"
-  resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e"
-  integrity sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==
-
 figures@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
@@ -9655,30 +8498,25 @@ file-entry-cache@^6.0.1:
   dependencies:
     flat-cache "^3.0.4"
 
-file-loader@6.1.1:
-  version "6.1.1"
-  resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-6.1.1.tgz#a6f29dfb3f5933a1c350b2dbaa20ac5be0539baa"
-  integrity sha512-Klt8C4BjWSXYQAfhpYYkG4qHNTna4toMHEbWrI5IuVoxbU6uiDKeKAP99R8mmbJi3lvewn/jQBOgU4+NS3tDQw==
+file-loader@^6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-6.2.0.tgz#baef7cf8e1840df325e4390b4484879480eebe4d"
+  integrity sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw==
   dependencies:
     loader-utils "^2.0.0"
     schema-utils "^3.0.0"
 
-file-uri-to-path@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
-  integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
-
-filelist@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.2.tgz#80202f21462d4d1c2e214119b1807c1bc0380e5b"
-  integrity sha512-z7O0IS8Plc39rTCq6i6iHxk43duYOn8uFJiWSewIq0Bww1RNybVHSCjahmcC87ZqAm4OTvFzlzeGu3XAzG1ctQ==
+filelist@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.4.tgz#f78978a1e944775ff9e62e744424f215e58352b5"
+  integrity sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==
   dependencies:
-    minimatch "^3.0.4"
+    minimatch "^5.0.1"
 
-filesize@6.1.0:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/filesize/-/filesize-6.1.0.tgz#e81bdaa780e2451d714d71c0d7a4f3238d37ad00"
-  integrity sha512-LpCHtPQ3sFx67z+uh2HnSyWSLLu5Jxo21795uRDuar/EOuYWXib5EmPaGIBuSnRqH2IODiKA2k5re/K9OnN/Yg==
+filesize@^8.0.6:
+  version "8.0.7"
+  resolved "https://registry.yarnpkg.com/filesize/-/filesize-8.0.7.tgz#695e70d80f4e47012c132d57a059e80c6b580bd8"
+  integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ==
 
 fill-range@^4.0.0:
   version "4.0.0"
@@ -9715,15 +8553,6 @@ finalhandler@1.2.0:
     statuses "2.0.1"
     unpipe "~1.0.0"
 
-find-cache-dir@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.1.0.tgz#8d0f94cd13fe43c6c7c261a0d86115ca918c05f7"
-  integrity sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==
-  dependencies:
-    commondir "^1.0.1"
-    make-dir "^2.0.0"
-    pkg-dir "^3.0.0"
-
 find-cache-dir@^3.3.1:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880"
@@ -9738,7 +8567,14 @@ find-root@^1.1.0:
   resolved "https://registry.yarnpkg.com/find-root/-/find-root-1.1.0.tgz#abcfc8ba76f708c42a97b3d685b7e9450bfb9ce4"
   integrity sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==
 
-find-up@4.1.0, find-up@^4.0.0, find-up@^4.1.0:
+find-up@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
+  integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
+  dependencies:
+    locate-path "^3.0.0"
+
+find-up@^4.0.0, find-up@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19"
   integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==
@@ -9746,19 +8582,13 @@ find-up@4.1.0, find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
-find-up@^2.0.0, find-up@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7"
-  integrity sha1-RdG35QbHF93UgndaK3eSCjwMV6c=
-  dependencies:
-    locate-path "^2.0.0"
-
-find-up@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
-  integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
+find-up@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc"
+  integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==
   dependencies:
-    locate-path "^3.0.0"
+    locate-path "^6.0.0"
+    path-exists "^4.0.0"
 
 find-webpack@2.2.1:
   version "2.2.1"
@@ -9790,19 +8620,6 @@ flatted@^3.1.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.1.tgz#c4b489e80096d9df1dfc97c79871aea7c617c469"
   integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==
 
-flatten@^1.0.2:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b"
-  integrity sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg==
-
-flush-write-stream@^1.0.0:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
-  integrity sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==
-  dependencies:
-    inherits "^2.0.3"
-    readable-stream "^2.3.6"
-
 focus-lock@^0.9.1:
   version "0.9.2"
   resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-0.9.2.tgz#9d30918aaa99b1b97677731053d017f82a540d5b"
@@ -9830,18 +8647,24 @@ forever-agent@~0.6.1:
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
   integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
 
-fork-ts-checker-webpack-plugin@4.1.6:
-  version "4.1.6"
-  resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-4.1.6.tgz#5055c703febcf37fa06405d400c122b905167fc5"
-  integrity sha512-DUxuQaKoqfNne8iikd14SAkh5uw4+8vNifp6gmA73yYNS6ywLIWSLD/n/mBzHQRpW3J7rbATEakmiA8JvkTyZw==
+fork-ts-checker-webpack-plugin@^6.5.0:
+  version "6.5.3"
+  resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-6.5.3.tgz#eda2eff6e22476a2688d10661688c47f611b37f3"
+  integrity sha512-SbH/l9ikmMWycd5puHJKTkZJKddF4iRLyW3DeZ08HTI7NGyLS38MXd/KGgeWumQO7YNQbW2u/NtPT2YowbPaGQ==
   dependencies:
-    "@babel/code-frame" "^7.5.5"
-    chalk "^2.4.1"
-    micromatch "^3.1.10"
+    "@babel/code-frame" "^7.8.3"
+    "@types/json-schema" "^7.0.5"
+    chalk "^4.1.0"
+    chokidar "^3.4.2"
+    cosmiconfig "^6.0.0"
+    deepmerge "^4.2.2"
+    fs-extra "^9.0.0"
+    glob "^7.1.6"
+    memfs "^3.1.2"
     minimatch "^3.0.4"
-    semver "^5.6.0"
+    schema-utils "2.7.0"
+    semver "^7.3.2"
     tapable "^1.0.0"
-    worker-rpc "^0.1.0"
 
 form-data@4.0.0:
   version "4.0.0"
@@ -9880,6 +8703,11 @@ forwarded@0.2.0:
   resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811"
   integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==
 
+fraction.js@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.2.0.tgz#448e5109a313a3527f5a3ab2119ec4cf0e0e2950"
+  integrity sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==
+
 fragment-cache@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19"
@@ -9892,19 +8720,20 @@ fresh@0.5.2:
   resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
   integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==
 
-from2@^2.1.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af"
-  integrity sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=
-  dependencies:
-    inherits "^2.0.1"
-    readable-stream "^2.0.0"
-
 from@~0:
   version "0.1.7"
   resolved "https://registry.npmjs.org/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe"
   integrity sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=
 
+fs-extra@^10.0.0:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
+  integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs-extra@^4.0.3:
   version "4.0.3"
   resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz#0d852122e5bc5beb453fb028e9c0c9bf36340c94"
@@ -9914,27 +8743,9 @@ fs-extra@^4.0.3:
     jsonfile "^4.0.0"
     universalify "^0.1.0"
 
-fs-extra@^7.0.0:
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
-  integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==
-  dependencies:
-    graceful-fs "^4.1.2"
-    jsonfile "^4.0.0"
-    universalify "^0.1.0"
-
-fs-extra@^8.1.0:
-  version "8.1.0"
-  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0"
-  integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==
-  dependencies:
-    graceful-fs "^4.2.0"
-    jsonfile "^4.0.0"
-    universalify "^0.1.0"
-
-fs-extra@^9.0.1:
+fs-extra@^9.0.0, fs-extra@^9.0.1:
   version "9.1.0"
-  resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d"
   integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==
   dependencies:
     at-least-node "^1.0.0"
@@ -9942,37 +8753,17 @@ fs-extra@^9.0.1:
     jsonfile "^6.0.1"
     universalify "^2.0.0"
 
-fs-minipass@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb"
-  integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==
-  dependencies:
-    minipass "^3.0.0"
-
-fs-write-stream-atomic@^1.0.8:
-  version "1.0.10"
-  resolved "https://registry.yarnpkg.com/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz#b47df53493ef911df75731e70a9ded0189db40c9"
-  integrity sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=
-  dependencies:
-    graceful-fs "^4.1.2"
-    iferr "^0.1.5"
-    imurmurhash "^0.1.4"
-    readable-stream "1 || 2"
+fs-monkey@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.4.tgz#ee8c1b53d3fe8bb7e5d2c5c5dfc0168afdd2f747"
+  integrity sha512-INM/fWAxMICjttnD0DX1rBvinKskj5G1w+oy/pnm9u/tSlnBrzFonJMcalKJ30P8RRsPzKcCG7Q8l0jx5Fh9YQ==
 
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
 
-fsevents@^1.2.7:
-  version "1.2.13"
-  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.2.13.tgz#f325cb0455592428bcf11b383370ef70e3bfcc38"
-  integrity sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==
-  dependencies:
-    bindings "^1.5.0"
-    nan "^2.12.1"
-
-fsevents@^2.1.2, fsevents@^2.1.3, fsevents@~2.3.1:
+fsevents@^2.3.2, fsevents@~2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
@@ -9992,17 +8783,12 @@ function.prototype.name@^1.1.5:
     es-abstract "^1.19.0"
     functions-have-names "^1.2.2"
 
-functional-red-black-tree@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
-  integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=
-
 functions-have-names@^1.2.2:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834"
   integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==
 
-gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
+gensync@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
   integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==
@@ -10017,16 +8803,7 @@ get-dom-document@^0.1.3:
   resolved "https://registry.yarnpkg.com/get-dom-document/-/get-dom-document-0.1.3.tgz#d0188090e43d38dd146c467ac6e3e1f2ace7af52"
   integrity sha512-bZ0O00gSQgMo+wz7gU6kbbWCPh4dfDsL9ZOmVhA8TOXszl5GV56TpTuW1/Qq/QctgpjK56yyvB1vBO+wzz8Szw==
 
-get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6"
-  integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==
-  dependencies:
-    function-bind "^1.1.1"
-    has "^1.0.3"
-    has-symbols "^1.0.1"
-
-get-intrinsic@^1.1.3:
+get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385"
   integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==
@@ -10045,14 +8822,14 @@ get-package-type@^0.1.0:
   resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a"
   integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==
 
-get-stream@^4.0.0, get-stream@^4.1.0:
+get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
   integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==
   dependencies:
     pump "^3.0.0"
 
-get-stream@^5.0.0, get-stream@^5.1.0:
+get-stream@^5.1.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3"
   integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==
@@ -10084,25 +8861,29 @@ getpass@^0.1.1:
   dependencies:
     assert-plus "^1.0.0"
 
-glob-parent@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae"
-  integrity sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=
-  dependencies:
-    is-glob "^3.1.0"
-    path-dirname "^1.0.0"
-
-glob-parent@^5.0.0, glob-parent@^5.1.0, glob-parent@^5.1.1, glob-parent@^5.1.2, glob-parent@~5.1.0:
+glob-parent@^5.1.2, glob-parent@~5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
   integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
   dependencies:
     is-glob "^4.0.1"
 
-glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6:
-  version "7.1.7"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90"
-  integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==
+glob-parent@^6.0.1, glob-parent@^6.0.2:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3"
+  integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==
+  dependencies:
+    is-glob "^4.0.3"
+
+glob-to-regexp@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e"
+  integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==
+
+glob@7.1.6:
+  version "7.1.6"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
+  integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
   dependencies:
     fs.realpath "^1.0.0"
     inflight "^1.0.4"
@@ -10111,7 +8892,7 @@ glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7.0.5:
+glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
   integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==
@@ -10123,7 +8904,7 @@ glob@^7.0.5:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-global-modules@2.0.0:
+global-modules@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780"
   integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==
@@ -10144,24 +8925,17 @@ globals@^11.1.0:
   resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e"
   integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==
 
-globals@^12.1.0:
-  version "12.4.0"
-  resolved "https://registry.yarnpkg.com/globals/-/globals-12.4.0.tgz#a18813576a41b00a24a97e7f815918c2e19925f8"
-  integrity sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==
-  dependencies:
-    type-fest "^0.8.1"
-
-globals@^13.6.0:
-  version "13.8.0"
-  resolved "https://registry.yarnpkg.com/globals/-/globals-13.8.0.tgz#3e20f504810ce87a8d72e55aecf8435b50f4c1b3"
-  integrity sha512-rHtdA6+PDBIjeEvA91rpqzEvk/k3/i7EeNQiryiWuJH0Hw9cpyJMAt2jtbAwUaRdhD+573X4vWw6IcjKPasi9Q==
+globals@^13.19.0:
+  version "13.20.0"
+  resolved "https://registry.yarnpkg.com/globals/-/globals-13.20.0.tgz#ea276a1e508ffd4f1612888f9d1bad1e2717bf82"
+  integrity sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==
   dependencies:
     type-fest "^0.20.2"
 
-globby@11.0.1:
-  version "11.0.1"
-  resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.1.tgz#9a2bf107a068f3ffeabc49ad702c79ede8cfd357"
-  integrity sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==
+globby@11.0.3:
+  version "11.0.3"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.3.tgz#9b1f0cb523e171dd1ad8c7b2a9fb4b644b9593cb"
+  integrity sha512-ffdmosjA807y7+lA1NM0jELARVmYul/715xiILEjo3hBLPTcirgQNnXECn5g3mtR8TOLCVbkfua1Hpen25/Xcg==
   dependencies:
     array-union "^2.1.0"
     dir-glob "^3.0.1"
@@ -10170,28 +8944,28 @@ globby@11.0.1:
     merge2 "^1.3.0"
     slash "^3.0.0"
 
-globby@11.0.3, globby@^11.0.1:
-  version "11.0.3"
-  resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.3.tgz#9b1f0cb523e171dd1ad8c7b2a9fb4b644b9593cb"
-  integrity sha512-ffdmosjA807y7+lA1NM0jELARVmYul/715xiILEjo3hBLPTcirgQNnXECn5g3mtR8TOLCVbkfua1Hpen25/Xcg==
+globby@^11.0.4, globby@^11.1.0:
+  version "11.1.0"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b"
+  integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==
   dependencies:
     array-union "^2.1.0"
     dir-glob "^3.0.1"
-    fast-glob "^3.1.1"
-    ignore "^5.1.4"
-    merge2 "^1.3.0"
+    fast-glob "^3.2.9"
+    ignore "^5.2.0"
+    merge2 "^1.4.1"
     slash "^3.0.0"
 
-globby@^6.1.0:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/globby/-/globby-6.1.0.tgz#f5a6d70e8395e21c858fb0489d64df02424d506c"
-  integrity sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=
+globby@^13.1.1:
+  version "13.2.0"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-13.2.0.tgz#7dd5678d765c4680c2e6d106230d86cb727cb1af"
+  integrity sha512-jWsQfayf13NvqKUIL3Ta+CIqMnvlaIDFveWE/dpOZ9+3AMEJozsxDvKA02zync9UuvOM8rOXzsD5GqKP4OnWPQ==
   dependencies:
-    array-union "^1.0.1"
-    glob "^7.0.3"
-    object-assign "^4.0.1"
-    pify "^2.0.0"
-    pinkie-promise "^2.0.0"
+    dir-glob "^3.0.1"
+    fast-glob "^3.2.11"
+    ignore "^5.2.0"
+    merge2 "^1.4.1"
+    slash "^4.0.0"
 
 good-listener@^1.2.2:
   version "1.2.2"
@@ -10217,10 +8991,20 @@ got@^9.6.0:
     to-readable-stream "^1.0.0"
     url-parse-lax "^3.0.0"
 
-graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4:
-  version "4.2.6"
-  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee"
-  integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==
+graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9:
+  version "4.2.11"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
+  integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
+
+grapheme-splitter@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e"
+  integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==
+
+graphemer@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6"
+  integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==
 
 graphql-config@^3.2.0:
   version "3.3.0"
@@ -10253,27 +9037,13 @@ graphql-tag@2.10.3:
   resolved "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.10.3.tgz#ea1baba5eb8fc6339e4c4cf049dabe522b0edf03"
   integrity sha512-4FOv3ZKfA4WdOKJeHdz6B3F/vxBLSgmBcGeAFPf4n1F64ltJUvOOerNj0rsJxONQGdhUMynQIvd6LzB+1J5oKA==
 
-graphql-tag@^2.10.1:
-  version "2.12.5"
-  resolved "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.12.5.tgz#5cff974a67b417747d05c8d9f5f3cb4495d0db8f"
-  integrity sha512-5xNhP4063d16Pz3HBtKprutsPrmHZi5IdUGOWRxA2B6VF7BIRGOHZ5WQvDmJXZuPcBg7rYwaFxvQYjqkSdR3TQ==
-  dependencies:
-    tslib "^2.1.0"
-
-graphql-tag@^2.11.0:
+graphql-tag@^2.10.1, graphql-tag@^2.11.0, graphql-tag@^2.12.0:
   version "2.12.6"
   resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.6.tgz#d441a569c1d2537ef10ca3d1633b48725329b5f1"
   integrity sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg==
   dependencies:
     tslib "^2.1.0"
 
-graphql-tag@^2.12.0:
-  version "2.12.4"
-  resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.4.tgz#d34066688a4f09e72d6f4663c74211e9b4b7c4bf"
-  integrity sha512-VV1U4O+9x99EkNpNmCUV5RZwq6MnK4+pGbRYWG+lA/m3uo7TSqJF81OkcOP148gFP6fzdl7JWYBrwWVTS9jXww==
-  dependencies:
-    tslib "^2.1.0"
-
 graphql-ws@^4.4.1:
   version "4.5.1"
   resolved "https://registry.yarnpkg.com/graphql-ws/-/graphql-ws-4.5.1.tgz#d9dc6e047c6d4ddb928ccbfb3ca3022580a89925"
@@ -10289,29 +9059,11 @@ graphql.macro@^1.4.2:
     babel-plugin-macros "^2.5.0"
     graphql-tag "^2.10.1"
 
-graphql@*:
-  version "15.5.0"
-  resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.5.0.tgz#39d19494dbe69d1ea719915b578bf920344a69d5"
-  integrity sha512-OmaM7y0kaK31NKG31q4YbD2beNYa6jBBKtMFT6gLYJljHLJr42IqJ8KX08u3Li/0ifzTU5HjmoOOrwa5BRLeDA==
-
-graphql@^15.0.0, graphql@^15.5.0:
+graphql@*, graphql@^15.0.0, graphql@^15.5.0:
   version "15.5.1"
   resolved "https://registry.npmjs.org/graphql/-/graphql-15.5.1.tgz#f2f84415d8985e7b84731e7f3536f8bb9d383aad"
   integrity sha512-FeTRX67T3LoE3LWAxxOlW2K3Bz+rMYAC18rRguK4wgXaTZMiJwSUwDmPFo3UadAKbzirKIg5Qy+sNJXbpPRnQw==
 
-growly@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081"
-  integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=
-
-gzip-size@5.1.1:
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-5.1.1.tgz#cb9bee692f87c0612b232840a873904e4c135274"
-  integrity sha512-FNHi6mmoHvs1mxZAds4PpdCS6QG8B4C1krxJsMutgxl5t3+GlRTzzI3NEkifXx2pVsOvJdOGSmIgDhQ55FwdPA==
-  dependencies:
-    duplexer "^0.1.1"
-    pify "^4.0.1"
-
 gzip-size@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462"
@@ -10349,11 +9101,6 @@ has-ansi@^2.0.0:
   dependencies:
     ansi-regex "^2.0.0"
 
-has-bigints@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.1.tgz#64fe6acb020673e3b78db035a5af69aa9d07b113"
-  integrity sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==
-
 has-bigints@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa"
@@ -10376,12 +9123,7 @@ has-property-descriptors@^1.0.0:
   dependencies:
     get-intrinsic "^1.1.1"
 
-has-symbols@^1.0.1, has-symbols@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423"
-  integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==
-
-has-symbols@^1.0.3:
+has-symbols@^1.0.1, has-symbols@^1.0.2, has-symbols@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8"
   integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==
@@ -10424,30 +9166,13 @@ has-values@^1.0.0:
     is-number "^3.0.0"
     kind-of "^4.0.0"
 
-has@^1.0.0, has@^1.0.3:
+has@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
   integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==
   dependencies:
     function-bind "^1.1.1"
 
-hash-base@^3.0.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/hash-base/-/hash-base-3.1.0.tgz#55c381d9e06e1d2997a883b4a3fddfe7f0d3af33"
-  integrity sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==
-  dependencies:
-    inherits "^2.0.4"
-    readable-stream "^3.6.0"
-    safe-buffer "^5.2.0"
-
-hash.js@^1.0.0, hash.js@^1.0.3:
-  version "1.1.7"
-  resolved "https://registry.yarnpkg.com/hash.js/-/hash.js-1.1.7.tgz#0babca538e8d4ee4a0f8988d68866537a003cf42"
-  integrity sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==
-  dependencies:
-    inherits "^2.0.3"
-    minimalistic-assert "^1.0.1"
-
 hast-to-hyperscript@^9.0.0:
   version "9.0.1"
   resolved "https://registry.yarnpkg.com/hast-to-hyperscript/-/hast-to-hyperscript-9.0.1.tgz#9b67fd188e4c81e8ad66f803855334173920218d"
@@ -10561,11 +9286,6 @@ header-case@^2.0.4:
     capital-case "^1.0.4"
     tslib "^2.0.3"
 
-hex-color-regex@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz#4c06fccb4602fe2602b3c93df82d7e7dbf1a8a8e"
-  integrity sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==
-
 highlight.js@^10.4.1, highlight.js@~10.7.0:
   version "10.7.3"
   resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-10.7.3.tgz#697272e3991356e40c3cac566a74eef681756531"
@@ -10590,15 +9310,6 @@ history@^5.0.0:
   dependencies:
     "@babel/runtime" "^7.7.6"
 
-hmac-drbg@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1"
-  integrity sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=
-  dependencies:
-    hash.js "^1.0.3"
-    minimalistic-assert "^1.0.0"
-    minimalistic-crypto-utils "^1.0.1"
-
 hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.1.0, hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2:
   version "3.3.2"
   resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45"
@@ -10611,11 +9322,6 @@ hoopy@^0.1.4:
   resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d"
   integrity sha512-HRcs+2mr52W0K+x8RzcLzuPPmVIKMSv97RGHy0Ea9y/mpcaK+xTrjICA04KAHi4GRzxliNqNJEFYWHghy3rSfQ==
 
-hosted-git-info@^2.1.4:
-  version "2.8.9"
-  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
-  integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==
-
 hpack.js@^2.1.6:
   version "2.1.6"
   resolved "https://registry.yarnpkg.com/hpack.js/-/hpack.js-2.1.6.tgz#87774c0949e513f42e84575b3c45681fade2a0b2"
@@ -10626,16 +9332,6 @@ hpack.js@^2.1.6:
     readable-stream "^2.0.1"
     wbuf "^1.1.0"
 
-hsl-regex@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/hsl-regex/-/hsl-regex-1.0.0.tgz#d49330c789ed819e276a4c0d272dffa30b18fe6e"
-  integrity sha1-1JMwx4ntgZ4nakwNJy3/owsY/m4=
-
-hsla-regex@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/hsla-regex/-/hsla-regex-1.0.0.tgz#c1ce7a3168c8c6614033a4b5f7877f3b225f9c38"
-  integrity sha1-wc56MWjIxmFAM6S194d/OyJfnDg=
-
 html-encoding-sniffer@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3"
@@ -10643,60 +9339,54 @@ html-encoding-sniffer@^2.0.1:
   dependencies:
     whatwg-encoding "^1.0.5"
 
-html-entities@^1.2.1, html-entities@^1.3.1:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.4.0.tgz#cfbd1b01d2afaf9adca1b10ae7dffab98c71d2dc"
-  integrity sha512-8nxjcBcd8wovbeKx7h3wTji4e6+rhaVuPNpMqwWgnHh+N9ToqsCs6XztWRBPQ+UtzsoMAdKZtUENoVzU/EMtZA==
+html-entities@^2.1.0, html-entities@^2.3.2:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.4.0.tgz#edd0cee70402584c8c76cc2c0556db09d1f45061"
+  integrity sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ==
 
 html-escaper@^2.0.0:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453"
   integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==
 
-html-minifier-terser@^5.0.1:
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz#922e96f1f3bb60832c2634b79884096389b1f054"
-  integrity sha512-ZPr5MNObqnV/T9akshPKbVgyOqLmy+Bxo7juKCfTfnjNniTAMdy4hz21YQqoofMBJD2kdREaqPPdThoR78Tgxg==
+html-minifier-terser@^6.0.2:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz#bfc818934cc07918f6b3669f5774ecdfd48f32ab"
+  integrity sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==
   dependencies:
-    camel-case "^4.1.1"
-    clean-css "^4.2.3"
-    commander "^4.1.1"
+    camel-case "^4.1.2"
+    clean-css "^5.2.2"
+    commander "^8.3.0"
     he "^1.2.0"
-    param-case "^3.0.3"
+    param-case "^3.0.4"
     relateurl "^0.2.7"
-    terser "^4.6.3"
+    terser "^5.10.0"
 
 html-void-elements@^1.0.0:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483"
   integrity sha512-uE/TxKuyNIcx44cIWnjr/rfIATDH7ZaOMmstu0CwhFG1Dunhlp4OC6/NMbhiwoq5BpW0ubi303qnEk/PZj614w==
 
-html-webpack-plugin@4.5.0:
-  version "4.5.0"
-  resolved "https://registry.yarnpkg.com/html-webpack-plugin/-/html-webpack-plugin-4.5.0.tgz#625097650886b97ea5dae331c320e3238f6c121c"
-  integrity sha512-MouoXEYSjTzCrjIxWwg8gxL5fE2X2WZJLmBYXlaJhQUH5K/b5OrqmV7T4dB7iu0xkmJ6JlUuV6fFVtnqbPopZw==
-  dependencies:
-    "@types/html-minifier-terser" "^5.0.0"
-    "@types/tapable" "^1.0.5"
-    "@types/webpack" "^4.41.8"
-    html-minifier-terser "^5.0.1"
-    loader-utils "^1.2.3"
-    lodash "^4.17.15"
-    pretty-error "^2.1.1"
-    tapable "^1.1.3"
-    util.promisify "1.0.0"
-
-htmlparser2@^3.10.1:
-  version "3.10.1"
-  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
-  integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==
-  dependencies:
-    domelementtype "^1.3.1"
-    domhandler "^2.3.0"
-    domutils "^1.5.1"
-    entities "^1.1.1"
-    inherits "^2.0.1"
-    readable-stream "^3.1.1"
+html-webpack-plugin@^5.5.0:
+  version "5.5.3"
+  resolved "https://registry.yarnpkg.com/html-webpack-plugin/-/html-webpack-plugin-5.5.3.tgz#72270f4a78e222b5825b296e5e3e1328ad525a3e"
+  integrity sha512-6YrDKTuqaP/TquFH7h4srYWsZx+x6k6+FbsTm0ziCwGHDP78Unr1r9F/H4+sGmMbX08GQcJ+K64x55b+7VM/jg==
+  dependencies:
+    "@types/html-minifier-terser" "^6.0.0"
+    html-minifier-terser "^6.0.2"
+    lodash "^4.17.21"
+    pretty-error "^4.0.0"
+    tapable "^2.0.0"
+
+htmlparser2@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
+  integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.0.0"
+    domutils "^2.5.2"
+    entities "^2.0.0"
 
 http-cache-semantics@^4.0.0:
   version "4.1.1"
@@ -10743,16 +9433,6 @@ http-proxy-agent@^4.0.1:
     agent-base "6"
     debug "4"
 
-http-proxy-middleware@0.19.1:
-  version "0.19.1"
-  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz#183c7dc4aa1479150306498c210cdaf96080a43a"
-  integrity sha512-yHYTgWMQO8VvwNS22eLLloAkvungsKdKTLO8AJlftYIKNfJr3GK3zK0ZCfzDDGUBttdGc8xFy1mCitvNKQtC3Q==
-  dependencies:
-    http-proxy "^1.17.0"
-    is-glob "^4.0.0"
-    lodash "^4.17.11"
-    micromatch "^3.1.10"
-
 http-proxy-middleware@2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.0.tgz#20d1ac3409199c83e5d0383ba6436b04e7acb9fe"
@@ -10764,7 +9444,18 @@ http-proxy-middleware@2.0.0:
     is-plain-obj "^3.0.0"
     micromatch "^4.0.2"
 
-http-proxy@^1.17.0, http-proxy@^1.18.1:
+http-proxy-middleware@^2.0.3:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f"
+  integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==
+  dependencies:
+    "@types/http-proxy" "^1.17.8"
+    http-proxy "^1.18.1"
+    is-glob "^4.0.1"
+    is-plain-obj "^3.0.0"
+    micromatch "^4.0.2"
+
+http-proxy@^1.18.1:
   version "1.18.1"
   resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549"
   integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==
@@ -10782,11 +9473,6 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-https-browserify@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"
-  integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=
-
 https-proxy-agent@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
@@ -10795,11 +9481,6 @@ https-proxy-agent@^5.0.0:
     agent-base "6"
     debug "4"
 
-human-signals@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3"
-  integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==
-
 human-signals@^2.1.0:
   version "2.1.0"
   resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
@@ -10817,12 +9498,17 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
-icss-utils@^4.0.0, icss-utils@^4.1.1:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-4.1.1.tgz#21170b53789ee27447c2f47dd683081403f9a467"
-  integrity sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA==
+iconv-lite@^0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
+  integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==
   dependencies:
-    postcss "^7.0.14"
+    safer-buffer ">= 2.1.2 < 3.0.0"
+
+icss-utils@^5.0.0, icss-utils@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae"
+  integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==
 
 idb-keyval@^5.0.2:
   version "5.1.5"
@@ -10831,63 +9517,43 @@ idb-keyval@^5.0.2:
   dependencies:
     safari-14-idb-fix "^1.0.6"
 
-identity-obj-proxy@3.0.0:
+idb@^7.0.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/idb/-/idb-7.1.1.tgz#d910ded866d32c7ced9befc5bfdf36f572ced72b"
+  integrity sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==
+
+identity-obj-proxy@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz#94d2bda96084453ef36fbc5aaec37e0f79f1fc14"
-  integrity sha1-lNK9qWCERT7zb7xarsN+D3nx/BQ=
+  integrity sha512-00n6YnVHKrinT9t0d9+5yZC6UBNJANpYEQvL2LlX6Ab9lnmxzIRcEmTPuyGScvl1+jKuCICX1Z0Ab1pPKKdikA==
   dependencies:
     harmony-reflect "^1.4.6"
 
-ieee754@^1.1.13, ieee754@^1.1.4:
+ieee754@^1.1.13:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
   integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
 
-iferr@^0.1.5:
-  version "0.1.5"
-  resolved "https://registry.yarnpkg.com/iferr/-/iferr-0.1.5.tgz#c60eed69e6d8fdb6b3104a1fcbca1c192dc5b501"
-  integrity sha1-xg7taebY/bazEEofy8ocGS3FtQE=
-
-ignore@^4.0.6:
-  version "4.0.6"
-  resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
-  integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==
-
-ignore@^5.1.4:
-  version "5.1.8"
-  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
-  integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
+ignore@^5.1.4, ignore@^5.2.0:
+  version "5.2.4"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324"
+  integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==
 
 image-size@~0.5.0:
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"
   integrity sha1-Cd/Uq50g4p6xw+gLiZA3jfnjy5w=
 
-immer@8.0.1:
-  version "8.0.1"
-  resolved "https://registry.yarnpkg.com/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656"
-  integrity sha512-aqXhGP7//Gui2+UrEtvxZxSquQVXTpZ7KDxfCcKAF3Vysvw0CViVaW9RZ1j1xlIYqaaaipBoqdqeibkc18PNvA==
+immer@^9.0.7:
+  version "9.0.21"
+  resolved "https://registry.yarnpkg.com/immer/-/immer-9.0.21.tgz#1e025ea31a40f24fb064f1fef23e931496330176"
+  integrity sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==
 
 immutable@~3.7.6:
   version "3.7.6"
   resolved "https://registry.yarnpkg.com/immutable/-/immutable-3.7.6.tgz#13b4d3cb12befa15482a26fe1b2ebae640071e4b"
   integrity sha512-AizQPcaofEtO11RZhPPHBOJRdo/20MKQF9mBLnVkBoyHi1/zXK8fzVdnEpSV9gxqtnh6Qomfp3F0xT5qP/vThw==
 
-import-cwd@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/import-cwd/-/import-cwd-2.1.0.tgz#aa6cf36e722761285cb371ec6519f53e2435b0a9"
-  integrity sha1-qmzzbnInYShcs3HsZRn1PiQ1sKk=
-  dependencies:
-    import-from "^2.1.0"
-
-import-fresh@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-2.0.0.tgz#d81355c15612d386c61f9ddd3922d4304822a546"
-  integrity sha1-2BNVwVYS04bGH53dOSLUMEgipUY=
-  dependencies:
-    caller-path "^2.0.0"
-    resolve-from "^3.0.0"
-
 import-fresh@^3.0.0, import-fresh@^3.1.0, import-fresh@^3.2.1:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b"
@@ -10908,21 +9574,6 @@ import-from@4.0.0:
   resolved "https://registry.yarnpkg.com/import-from/-/import-from-4.0.0.tgz#2710b8d66817d232e16f4166e319248d3d5492e2"
   integrity sha512-P9J71vT5nLlDeV8FHs5nNxaLbrpfAV5cF5srvbZfpwpcJoM/xZR3hiv+q+SAnuSmuGbXMWud063iIMx/V/EWZQ==
 
-import-from@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/import-from/-/import-from-2.1.0.tgz#335db7f2a7affd53aaa471d4b8021dee36b7f3b1"
-  integrity sha1-M1238qev/VOqpHHUuAId7ja387E=
-  dependencies:
-    resolve-from "^3.0.0"
-
-import-local@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/import-local/-/import-local-2.0.0.tgz#55070be38a5993cf18ef6db7e961f5bee5c5a09d"
-  integrity sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ==
-  dependencies:
-    pkg-dir "^3.0.0"
-    resolve-cwd "^2.0.0"
-
 import-local@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.0.2.tgz#a8cfd0431d1de4a2199703d003e3e62364fa6db6"
@@ -10946,16 +9597,6 @@ indent-string@^4.0.0:
   resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251"
   integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==
 
-indexes-of@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607"
-  integrity sha1-8w9xbI4r00bHtn0985FVZqfAVgc=
-
-infer-owner@^1.0.3, infer-owner@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467"
-  integrity sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==
-
 inflected@^2.0.4:
   version "2.1.0"
   resolved "https://registry.npmjs.org/inflected/-/inflected-2.1.0.tgz#2816ac17a570bbbc8303ca05bca8bf9b3f959687"
@@ -10969,16 +9610,11 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
 
-inherits@2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.1.tgz#b17d08d326b4423e568eff719f91b0b1cbdf69f1"
-  integrity sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=
-
 inherits@2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
@@ -11021,14 +9657,6 @@ inquirer@^7.3.3:
     strip-ansi "^6.0.0"
     through "^2.3.6"
 
-internal-ip@^4.3.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/internal-ip/-/internal-ip-4.3.0.tgz#845452baad9d2ca3b69c635a137acb9a0dad0907"
-  integrity sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg==
-  dependencies:
-    default-gateway "^4.2.0"
-    ipaddr.js "^1.9.0"
-
 internal-slot@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.3.tgz#7347e307deeea2faac2ac6205d4bc7d34967f59c"
@@ -11038,10 +9666,10 @@ internal-slot@^1.0.3:
     has "^1.0.3"
     side-channel "^1.0.4"
 
-internmap@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/internmap/-/internmap-1.0.1.tgz#0017cc8a3b99605f0302f2b198d272e015e5df95"
-  integrity sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==
+"internmap@1 - 2", internmap@2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.3.tgz#6685f23755e43c524e251d29cbc97248e3061009"
+  integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==
 
 invariant@^2.2.4:
   version "2.2.4"
@@ -11050,30 +9678,15 @@ invariant@^2.2.4:
   dependencies:
     loose-envify "^1.0.0"
 
-ip-regex@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
-  integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=
-
-ip@^1.1.0, ip@^1.1.5:
-  version "1.1.5"
-  resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a"
-  integrity sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=
-
-ipaddr.js@1.9.1, ipaddr.js@^1.9.0:
+ipaddr.js@1.9.1:
   version "1.9.1"
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3"
   integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==
 
-is-absolute-url@^2.0.0:
+ipaddr.js@^2.0.1:
   version "2.1.0"
-  resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-2.1.0.tgz#50530dfb84fcc9aa7dbe7852e83a37b93b9f2aa6"
-  integrity sha1-UFMN+4T8yap9vnhS6Do3uTufKqY=
-
-is-absolute-url@^3.0.3:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-3.0.3.tgz#96c6a22b6a23929b11ea0afb1836c36ad4a5d698"
-  integrity sha512-opmNIX7uFnS96NtPmhWQgQx6/NYFgsUXYMllcfzwWKUMwfo8kku1TvE6hkNcH+Q1ts5cMVrsY7j0bxXQDciu9Q==
+  resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.1.0.tgz#2119bc447ff8c257753b196fc5f1ce08a4cdf39f"
+  integrity sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==
 
 is-absolute@^1.0.0:
   version "1.0.0"
@@ -11110,35 +9723,16 @@ is-alphanumerical@^1.0.0:
     is-alphabetical "^1.0.0"
     is-decimal "^1.0.0"
 
-is-arguments@^1.0.4:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.0.tgz#62353031dfbee07ceb34656a6bde59efecae8dd9"
-  integrity sha512-1Ij4lOMPl/xB5kBDn7I+b2ttPMKa8szhEIrXDuXQD/oe3HJLTLhqhgGspwgyGd6MOywBUqVvYicF72lkgDnIHg==
-  dependencies:
-    call-bind "^1.0.0"
-
 is-arrayish@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
   integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=
 
-is-arrayish@^0.3.1:
-  version "0.3.2"
-  resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"
-  integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==
-
 is-bigint@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.2.tgz#ffb381442503235ad245ea89e45b3dbff040ee5a"
   integrity sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==
 
-is-binary-path@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-1.0.1.tgz#75f16642b480f187a711c814161fd3a4a7655898"
-  integrity sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=
-  dependencies:
-    binary-extensions "^1.0.0"
-
 is-binary-path@~2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09"
@@ -11163,46 +9757,15 @@ is-buffer@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191"
   integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==
 
-is-callable@^1.1.4, is-callable@^1.2.3:
-  version "1.2.3"
-  resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.3.tgz#8b1e0500b73a1d76c70487636f368e519de8db8e"
-  integrity sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==
-
-is-callable@^1.2.7:
+is-callable@^1.1.4, is-callable@^1.2.7:
   version "1.2.7"
   resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055"
   integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==
 
-is-ci@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c"
-  integrity sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==
-  dependencies:
-    ci-info "^2.0.0"
-
-is-color-stop@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/is-color-stop/-/is-color-stop-1.1.0.tgz#cfff471aee4dd5c9e158598fbe12967b5cdad345"
-  integrity sha1-z/9HGu5N1cnhWFmPvhKWe1za00U=
-  dependencies:
-    css-color-names "^0.0.4"
-    hex-color-regex "^1.1.0"
-    hsl-regex "^1.0.0"
-    hsla-regex "^1.0.0"
-    rgb-regex "^1.0.1"
-    rgba-regex "^1.0.0"
-
-is-core-module@^2.0.0, is-core-module@^2.2.0, is-core-module@^2.4.0:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.4.0.tgz#8e9fc8e15027b011418026e98f0e6f4d86305cc1"
-  integrity sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==
-  dependencies:
-    has "^1.0.3"
-
-is-core-module@^2.9.0:
-  version "2.11.0"
-  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.11.0.tgz#ad4cb3e3863e814523c96f3f58d26cc570ff0144"
-  integrity sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==
+is-core-module@^2.11.0, is-core-module@^2.9.0:
+  version "2.12.1"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.12.1.tgz#0c0b6885b6f80011c71541ce15c8d66cf5a4f9fd"
+  integrity sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==
   dependencies:
     has "^1.0.3"
 
@@ -11248,12 +9811,7 @@ is-descriptor@^1.0.0, is-descriptor@^1.0.2:
     is-data-descriptor "^1.0.0"
     kind-of "^6.0.2"
 
-is-directory@^0.3.1:
-  version "0.3.1"
-  resolved "https://registry.yarnpkg.com/is-directory/-/is-directory-0.3.1.tgz#61339b6f2475fc772fd9c9d83f5c8575dc154ae1"
-  integrity sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE=
-
-is-docker@^2.0.0:
+is-docker@^2.0.0, is-docker@^2.1.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa"
   integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==
@@ -11270,7 +9828,7 @@ is-extendable@^1.0.0, is-extendable@^1.0.1:
   dependencies:
     is-plain-object "^2.0.4"
 
-is-extglob@^2.1.0, is-extglob@^2.1.1:
+is-extglob@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
   integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=
@@ -11309,19 +9867,19 @@ is-generator-fn@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118"
   integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==
 
-is-glob@4.0.1, is-glob@^4.0.0, is-glob@^4.0.1, is-glob@~4.0.1:
+is-glob@4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc"
   integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==
   dependencies:
     is-extglob "^2.1.1"
 
-is-glob@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a"
-  integrity sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=
+is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1:
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
+  integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==
   dependencies:
-    is-extglob "^2.1.0"
+    is-extglob "^2.1.1"
 
 is-hexadecimal@^1.0.0:
   version "1.0.4"
@@ -11347,11 +9905,6 @@ is-module@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591"
   integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=
 
-is-negative-zero@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24"
-  integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==
-
 is-negative-zero@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150"
@@ -11379,11 +9932,6 @@ is-obj@^1.0.1:
   resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f"
   integrity sha1-PkcprB9f3gJc19g6iW2rn09n2w8=
 
-is-obj@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982"
-  integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==
-
 is-observable@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-observable/-/is-observable-1.1.0.tgz#b3e986c8f44de950867cab5403f5a3465005975e"
@@ -11391,29 +9939,10 @@ is-observable@^1.1.0:
   dependencies:
     symbol-observable "^1.1.0"
 
-is-path-cwd@^2.0.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz#67d43b82664a7b5191fd9119127eb300048a9fdb"
-  integrity sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==
-
-is-path-in-cwd@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz#bfe2dca26c69f397265a4009963602935a053acb"
-  integrity sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==
-  dependencies:
-    is-path-inside "^2.1.0"
-
-is-path-inside@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-2.1.0.tgz#7c9810587d659a40d27bcdb4d5616eab059494b2"
-  integrity sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==
-  dependencies:
-    path-is-inside "^1.0.2"
-
-is-plain-obj@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e"
-  integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4=
+is-path-inside@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283"
+  integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==
 
 is-plain-obj@^2.0.0:
   version "2.1.0"
@@ -11447,14 +9976,6 @@ is-promise@^2.1.0:
   resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1"
   integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==
 
-is-regex@^1.0.4, is-regex@^1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.3.tgz#d029f9aff6448b93ebbe3f33dac71511fdcbef9f"
-  integrity sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==
-  dependencies:
-    call-bind "^1.0.2"
-    has-symbols "^1.0.2"
-
 is-regex@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958"
@@ -11475,12 +9996,7 @@ is-relative@^1.0.0:
   dependencies:
     is-unc-path "^1.0.0"
 
-is-resolvable@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88"
-  integrity sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==
-
-is-root@2.1.0:
+is-root@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/is-root/-/is-root-2.1.0.tgz#809e18129cf1129644302a4f8544035d51984a9c"
   integrity sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg==
@@ -11502,12 +10018,7 @@ is-stream@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3"
   integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==
 
-is-string@^1.0.5, is-string@^1.0.6:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.6.tgz#3fe5d5992fb0d93404f32584d4b0179a71b54a5f"
-  integrity sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w==
-
-is-string@^1.0.7:
+is-string@^1.0.5, is-string@^1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd"
   integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==
@@ -11562,11 +10073,6 @@ is-windows@^1.0.1, is-windows@^1.0.2:
   resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
   integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==
 
-is-wsl@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-1.1.0.tgz#1f16e4aa22b04d1336b66188a66af3c600c3a66d"
-  integrity sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=
-
 is-wsl@^2.1.1, is-wsl@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271"
@@ -11579,7 +10085,7 @@ isarray@0.0.1:
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
   integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=
 
-isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
+isarray@1.0.0, isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
   integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
@@ -11624,19 +10130,20 @@ isstream@~0.1.2:
   resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
   integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
 
-istanbul-lib-coverage@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz#f5944a37c70b550b02a78a5c3b2055b280cec8ec"
-  integrity sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg==
+istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz#189e7909d0a39fa5a3dfad5b03f71947770191d3"
+  integrity sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==
 
-istanbul-lib-instrument@^4.0.0, istanbul-lib-instrument@^4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz#873c6fff897450118222774696a3f28902d77c1d"
-  integrity sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==
+istanbul-lib-instrument@^5.0.4, istanbul-lib-instrument@^5.1.0:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz#d10c8885c2125574e1c231cacadf955675e1ce3d"
+  integrity sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==
   dependencies:
-    "@babel/core" "^7.7.5"
+    "@babel/core" "^7.12.3"
+    "@babel/parser" "^7.14.7"
     "@istanbuljs/schema" "^0.1.2"
-    istanbul-lib-coverage "^3.0.0"
+    istanbul-lib-coverage "^3.2.0"
     semver "^6.3.0"
 
 istanbul-lib-report@^3.0.0:
@@ -11657,10 +10164,10 @@ istanbul-lib-source-maps@^4.0.0:
     istanbul-lib-coverage "^3.0.0"
     source-map "^0.6.1"
 
-istanbul-reports@^3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.0.2.tgz#d593210e5000683750cb09fc0644e4b6e27fd53b"
-  integrity sha512-9tZvz7AiR3PEDNGiV9vIouQ/EAcqMXFmkcA1CDFTwOB98OZVDL0PH9glHotf5Ugp6GCOTypfzGWI/OqjWNCRUw==
+istanbul-reports@^3.1.3:
+  version "3.1.5"
+  resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.1.5.tgz#cc9a6ab25cb25659810e4785ed9d9fb742578bae"
+  integrity sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==
   dependencies:
     html-escaper "^2.0.0"
     istanbul-lib-report "^3.0.0"
@@ -11670,96 +10177,99 @@ iterall@^1.2.1:
   resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea"
   integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==
 
-jake@^10.6.1:
-  version "10.8.2"
-  resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.2.tgz#ebc9de8558160a66d82d0eadc6a2e58fbc500a7b"
-  integrity sha512-eLpKyrfG3mzvGE2Du8VoPbeSkRry093+tyNjdYaBbJS9v17knImYGNXQCUV0gLxQtF82m3E8iRb/wdSQZLoq7A==
-  dependencies:
-    async "0.9.x"
-    chalk "^2.4.2"
-    filelist "^1.0.1"
-    minimatch "^3.0.4"
-
-jest-changed-files@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-26.6.2.tgz#f6198479e1cc66f22f9ae1e22acaa0b429c042d0"
-  integrity sha512-fDS7szLcY9sCtIip8Fjry9oGf3I2ht/QT21bAHm5Dmf0mD4X3ReNUf17y+bO6fR8WgbIZTlbyG1ak/53cbRzKQ==
-  dependencies:
-    "@jest/types" "^26.6.2"
-    execa "^4.0.0"
-    throat "^5.0.0"
-
-jest-circus@26.6.0:
-  version "26.6.0"
-  resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-26.6.0.tgz#7d9647b2e7f921181869faae1f90a2629fd70705"
-  integrity sha512-L2/Y9szN6FJPWFK8kzWXwfp+FOR7xq0cUL4lIsdbIdwz3Vh6P1nrpcqOleSzr28zOtSHQNV9Z7Tl+KkuK7t5Ng==
-  dependencies:
-    "@babel/traverse" "^7.1.0"
-    "@jest/environment" "^26.6.0"
-    "@jest/test-result" "^26.6.0"
-    "@jest/types" "^26.6.0"
-    "@types/babel__traverse" "^7.0.4"
+jake@^10.8.5:
+  version "10.8.7"
+  resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.7.tgz#63a32821177940c33f356e0ba44ff9d34e1c7d8f"
+  integrity sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==
+  dependencies:
+    async "^3.2.3"
+    chalk "^4.0.2"
+    filelist "^1.0.4"
+    minimatch "^3.1.2"
+
+jest-changed-files@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-27.5.1.tgz#a348aed00ec9bf671cc58a66fcbe7c3dfd6a68f5"
+  integrity sha512-buBLMiByfWGCoMsLLzGUUSpAmIAGnbR2KJoMN10ziLhOLvP4e0SlypHnAel8iqQXTrcbmfEY9sSqae5sgUsTvw==
+  dependencies:
+    "@jest/types" "^27.5.1"
+    execa "^5.0.0"
+    throat "^6.0.1"
+
+jest-circus@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-27.5.1.tgz#37a5a4459b7bf4406e53d637b49d22c65d125ecc"
+  integrity sha512-D95R7x5UtlMA5iBYsOHFFbMD/GVA4R/Kdq15f7xYWUfWHBto9NYRsOvnSauTgdF+ogCpJ4tyKOXhUifxS65gdw==
+  dependencies:
+    "@jest/environment" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
     chalk "^4.0.0"
     co "^4.6.0"
     dedent "^0.7.0"
-    expect "^26.6.0"
+    expect "^27.5.1"
     is-generator-fn "^2.0.0"
-    jest-each "^26.6.0"
-    jest-matcher-utils "^26.6.0"
-    jest-message-util "^26.6.0"
-    jest-runner "^26.6.0"
-    jest-runtime "^26.6.0"
-    jest-snapshot "^26.6.0"
-    jest-util "^26.6.0"
-    pretty-format "^26.6.0"
-    stack-utils "^2.0.2"
-    throat "^5.0.0"
-
-jest-cli@^26.6.0:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-26.6.3.tgz#43117cfef24bc4cd691a174a8796a532e135e92a"
-  integrity sha512-GF9noBSa9t08pSyl3CY4frMrqp+aQXFGFkf5hEPbh/pIUFYWMK6ZLTfbmadxJVcJrdRoChlWQsA2VkJcDFK8hg==
-  dependencies:
-    "@jest/core" "^26.6.3"
-    "@jest/test-result" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    jest-each "^27.5.1"
+    jest-matcher-utils "^27.5.1"
+    jest-message-util "^27.5.1"
+    jest-runtime "^27.5.1"
+    jest-snapshot "^27.5.1"
+    jest-util "^27.5.1"
+    pretty-format "^27.5.1"
+    slash "^3.0.0"
+    stack-utils "^2.0.3"
+    throat "^6.0.1"
+
+jest-cli@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-27.5.1.tgz#278794a6e6458ea8029547e6c6cbf673bd30b145"
+  integrity sha512-Hc6HOOwYq4/74/c62dEE3r5elx8wjYqxY0r0G/nFrLDPMFRu6RA/u8qINOIkvhxG7mMQ5EJsOGfRpI8L6eFUVw==
+  dependencies:
+    "@jest/core" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/types" "^27.5.1"
     chalk "^4.0.0"
     exit "^0.1.2"
-    graceful-fs "^4.2.4"
+    graceful-fs "^4.2.9"
     import-local "^3.0.2"
-    is-ci "^2.0.0"
-    jest-config "^26.6.3"
-    jest-util "^26.6.2"
-    jest-validate "^26.6.2"
+    jest-config "^27.5.1"
+    jest-util "^27.5.1"
+    jest-validate "^27.5.1"
     prompts "^2.0.1"
-    yargs "^15.4.1"
+    yargs "^16.2.0"
 
-jest-config@^26.6.3:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-26.6.3.tgz#64f41444eef9eb03dc51d5c53b75c8c71f645349"
-  integrity sha512-t5qdIj/bCj2j7NFVHb2nFB4aUdfucDn3JRKgrZnplb8nieAirAzRSHP8uDEd+qV6ygzg9Pz4YG7UTJf94LPSyg==
+jest-config@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-27.5.1.tgz#5c387de33dca3f99ad6357ddeccd91bf3a0e4a41"
+  integrity sha512-5sAsjm6tGdsVbW9ahcChPAFCk4IlkQUknH5AvKjuLTSlcO/wCZKyFdn7Rg0EkC+OGgWODEy2hDpWB1PgzH0JNA==
   dependencies:
-    "@babel/core" "^7.1.0"
-    "@jest/test-sequencer" "^26.6.3"
-    "@jest/types" "^26.6.2"
-    babel-jest "^26.6.3"
+    "@babel/core" "^7.8.0"
+    "@jest/test-sequencer" "^27.5.1"
+    "@jest/types" "^27.5.1"
+    babel-jest "^27.5.1"
     chalk "^4.0.0"
+    ci-info "^3.2.0"
     deepmerge "^4.2.2"
     glob "^7.1.1"
-    graceful-fs "^4.2.4"
-    jest-environment-jsdom "^26.6.2"
-    jest-environment-node "^26.6.2"
-    jest-get-type "^26.3.0"
-    jest-jasmine2 "^26.6.3"
-    jest-regex-util "^26.0.0"
-    jest-resolve "^26.6.2"
-    jest-util "^26.6.2"
-    jest-validate "^26.6.2"
-    micromatch "^4.0.2"
-    pretty-format "^26.6.2"
+    graceful-fs "^4.2.9"
+    jest-circus "^27.5.1"
+    jest-environment-jsdom "^27.5.1"
+    jest-environment-node "^27.5.1"
+    jest-get-type "^27.5.1"
+    jest-jasmine2 "^27.5.1"
+    jest-regex-util "^27.5.1"
+    jest-resolve "^27.5.1"
+    jest-runner "^27.5.1"
+    jest-util "^27.5.1"
+    jest-validate "^27.5.1"
+    micromatch "^4.0.4"
+    parse-json "^5.2.0"
+    pretty-format "^27.5.1"
+    slash "^3.0.0"
+    strip-json-comments "^3.1.1"
 
-jest-diff@^26.0.0, jest-diff@^26.6.2:
+jest-diff@^26.0.0:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.6.2.tgz#1aa7468b52c3a68d7d5c5fdcdfcd5e49bd164394"
   integrity sha512-6m+9Z3Gv9wN0WFVasqjCL/06+EFCMTqDEUl/b87HYK2rAPTyfz4ZIuSlPhY51PIQRWx5TaxeF1qmXKe9gfN3sA==
@@ -11769,138 +10279,166 @@ jest-diff@^26.0.0, jest-diff@^26.6.2:
     jest-get-type "^26.3.0"
     pretty-format "^26.6.2"
 
-jest-docblock@^26.0.0:
-  version "26.0.0"
-  resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-26.0.0.tgz#3e2fa20899fc928cb13bd0ff68bd3711a36889b5"
-  integrity sha512-RDZ4Iz3QbtRWycd8bUEPxQsTlYazfYn/h5R65Fc6gOfwozFhoImx+affzky/FFBuqISPTqjXomoIGJVKBWoo0w==
-  dependencies:
-    detect-newline "^3.0.0"
-
-jest-each@^26.6.0, jest-each@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-26.6.2.tgz#02526438a77a67401c8a6382dfe5999952c167cb"
-  integrity sha512-Mer/f0KaATbjl8MCJ+0GEpNdqmnVmDYqCTJYTvoo7rqmRiDllmp2AYN+06F93nXcY3ur9ShIjS+CO/uD+BbH4A==
+jest-diff@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.5.1.tgz#a07f5011ac9e6643cf8a95a462b7b1ecf6680def"
+  integrity sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==
   dependencies:
-    "@jest/types" "^26.6.2"
     chalk "^4.0.0"
-    jest-get-type "^26.3.0"
-    jest-util "^26.6.2"
-    pretty-format "^26.6.2"
+    diff-sequences "^27.5.1"
+    jest-get-type "^27.5.1"
+    pretty-format "^27.5.1"
 
-jest-environment-jsdom@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-26.6.2.tgz#78d09fe9cf019a357009b9b7e1f101d23bd1da3e"
-  integrity sha512-jgPqCruTlt3Kwqg5/WVFyHIOJHsiAvhcp2qiR2QQstuG9yWox5+iHpU3ZrcBxW14T4fe5Z68jAfLRh7joCSP2Q==
+jest-docblock@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-27.5.1.tgz#14092f364a42c6108d42c33c8cf30e058e25f6c0"
+  integrity sha512-rl7hlABeTsRYxKiUfpHrQrG4e2obOiTQWfMEH3PxPjOtdsfLQO4ReWSZaQ7DETm4xu07rl4q/h4zcKXyU0/OzQ==
   dependencies:
-    "@jest/environment" "^26.6.2"
-    "@jest/fake-timers" "^26.6.2"
-    "@jest/types" "^26.6.2"
-    "@types/node" "*"
-    jest-mock "^26.6.2"
-    jest-util "^26.6.2"
-    jsdom "^16.4.0"
+    detect-newline "^3.0.0"
 
-jest-environment-node@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-26.6.2.tgz#824e4c7fb4944646356f11ac75b229b0035f2b0c"
-  integrity sha512-zhtMio3Exty18dy8ee8eJ9kjnRyZC1N4C1Nt/VShN1apyXc8rWGtJ9lI7vqiWcyyXS4BVSEn9lxAM2D+07/Tag==
+jest-each@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-27.5.1.tgz#5bc87016f45ed9507fed6e4702a5b468a5b2c44e"
+  integrity sha512-1Ff6p+FbhT/bXQnEouYy00bkNSY7OUpfIcmdl8vZ31A1UUaurOLPA8a8BbJOF2RDUElwJhmeaV7LnagI+5UwNQ==
   dependencies:
-    "@jest/environment" "^26.6.2"
-    "@jest/fake-timers" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^27.5.1"
+    chalk "^4.0.0"
+    jest-get-type "^27.5.1"
+    jest-util "^27.5.1"
+    pretty-format "^27.5.1"
+
+jest-environment-jsdom@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-27.5.1.tgz#ea9ccd1fc610209655a77898f86b2b559516a546"
+  integrity sha512-TFBvkTC1Hnnnrka/fUb56atfDtJ9VMZ94JkjTbggl1PEpwrYtUBKMezB3inLmWqQsXYLcMwNoDQwoBTAvFfsfw==
+  dependencies:
+    "@jest/environment" "^27.5.1"
+    "@jest/fake-timers" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
-    jest-mock "^26.6.2"
-    jest-util "^26.6.2"
+    jest-mock "^27.5.1"
+    jest-util "^27.5.1"
+    jsdom "^16.6.0"
+
+jest-environment-node@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-27.5.1.tgz#dedc2cfe52fab6b8f5714b4808aefa85357a365e"
+  integrity sha512-Jt4ZUnxdOsTGwSRAfKEnE6BcwsSPNOijjwifq5sDFSA2kesnXTvNqKHYgM0hDq3549Uf/KzdXNYn4wMZJPlFLw==
+  dependencies:
+    "@jest/environment" "^27.5.1"
+    "@jest/fake-timers" "^27.5.1"
+    "@jest/types" "^27.5.1"
+    "@types/node" "*"
+    jest-mock "^27.5.1"
+    jest-util "^27.5.1"
 
 jest-get-type@^26.3.0:
   version "26.3.0"
   resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0"
   integrity sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==
 
-jest-haste-map@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-26.6.2.tgz#dd7e60fe7dc0e9f911a23d79c5ff7fb5c2cafeaa"
-  integrity sha512-easWIJXIw71B2RdR8kgqpjQrbMRWQBgiBwXYEhtGUTaX+doCjBheluShdDMeR8IMfJiTqH4+zfhtg29apJf/8w==
+jest-get-type@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1"
+  integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw==
+
+jest-haste-map@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-27.5.1.tgz#9fd8bd7e7b4fa502d9c6164c5640512b4e811e7f"
+  integrity sha512-7GgkZ4Fw4NFbMSDSpZwXeBiIbx+t/46nJ2QitkOjvwPYyZmqttu2TDSimMHP1EkPOi4xUZAN1doE5Vd25H4Jng==
   dependencies:
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^27.5.1"
     "@types/graceful-fs" "^4.1.2"
     "@types/node" "*"
     anymatch "^3.0.3"
     fb-watchman "^2.0.0"
-    graceful-fs "^4.2.4"
-    jest-regex-util "^26.0.0"
-    jest-serializer "^26.6.2"
-    jest-util "^26.6.2"
-    jest-worker "^26.6.2"
-    micromatch "^4.0.2"
-    sane "^4.0.3"
+    graceful-fs "^4.2.9"
+    jest-regex-util "^27.5.1"
+    jest-serializer "^27.5.1"
+    jest-util "^27.5.1"
+    jest-worker "^27.5.1"
+    micromatch "^4.0.4"
     walker "^1.0.7"
   optionalDependencies:
-    fsevents "^2.1.2"
+    fsevents "^2.3.2"
 
-jest-jasmine2@^26.6.3:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/jest-jasmine2/-/jest-jasmine2-26.6.3.tgz#adc3cf915deacb5212c93b9f3547cd12958f2edd"
-  integrity sha512-kPKUrQtc8aYwBV7CqBg5pu+tmYXlvFlSFYn18ev4gPFtrRzB15N2gW/Roew3187q2w2eHuu0MU9TJz6w0/nPEg==
+jest-jasmine2@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-jasmine2/-/jest-jasmine2-27.5.1.tgz#a037b0034ef49a9f3d71c4375a796f3b230d1ac4"
+  integrity sha512-jtq7VVyG8SqAorDpApwiJJImd0V2wv1xzdheGHRGyuT7gZm6gG47QEskOlzsN1PG/6WNaCo5pmwMHDf3AkG2pQ==
   dependencies:
-    "@babel/traverse" "^7.1.0"
-    "@jest/environment" "^26.6.2"
-    "@jest/source-map" "^26.6.2"
-    "@jest/test-result" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/environment" "^27.5.1"
+    "@jest/source-map" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
     chalk "^4.0.0"
     co "^4.6.0"
-    expect "^26.6.2"
+    expect "^27.5.1"
     is-generator-fn "^2.0.0"
-    jest-each "^26.6.2"
-    jest-matcher-utils "^26.6.2"
-    jest-message-util "^26.6.2"
-    jest-runtime "^26.6.3"
-    jest-snapshot "^26.6.2"
-    jest-util "^26.6.2"
-    pretty-format "^26.6.2"
-    throat "^5.0.0"
-
-jest-leak-detector@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-26.6.2.tgz#7717cf118b92238f2eba65054c8a0c9c653a91af"
-  integrity sha512-i4xlXpsVSMeKvg2cEKdfhh0H39qlJlP5Ex1yQxwF9ubahboQYMgTtz5oML35AVA3B4Eu+YsmwaiKVev9KCvLxg==
+    jest-each "^27.5.1"
+    jest-matcher-utils "^27.5.1"
+    jest-message-util "^27.5.1"
+    jest-runtime "^27.5.1"
+    jest-snapshot "^27.5.1"
+    jest-util "^27.5.1"
+    pretty-format "^27.5.1"
+    throat "^6.0.1"
+
+jest-leak-detector@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-27.5.1.tgz#6ec9d54c3579dd6e3e66d70e3498adf80fde3fb8"
+  integrity sha512-POXfWAMvfU6WMUXftV4HolnJfnPOGEu10fscNCA76KBpRRhcMN2c8d3iT2pxQS3HLbA+5X4sOUPzYO2NUyIlHQ==
+  dependencies:
+    jest-get-type "^27.5.1"
+    pretty-format "^27.5.1"
+
+jest-matcher-utils@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab"
+  integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw==
   dependencies:
-    jest-get-type "^26.3.0"
-    pretty-format "^26.6.2"
+    chalk "^4.0.0"
+    jest-diff "^27.5.1"
+    jest-get-type "^27.5.1"
+    pretty-format "^27.5.1"
 
-jest-matcher-utils@^26.6.0, jest-matcher-utils@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-26.6.2.tgz#8e6fd6e863c8b2d31ac6472eeb237bc595e53e7a"
-  integrity sha512-llnc8vQgYcNqDrqRDXWwMr9i7rS5XFiCwvh6DTP7Jqa2mqpcCBBlpCbn+trkG0KNhPu/h8rzyBkriOtBstvWhw==
+jest-message-util@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-27.5.1.tgz#bdda72806da10d9ed6425e12afff38cd1458b6cf"
+  integrity sha512-rMyFe1+jnyAAf+NHwTclDz0eAaLkVDdKVHHBFWsBWHnnh5YeJMNWWsv7AbFYXfK3oTqvL7VTWkhNLu1jX24D+g==
   dependencies:
+    "@babel/code-frame" "^7.12.13"
+    "@jest/types" "^27.5.1"
+    "@types/stack-utils" "^2.0.0"
     chalk "^4.0.0"
-    jest-diff "^26.6.2"
-    jest-get-type "^26.3.0"
-    pretty-format "^26.6.2"
+    graceful-fs "^4.2.9"
+    micromatch "^4.0.4"
+    pretty-format "^27.5.1"
+    slash "^3.0.0"
+    stack-utils "^2.0.3"
 
-jest-message-util@^26.6.0, jest-message-util@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-26.6.2.tgz#58173744ad6fc0506b5d21150b9be56ef001ca07"
-  integrity sha512-rGiLePzQ3AzwUshu2+Rn+UMFk0pHN58sOG+IaJbk5Jxuqo3NYO1U2/MIR4S1sKgsoYSXSzdtSa0TgrmtUwEbmA==
+jest-message-util@^28.1.3:
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-28.1.3.tgz#232def7f2e333f1eecc90649b5b94b0055e7c43d"
+  integrity sha512-PFdn9Iewbt575zKPf1286Ht9EPoJmYT7P0kY+RibeYZ2XtOr53pDLEFoTWXbd1h4JiGiWpTBC84fc8xMXQMb7g==
   dependencies:
-    "@babel/code-frame" "^7.0.0"
-    "@jest/types" "^26.6.2"
+    "@babel/code-frame" "^7.12.13"
+    "@jest/types" "^28.1.3"
     "@types/stack-utils" "^2.0.0"
     chalk "^4.0.0"
-    graceful-fs "^4.2.4"
-    micromatch "^4.0.2"
-    pretty-format "^26.6.2"
+    graceful-fs "^4.2.9"
+    micromatch "^4.0.4"
+    pretty-format "^28.1.3"
     slash "^3.0.0"
-    stack-utils "^2.0.2"
+    stack-utils "^2.0.3"
 
-jest-mock@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-26.6.2.tgz#d6cb712b041ed47fe0d9b6fc3474bc6543feb302"
-  integrity sha512-YyFjePHHp1LzpzYcmgqkJ0nm0gg/lJx2aZFzFy1S6eUqNjXsOqTK10zNRff2dNfssgokjkG65OlWNcIlgd3zew==
+jest-mock@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-27.5.1.tgz#19948336d49ef4d9c52021d34ac7b5f36ff967d6"
+  integrity sha512-K4jKbY1d4ENhbrG2zuPWaQBvDly+iZ2yAW+T1fATN78hc0sInwn7wZB8XtlNnvHug5RMwV897Xm4LqmPM4e2Og==
   dependencies:
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
 
 jest-pnp-resolver@^1.2.2:
@@ -11908,196 +10446,209 @@ jest-pnp-resolver@^1.2.2:
   resolved "https://registry.yarnpkg.com/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz#b704ac0ae028a89108a4d040b3f919dfddc8e33c"
   integrity sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w==
 
-jest-regex-util@^26.0.0:
-  version "26.0.0"
-  resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-26.0.0.tgz#d25e7184b36e39fd466c3bc41be0971e821fee28"
-  integrity sha512-Gv3ZIs/nA48/Zvjrl34bf+oD76JHiGDUxNOVgUjh3j890sblXryjY4rss71fPtD/njchl6PSE2hIhvyWa1eT0A==
+jest-regex-util@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-27.5.1.tgz#4da143f7e9fd1e542d4aa69617b38e4a78365b95"
+  integrity sha512-4bfKq2zie+x16okqDXjXn9ql2B0dScQu+vcwe4TvFVhkVyuWLqpZrZtXxLLWoXYgn0E87I6r6GRYHF7wFZBUvg==
 
-jest-resolve-dependencies@^26.6.3:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-26.6.3.tgz#6680859ee5d22ee5dcd961fe4871f59f4c784fb6"
-  integrity sha512-pVwUjJkxbhe4RY8QEWzN3vns2kqyuldKpxlxJlzEYfKSvY6/bMvxoFrYYzUO1Gx28yKWN37qyV7rIoIp2h8fTg==
-  dependencies:
-    "@jest/types" "^26.6.2"
-    jest-regex-util "^26.0.0"
-    jest-snapshot "^26.6.2"
+jest-regex-util@^28.0.0:
+  version "28.0.2"
+  resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-28.0.2.tgz#afdc377a3b25fb6e80825adcf76c854e5bf47ead"
+  integrity sha512-4s0IgyNIy0y9FK+cjoVYoxamT7Zeo7MhzqRGx7YDYmaQn1wucY9rotiGkBzzcMXTtjrCAP/f7f+E0F7+fxPNdw==
 
-jest-resolve@26.6.0:
-  version "26.6.0"
-  resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-26.6.0.tgz#070fe7159af87b03e50f52ea5e17ee95bbee40e1"
-  integrity sha512-tRAz2bwraHufNp+CCmAD8ciyCpXCs1NQxB5EJAmtCFy6BN81loFEGWKzYu26Y62lAJJe4X4jg36Kf+NsQyiStQ==
+jest-resolve-dependencies@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-27.5.1.tgz#d811ecc8305e731cc86dd79741ee98fed06f1da8"
+  integrity sha512-QQOOdY4PE39iawDn5rzbIePNigfe5B9Z91GDD1ae/xNDlu9kaat8QQ5EKnNmVWPV54hUdxCVwwj6YMgR2O7IOg==
   dependencies:
-    "@jest/types" "^26.6.0"
-    chalk "^4.0.0"
-    graceful-fs "^4.2.4"
-    jest-pnp-resolver "^1.2.2"
-    jest-util "^26.6.0"
-    read-pkg-up "^7.0.1"
-    resolve "^1.17.0"
-    slash "^3.0.0"
+    "@jest/types" "^27.5.1"
+    jest-regex-util "^27.5.1"
+    jest-snapshot "^27.5.1"
 
-jest-resolve@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-26.6.2.tgz#a3ab1517217f469b504f1b56603c5bb541fbb507"
-  integrity sha512-sOxsZOq25mT1wRsfHcbtkInS+Ek7Q8jCHUB0ZUTP0tc/c41QHriU/NunqMfCUWsL4H3MHpvQD4QR9kSYhS7UvQ==
+jest-resolve@^27.4.2, jest-resolve@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-27.5.1.tgz#a2f1c5a0796ec18fe9eb1536ac3814c23617b384"
+  integrity sha512-FFDy8/9E6CV83IMbDpcjOhumAQPDyETnU2KZ1O98DwTnz8AOBsW/Xv3GySr1mOZdItLR+zDZ7I/UdTFbgSOVCw==
   dependencies:
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^27.5.1"
     chalk "^4.0.0"
-    graceful-fs "^4.2.4"
+    graceful-fs "^4.2.9"
+    jest-haste-map "^27.5.1"
     jest-pnp-resolver "^1.2.2"
-    jest-util "^26.6.2"
-    read-pkg-up "^7.0.1"
-    resolve "^1.18.1"
+    jest-util "^27.5.1"
+    jest-validate "^27.5.1"
+    resolve "^1.20.0"
+    resolve.exports "^1.1.0"
     slash "^3.0.0"
 
-jest-runner@^26.6.0, jest-runner@^26.6.3:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-26.6.3.tgz#2d1fed3d46e10f233fd1dbd3bfaa3fe8924be159"
-  integrity sha512-atgKpRHnaA2OvByG/HpGA4g6CSPS/1LK0jK3gATJAoptC1ojltpmVlYC3TYgdmGp+GLuhzpH30Gvs36szSL2JQ==
+jest-runner@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-27.5.1.tgz#071b27c1fa30d90540805c5645a0ec167c7b62e5"
+  integrity sha512-g4NPsM4mFCOwFKXO4p/H/kWGdJp9V8kURY2lX8Me2drgXqG7rrZAx5kv+5H7wtt/cdFIjhqYx1HrlqWHaOvDaQ==
   dependencies:
-    "@jest/console" "^26.6.2"
-    "@jest/environment" "^26.6.2"
-    "@jest/test-result" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/console" "^27.5.1"
+    "@jest/environment" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/transform" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
     chalk "^4.0.0"
-    emittery "^0.7.1"
-    exit "^0.1.2"
-    graceful-fs "^4.2.4"
-    jest-config "^26.6.3"
-    jest-docblock "^26.0.0"
-    jest-haste-map "^26.6.2"
-    jest-leak-detector "^26.6.2"
-    jest-message-util "^26.6.2"
-    jest-resolve "^26.6.2"
-    jest-runtime "^26.6.3"
-    jest-util "^26.6.2"
-    jest-worker "^26.6.2"
+    emittery "^0.8.1"
+    graceful-fs "^4.2.9"
+    jest-docblock "^27.5.1"
+    jest-environment-jsdom "^27.5.1"
+    jest-environment-node "^27.5.1"
+    jest-haste-map "^27.5.1"
+    jest-leak-detector "^27.5.1"
+    jest-message-util "^27.5.1"
+    jest-resolve "^27.5.1"
+    jest-runtime "^27.5.1"
+    jest-util "^27.5.1"
+    jest-worker "^27.5.1"
     source-map-support "^0.5.6"
-    throat "^5.0.0"
-
-jest-runtime@^26.6.0, jest-runtime@^26.6.3:
-  version "26.6.3"
-  resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-26.6.3.tgz#4f64efbcfac398331b74b4b3c82d27d401b8fa2b"
-  integrity sha512-lrzyR3N8sacTAMeonbqpnSka1dHNux2uk0qqDXVkMv2c/A3wYnvQ4EXuI013Y6+gSKSCxdaczvf4HF0mVXHRdw==
-  dependencies:
-    "@jest/console" "^26.6.2"
-    "@jest/environment" "^26.6.2"
-    "@jest/fake-timers" "^26.6.2"
-    "@jest/globals" "^26.6.2"
-    "@jest/source-map" "^26.6.2"
-    "@jest/test-result" "^26.6.2"
-    "@jest/transform" "^26.6.2"
-    "@jest/types" "^26.6.2"
-    "@types/yargs" "^15.0.0"
+    throat "^6.0.1"
+
+jest-runtime@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-27.5.1.tgz#4896003d7a334f7e8e4a53ba93fb9bcd3db0a1af"
+  integrity sha512-o7gxw3Gf+H2IGt8fv0RiyE1+r83FJBRruoA+FXrlHw6xEyBsU8ugA6IPfTdVyA0w8HClpbK+DGJxH59UrNMx8A==
+  dependencies:
+    "@jest/environment" "^27.5.1"
+    "@jest/fake-timers" "^27.5.1"
+    "@jest/globals" "^27.5.1"
+    "@jest/source-map" "^27.5.1"
+    "@jest/test-result" "^27.5.1"
+    "@jest/transform" "^27.5.1"
+    "@jest/types" "^27.5.1"
     chalk "^4.0.0"
-    cjs-module-lexer "^0.6.0"
+    cjs-module-lexer "^1.0.0"
     collect-v8-coverage "^1.0.0"
-    exit "^0.1.2"
+    execa "^5.0.0"
     glob "^7.1.3"
-    graceful-fs "^4.2.4"
-    jest-config "^26.6.3"
-    jest-haste-map "^26.6.2"
-    jest-message-util "^26.6.2"
-    jest-mock "^26.6.2"
-    jest-regex-util "^26.0.0"
-    jest-resolve "^26.6.2"
-    jest-snapshot "^26.6.2"
-    jest-util "^26.6.2"
-    jest-validate "^26.6.2"
+    graceful-fs "^4.2.9"
+    jest-haste-map "^27.5.1"
+    jest-message-util "^27.5.1"
+    jest-mock "^27.5.1"
+    jest-regex-util "^27.5.1"
+    jest-resolve "^27.5.1"
+    jest-snapshot "^27.5.1"
+    jest-util "^27.5.1"
     slash "^3.0.0"
     strip-bom "^4.0.0"
-    yargs "^15.4.1"
 
-jest-serializer@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-serializer/-/jest-serializer-26.6.2.tgz#d139aafd46957d3a448f3a6cdabe2919ba0742d1"
-  integrity sha512-S5wqyz0DXnNJPd/xfIzZ5Xnp1HrJWBczg8mMfMpN78OJ5eDxXyf+Ygld9wX1DnUWbIbhM1YDY95NjR4CBXkb2g==
+jest-serializer@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-serializer/-/jest-serializer-27.5.1.tgz#81438410a30ea66fd57ff730835123dea1fb1f64"
+  integrity sha512-jZCyo6iIxO1aqUxpuBlwTDMkzOAJS4a3eYz3YzgxxVQFwLeSA7Jfq5cbqCY+JLvTDrWirgusI/0KwxKMgrdf7w==
   dependencies:
     "@types/node" "*"
-    graceful-fs "^4.2.4"
+    graceful-fs "^4.2.9"
 
-jest-snapshot@^26.6.0, jest-snapshot@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-26.6.2.tgz#f3b0af1acb223316850bd14e1beea9837fb39c84"
-  integrity sha512-OLhxz05EzUtsAmOMzuupt1lHYXCNib0ECyuZ/PZOx9TrZcC8vL0x+DUG3TL+GLX3yHG45e6YGjIm0XwDc3q3og==
+jest-snapshot@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-27.5.1.tgz#b668d50d23d38054a51b42c4039cab59ae6eb6a1"
+  integrity sha512-yYykXI5a0I31xX67mgeLw1DZ0bJB+gpq5IpSuCAoyDi0+BhgU/RIrL+RTzDmkNTchvDFWKP8lp+w/42Z3us5sA==
   dependencies:
+    "@babel/core" "^7.7.2"
+    "@babel/generator" "^7.7.2"
+    "@babel/plugin-syntax-typescript" "^7.7.2"
+    "@babel/traverse" "^7.7.2"
     "@babel/types" "^7.0.0"
-    "@jest/types" "^26.6.2"
+    "@jest/transform" "^27.5.1"
+    "@jest/types" "^27.5.1"
     "@types/babel__traverse" "^7.0.4"
-    "@types/prettier" "^2.0.0"
+    "@types/prettier" "^2.1.5"
+    babel-preset-current-node-syntax "^1.0.0"
     chalk "^4.0.0"
-    expect "^26.6.2"
-    graceful-fs "^4.2.4"
-    jest-diff "^26.6.2"
-    jest-get-type "^26.3.0"
-    jest-haste-map "^26.6.2"
-    jest-matcher-utils "^26.6.2"
-    jest-message-util "^26.6.2"
-    jest-resolve "^26.6.2"
+    expect "^27.5.1"
+    graceful-fs "^4.2.9"
+    jest-diff "^27.5.1"
+    jest-get-type "^27.5.1"
+    jest-haste-map "^27.5.1"
+    jest-matcher-utils "^27.5.1"
+    jest-message-util "^27.5.1"
+    jest-util "^27.5.1"
     natural-compare "^1.4.0"
-    pretty-format "^26.6.2"
+    pretty-format "^27.5.1"
     semver "^7.3.2"
 
-jest-util@^26.6.0, jest-util@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.6.2.tgz#907535dbe4d5a6cb4c47ac9b926f6af29576cbc1"
-  integrity sha512-MDW0fKfsn0OI7MS7Euz6h8HNDXVQ0gaM9uW6RjfDmd1DAFcaxX9OqIakHIqhbnmF08Cf2DLDG+ulq8YQQ0Lp0Q==
+jest-util@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-27.5.1.tgz#3ba9771e8e31a0b85da48fe0b0891fb86c01c2f9"
+  integrity sha512-Kv2o/8jNvX1MQ0KGtw480E/w4fBCDOnH6+6DmeKi6LZUIlKA5kwY0YNdlzaWTiVgxqAqik11QyxDOKk543aKXw==
   dependencies:
-    "@jest/types" "^26.6.2"
+    "@jest/types" "^27.5.1"
     "@types/node" "*"
     chalk "^4.0.0"
-    graceful-fs "^4.2.4"
-    is-ci "^2.0.0"
-    micromatch "^4.0.2"
+    ci-info "^3.2.0"
+    graceful-fs "^4.2.9"
+    picomatch "^2.2.3"
 
-jest-validate@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-26.6.2.tgz#23d380971587150467342911c3d7b4ac57ab20ec"
-  integrity sha512-NEYZ9Aeyj0i5rQqbq+tpIOom0YS1u2MVu6+euBsvpgIme+FOfRmoC4R5p0JiAUpaFvFy24xgrpMknarR/93XjQ==
+jest-util@^28.1.3:
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-28.1.3.tgz#f4f932aa0074f0679943220ff9cbba7e497028b0"
+  integrity sha512-XdqfpHwpcSRko/C35uLYFM2emRAltIIKZiJ9eAmhjsj0CqZMa0p1ib0R5fWIqGhn1a103DebTbpqIaP1qCQ6tQ==
   dependencies:
-    "@jest/types" "^26.6.2"
-    camelcase "^6.0.0"
+    "@jest/types" "^28.1.3"
+    "@types/node" "*"
     chalk "^4.0.0"
-    jest-get-type "^26.3.0"
+    ci-info "^3.2.0"
+    graceful-fs "^4.2.9"
+    picomatch "^2.2.3"
+
+jest-validate@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-27.5.1.tgz#9197d54dc0bdb52260b8db40b46ae668e04df067"
+  integrity sha512-thkNli0LYTmOI1tDB3FI1S1RTp/Bqyd9pTarJwL87OIBFuqEb5Apv5EaApEudYg4g86e3CT6kM0RowkhtEnCBQ==
+  dependencies:
+    "@jest/types" "^27.5.1"
+    camelcase "^6.2.0"
+    chalk "^4.0.0"
+    jest-get-type "^27.5.1"
     leven "^3.1.0"
-    pretty-format "^26.6.2"
+    pretty-format "^27.5.1"
 
-jest-watch-typeahead@0.6.1:
-  version "0.6.1"
-  resolved "https://registry.yarnpkg.com/jest-watch-typeahead/-/jest-watch-typeahead-0.6.1.tgz#45221b86bb6710b7e97baaa1640ae24a07785e63"
-  integrity sha512-ITVnHhj3Jd/QkqQcTqZfRgjfyRhDFM/auzgVo2RKvSwi18YMvh0WvXDJFoFED6c7jd/5jxtu4kSOb9PTu2cPVg==
+jest-watch-typeahead@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/jest-watch-typeahead/-/jest-watch-typeahead-1.1.0.tgz#b4a6826dfb9c9420da2f7bc900de59dad11266a9"
+  integrity sha512-Va5nLSJTN7YFtC2jd+7wsoe1pNe5K4ShLux/E5iHEwlB9AxaxmggY7to9KUqKojhaJw3aXqt5WAb4jGPOolpEw==
   dependencies:
     ansi-escapes "^4.3.1"
     chalk "^4.0.0"
-    jest-regex-util "^26.0.0"
-    jest-watcher "^26.3.0"
-    slash "^3.0.0"
+    jest-regex-util "^28.0.0"
+    jest-watcher "^28.0.0"
+    slash "^4.0.0"
+    string-length "^5.0.1"
+    strip-ansi "^7.0.1"
+
+jest-watcher@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-27.5.1.tgz#71bd85fb9bde3a2c2ec4dc353437971c43c642a2"
+  integrity sha512-z676SuD6Z8o8qbmEGhoEUFOM1+jfEiL3DXHK/xgEiG2EyNYfFG60jluWcupY6dATjfEsKQuibReS1djInQnoVw==
+  dependencies:
+    "@jest/test-result" "^27.5.1"
+    "@jest/types" "^27.5.1"
+    "@types/node" "*"
+    ansi-escapes "^4.2.1"
+    chalk "^4.0.0"
+    jest-util "^27.5.1"
     string-length "^4.0.1"
-    strip-ansi "^6.0.0"
 
-jest-watcher@^26.3.0, jest-watcher@^26.6.2:
-  version "26.6.2"
-  resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-26.6.2.tgz#a5b683b8f9d68dbcb1d7dae32172d2cca0592975"
-  integrity sha512-WKJob0P/Em2csiVthsI68p6aGKTIcsfjH9Gsx1f0A3Italz43e3ho0geSAVsmj09RWOELP1AZ/DXyJgOgDKxXQ==
+jest-watcher@^28.0.0:
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-28.1.3.tgz#c6023a59ba2255e3b4c57179fc94164b3e73abd4"
+  integrity sha512-t4qcqj9hze+jviFPUN3YAtAEeFnr/azITXQEMARf5cMwKY2SMBRnCQTXLixTl20OR6mLh9KLMrgVJgJISym+1g==
   dependencies:
-    "@jest/test-result" "^26.6.2"
-    "@jest/types" "^26.6.2"
+    "@jest/test-result" "^28.1.3"
+    "@jest/types" "^28.1.3"
     "@types/node" "*"
     ansi-escapes "^4.2.1"
     chalk "^4.0.0"
-    jest-util "^26.6.2"
+    emittery "^0.10.2"
+    jest-util "^28.1.3"
     string-length "^4.0.1"
 
-jest-worker@^24.9.0:
-  version "24.9.0"
-  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.9.0.tgz#5dbfdb5b2d322e98567898238a9697bcce67b3e5"
-  integrity sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw==
-  dependencies:
-    merge-stream "^2.0.0"
-    supports-color "^6.1.0"
-
-jest-worker@^26.5.0, jest-worker@^26.6.2:
+jest-worker@^26.2.1:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-26.6.2.tgz#7f72cbc4d643c365e27b9fd775f9d0eaa9c7a8ed"
   integrity sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==
@@ -12106,14 +10657,37 @@ jest-worker@^26.5.0, jest-worker@^26.6.2:
     merge-stream "^2.0.0"
     supports-color "^7.0.0"
 
-jest@26.6.0:
-  version "26.6.0"
-  resolved "https://registry.yarnpkg.com/jest/-/jest-26.6.0.tgz#546b25a1d8c888569dbbe93cae131748086a4a25"
-  integrity sha512-jxTmrvuecVISvKFFhOkjsWRZV7sFqdSUAd1ajOKY+/QE/aLBVstsJ/dX8GczLzwiT6ZEwwmZqtCUHLHHQVzcfA==
+jest-worker@^27.0.2, jest-worker@^27.4.5, jest-worker@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.5.1.tgz#8d146f0900e8973b106b6f73cc1e9a8cb86f8db0"
+  integrity sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==
+  dependencies:
+    "@types/node" "*"
+    merge-stream "^2.0.0"
+    supports-color "^8.0.0"
+
+jest-worker@^28.0.2:
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-28.1.3.tgz#7e3c4ce3fa23d1bb6accb169e7f396f98ed4bb98"
+  integrity sha512-CqRA220YV/6jCo8VWvAt1KKx6eek1VIHMPeLEbpcfSfkEeWyBNppynM/o6q+Wmw+sOhos2ml34wZbSX3G13//g==
+  dependencies:
+    "@types/node" "*"
+    merge-stream "^2.0.0"
+    supports-color "^8.0.0"
+
+jest@^27.4.3:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/jest/-/jest-27.5.1.tgz#dadf33ba70a779be7a6fc33015843b51494f63fc"
+  integrity sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==
   dependencies:
-    "@jest/core" "^26.6.0"
+    "@jest/core" "^27.5.1"
     import-local "^3.0.2"
-    jest-cli "^26.6.0"
+    jest-cli "^27.5.1"
+
+jiti@^1.18.2:
+  version "1.18.2"
+  resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.18.2.tgz#80c3ef3d486ebf2450d9335122b32d121f2a83cd"
+  integrity sha512-QAdOptna2NYiSSpv0O/BwoHBSmz4YhpzJHyi+fnMRTXFjp7B8i/YG5Z8IfusxB1ufjcD2Sre1F3R+nX3fvy7gg==
 
 joi@^17.3.0:
   version "17.4.1"
@@ -12144,7 +10718,7 @@ js-yaml@^3.13.1:
     argparse "^1.0.7"
     esprima "^4.0.0"
 
-js-yaml@^4.0.0:
+js-yaml@^4.0.0, js-yaml@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602"
   integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==
@@ -12156,10 +10730,10 @@ jsbn@~0.1.0:
   resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
   integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM=
 
-jsdom@^16.4.0:
-  version "16.6.0"
-  resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.6.0.tgz#f79b3786682065492a3da6a60a4695da983805ac"
-  integrity sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==
+jsdom@^16.6.0:
+  version "16.7.0"
+  resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.7.0.tgz#918ae71965424b197c819f8183a754e18977b710"
+  integrity sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==
   dependencies:
     abab "^2.0.5"
     acorn "^8.2.4"
@@ -12186,7 +10760,7 @@ jsdom@^16.4.0:
     whatwg-encoding "^1.0.5"
     whatwg-mimetype "^2.3.0"
     whatwg-url "^8.5.0"
-    ws "^7.4.5"
+    ws "^7.4.6"
     xml-name-validator "^3.0.0"
 
 jsesc@^2.5.1:
@@ -12204,12 +10778,7 @@ json-buffer@3.0.0:
   resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898"
   integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=
 
-json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
-  integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==
-
-json-parse-even-better-errors@^2.3.0:
+json-parse-even-better-errors@^2.3.0, json-parse-even-better-errors@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d"
   integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==
@@ -12229,6 +10798,11 @@ json-schema@0.2.3:
   resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
   integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=
 
+json-schema@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5"
+  integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==
+
 json-stable-stringify-without-jsonify@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
@@ -12261,26 +10835,14 @@ json2mq@^0.2.0:
   dependencies:
     string-convert "^0.2.0"
 
-json3@^3.3.3:
-  version "3.3.3"
-  resolved "https://registry.yarnpkg.com/json3/-/json3-3.3.3.tgz#7fc10e375fc5ae42c4705a5cc0aa6f62be305b81"
-  integrity sha512-c7/8mbUsKigAbLkD5B010BK4D9LZm7A1pNItkEwiUZRpIN66exu/e7YQWysGun+TRKaJp8MhemM+VkfWv42aCA==
-
-json5@^1.0.1:
+json5@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
   integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==
   dependencies:
     minimist "^1.2.0"
 
-json5@^2.1.2:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3"
-  integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==
-  dependencies:
-    minimist "^1.2.5"
-
-json5@^2.2.2:
+json5@^2.1.2, json5@^2.2.0, json5@^2.2.2:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
   integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
@@ -12306,6 +10868,11 @@ jsonify@~0.0.0:
   resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
   integrity sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=
 
+jsonpointer@^5.0.0:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559"
+  integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==
+
 jsonwebtoken@^8.5.1:
   version "8.5.1"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
@@ -12332,13 +10899,13 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
-"jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.1.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-3.2.0.tgz#41108d2cec408c3453c1bbe8a4aae9e1e2bd8f82"
-  integrity sha512-EIsmt3O3ljsU6sot/J4E1zDRxfBNrhjyf/OKjlydwgEimQuznlM4Wv7U+ueONJMyEn1WRE0K8dhi3dVAXYT24Q==
+"jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.3.3:
+  version "3.3.3"
+  resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz#76b3e6e6cece5c69d49a5792c3d01bd1a0cdc7ea"
+  integrity sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==
   dependencies:
-    array-includes "^3.1.2"
-    object.assign "^4.1.2"
+    array-includes "^3.1.5"
+    object.assign "^4.1.3"
 
 jsx-dom-cjs@^8.0.0:
   version "8.0.3"
@@ -12376,11 +10943,6 @@ keyv@^3.0.0:
   dependencies:
     json-buffer "3.0.0"
 
-killable@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/killable/-/killable-1.0.1.tgz#4c8ce441187a061c7474fb87ca08e2a638194892"
-  integrity sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg==
-
 kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
@@ -12410,31 +10972,23 @@ kleur@^3.0.3:
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
   integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==
 
-klona@^2.0.4:
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.4.tgz#7bb1e3affb0cb8624547ef7e8f6708ea2e39dfc0"
-  integrity sha512-ZRbnvdg/NxqzC7L9Uyqzf4psi1OM4Cuc+sJAkQPjO6XkQIJTNbfK2Rsmbw8fx1p2mkZdp2FZYo2+LwXYY/uwIA==
+klona@^2.0.4, klona@^2.0.5:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.6.tgz#85bffbf819c03b2f53270412420a4555ef882e22"
+  integrity sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==
 
 language-subtag-registry@~0.3.2:
   version "0.3.21"
   resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a"
   integrity sha512-L0IqwlIXjilBVVYKFT37X9Ih11Um5NEl9cbJIuU/SwP/zEEAbBPOnEeeuxVMf45ydWQRDQN3Nqc96OgbH1K+Pg==
 
-language-tags@^1.0.5:
+language-tags@=1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/language-tags/-/language-tags-1.0.5.tgz#d321dbc4da30ba8bf3024e040fa5c14661f9193a"
-  integrity sha1-0yHbxNowuovzAk4ED6XBRmH5GTo=
+  integrity sha512-qJhlO9cGXi6hBGKoxEG/sKZDAHD5Hnu9Hs4WbOY3pCWXDhw0N8x1NenNzm2EnNLkLkk7J2SdxAkDSbb6ftT+UQ==
   dependencies:
     language-subtag-registry "~0.3.2"
 
-last-call-webpack-plugin@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/last-call-webpack-plugin/-/last-call-webpack-plugin-3.0.0.tgz#9742df0e10e3cf46e5c0381c2de90d3a7a2d7555"
-  integrity sha512-7KI2l2GIZa9p2spzPIVZBYyNKkN+e/SQPpnjlTiPhdbDW3F86tdKKELxKpzJ5sgU19wQWsACULZmpTPYHeWO5w==
-  dependencies:
-    lodash "^4.17.5"
-    webpack-sources "^1.1.0"
-
 latest-version@5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face"
@@ -12442,42 +10996,50 @@ latest-version@5.1.0:
   dependencies:
     package-json "^6.3.0"
 
+launch-editor@^2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/launch-editor/-/launch-editor-2.6.0.tgz#4c0c1a6ac126c572bd9ff9a30da1d2cae66defd7"
+  integrity sha512-JpDCcQnyAAzZZaZ7vEiSqL690w7dAEyLao+KC96zBplnYbJS7TYNjvM3M7y3dGz+v7aIsJk3hllWuc0kWAjyRQ==
+  dependencies:
+    picocolors "^1.0.0"
+    shell-quote "^1.7.3"
+
 lazy-ass@1.6.0:
   version "1.6.0"
   resolved "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513"
   integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM=
 
-less-loader@^6.1.0:
-  version "6.2.0"
-  resolved "https://registry.yarnpkg.com/less-loader/-/less-loader-6.2.0.tgz#8b26f621c155b342eefc24f5bd6e9dc40c42a719"
-  integrity sha512-Cl5h95/Pz/PWub/tCBgT1oNMFeH1WTD33piG80jn5jr12T4XbxZcjThwNXDQ7AG649WEynuIzO4b0+2Tn9Qolg==
+less-loader@^7.3.0:
+  version "7.3.0"
+  resolved "https://registry.yarnpkg.com/less-loader/-/less-loader-7.3.0.tgz#f9d6d36d18739d642067a05fb5bd70c8c61317e5"
+  integrity sha512-Mi8915g7NMaLlgi77mgTTQvK022xKRQBIVDSyfl3ErTuBhmZBQab0mjeJjNNqGbdR+qrfTleKXqbGI4uEFavxg==
   dependencies:
-    clone "^2.1.2"
-    less "^3.11.3"
+    klona "^2.0.4"
     loader-utils "^2.0.0"
-    schema-utils "^2.7.0"
+    schema-utils "^3.0.0"
 
-less-vars-to-js@^1.3.0:
+less-vars-to-js@1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/less-vars-to-js/-/less-vars-to-js-1.3.0.tgz#c322cf43a3c8fc3fab655da3e51a14c1499ab571"
   integrity sha512-xeiLLn/IMCGtdyCkYQnW8UuzoW2oYMCKg9boZRaGI58fLz5r90bNJDlqGzmVt/1Uqk75/DxIVtQSNCMkE5fRZQ==
   dependencies:
     strip-json-comments "^2.0.1"
 
-less@^3.11.1, less@^3.11.3:
-  version "3.13.1"
-  resolved "https://registry.yarnpkg.com/less/-/less-3.13.1.tgz#0ebc91d2a0e9c0c6735b83d496b0ab0583077909"
-  integrity sha512-SwA1aQXGUvp+P5XdZslUOhhLnClSLIjWvJhmd+Vgib5BFIr9lMNlQwmwUNOjXThF/A0x+MCYYPeWEfeWiLRnTw==
+less@^4.1.1:
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/less/-/less-4.1.3.tgz#175be9ddcbf9b250173e0a00b4d6920a5b770246"
+  integrity sha512-w16Xk/Ta9Hhyei0Gpz9m7VS8F28nieJaL/VyShID7cYvP6IL5oHeL6p4TXSDJqZE/lNv0oJ2pGVjJsRkfwm5FA==
   dependencies:
     copy-anything "^2.0.1"
-    tslib "^1.10.0"
+    parse-node-version "^1.0.1"
+    tslib "^2.3.0"
   optionalDependencies:
     errno "^0.1.1"
     graceful-fs "^4.1.2"
     image-size "~0.5.0"
     make-dir "^2.1.0"
     mime "^1.4.1"
-    native-request "^1.0.5"
+    needle "^3.1.0"
     source-map "~0.6.0"
 
 leven@^3.1.0:
@@ -12508,6 +11070,11 @@ lib0@^0.2.42, lib0@^0.2.49:
   dependencies:
     isomorphic.js "^0.2.4"
 
+lilconfig@^2.0.3, lilconfig@^2.0.5, lilconfig@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.1.0.tgz#78e23ac89ebb7e1bfbf25b18043de756548e7f52"
+  integrity sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==
+
 lines-and-columns@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -12557,55 +11124,24 @@ listr@^0.14.3:
     p-map "^2.0.0"
     rxjs "^6.3.3"
 
-load-json-file@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-4.0.0.tgz#2f5f45ab91e33216234fd53adab668eb4ec0993b"
-  integrity sha1-L19Fq5HjMhYjT9U62rZo607AmTs=
-  dependencies:
-    graceful-fs "^4.1.2"
-    parse-json "^4.0.0"
-    pify "^3.0.0"
-    strip-bom "^3.0.0"
-
-loader-runner@^2.4.0:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
-  integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==
-
-loader-utils@1.2.3:
-  version "1.2.3"
-  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7"
-  integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==
-  dependencies:
-    big.js "^5.2.2"
-    emojis-list "^2.0.0"
-    json5 "^1.0.1"
+loader-runner@^4.2.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1"
+  integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==
 
-loader-utils@2.0.0, loader-utils@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.0.tgz#e4cace5b816d425a166b5f097e10cd12b36064b0"
-  integrity sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==
+loader-utils@^2.0.0, loader-utils@^2.0.4:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c"
+  integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==
   dependencies:
     big.js "^5.2.2"
     emojis-list "^3.0.0"
     json5 "^2.1.2"
 
-loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613"
-  integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==
-  dependencies:
-    big.js "^5.2.2"
-    emojis-list "^3.0.0"
-    json5 "^1.0.1"
-
-locate-path@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"
-  integrity sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=
-  dependencies:
-    p-locate "^2.0.0"
-    path-exists "^3.0.0"
+loader-utils@^3.2.0:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-3.2.1.tgz#4fb104b599daafd82ef3e1a41fb9265f87e1f576"
+  integrity sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw==
 
 locate-path@^3.0.0:
   version "3.0.0"
@@ -12622,16 +11158,18 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+locate-path@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286"
+  integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==
+  dependencies:
+    p-locate "^5.0.0"
+
 lodash-es@^4.17.15:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee"
   integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==
 
-lodash._reinterpolate@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
-  integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=
-
 lodash.assign@^4.2.0:
   version "4.2.0"
   resolved "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
@@ -12772,25 +11310,10 @@ lodash.snakecase@^4.1.1:
   resolved "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz#39d714a35357147837aefd64b5dcbb16becd8f8d"
   integrity sha1-OdcUo1NXFHg3rv1ktdy7Fr7Nj40=
 
-lodash.template@^4.5.0:
-  version "4.5.0"
-  resolved "https://registry.yarnpkg.com/lodash.template/-/lodash.template-4.5.0.tgz#f976195cf3f347d0d5f52483569fe8031ccce8ab"
-  integrity sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==
-  dependencies:
-    lodash._reinterpolate "^3.0.0"
-    lodash.templatesettings "^4.0.0"
-
-lodash.templatesettings@^4.0.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz#e481310f049d3cf6d47e912ad09313b154f0fb33"
-  integrity sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==
-  dependencies:
-    lodash._reinterpolate "^3.0.0"
-
-lodash.truncate@^4.4.2:
-  version "4.4.2"
-  resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193"
-  integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=
+lodash.sortby@^4.7.0:
+  version "4.7.0"
+  resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
+  integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==
 
 lodash.uniq@^4.5.0:
   version "4.5.0"
@@ -12807,7 +11330,7 @@ lodash.values@^4.3.0:
   resolved "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347"
   integrity sha1-o6bCsOvsxcLLocF+bmIP6BtT00c=
 
-lodash@4.17.21, "lodash@>=3.5 <5", lodash@^4.0.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.7.0, lodash@~4.17.0:
+lodash@4.17.21, lodash@^4.0.1, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0, lodash@~4.17.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -12836,11 +11359,6 @@ log-update@^2.3.0:
     cli-cursor "^2.0.0"
     wrap-ansi "^3.0.1"
 
-loglevel@^1.6.8:
-  version "1.7.1"
-  resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.7.1.tgz#005fde2f5e6e47068f935ff28573e125ef72f197"
-  integrity sha512-Hesni4s5UkWkwCGJMQGAh71PaLUmKFM60dHvq0zi/vDhhrzuk+4GgNbTXJ12YYQJn6ZKBDNIjYcuQGKudvqrIw==
-
 longest-streak@^2.0.0:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-2.0.4.tgz#b8599957da5b5dab64dee3fe316fa774597d90e4"
@@ -12911,7 +11429,7 @@ magic-string@^0.25.0, magic-string@^0.25.7:
   dependencies:
     sourcemap-codec "^1.4.4"
 
-make-dir@^2.0.0, make-dir@^2.1.0:
+make-dir@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
   integrity sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==
@@ -12990,15 +11508,6 @@ math-expression-evaluator@^1.2.14:
   resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-1.3.7.tgz#1b62225db86af06f7ea1fd9576a34af605a5b253"
   integrity sha512-nrbaifCl42w37hYd6oRLvoymFK42tWB+WQTMFtksDGQMi5GvlJwnz/CsS30FFAISFLtX+A0csJ0xLiuuyyec7w==
 
-md5.js@^1.3.4:
-  version "1.3.5"
-  resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"
-  integrity sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==
-  dependencies:
-    hash-base "^3.0.0"
-    inherits "^2.0.1"
-    safe-buffer "^5.1.2"
-
 mdast-util-definitions@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/mdast-util-definitions/-/mdast-util-definitions-4.0.0.tgz#c5c1a84db799173b4dcf7643cda999e440c24db2"
@@ -13119,21 +11628,12 @@ media-typer@0.3.0:
   resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
   integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==
 
-memory-fs@^0.4.1:
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"
-  integrity sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=
-  dependencies:
-    errno "^0.1.3"
-    readable-stream "^2.0.1"
-
-memory-fs@^0.5.0:
-  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.5.0.tgz#324c01288b88652966d161db77838720845a8e3c"
-  integrity sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==
+memfs@^3.1.2, memfs@^3.4.3:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.6.0.tgz#d7a2110f86f79dd950a8b6df6d57bc984aa185f6"
+  integrity sha512-EGowvkkgbMcIChjMTMkESFDbZeSh8xZ7kNSF0hAiAN4Jh6jgHCRS0Ga/+C8y6Au+oqpezRHCfPsmJ2+DwAgiwQ==
   dependencies:
-    errno "^0.1.3"
-    readable-stream "^2.0.1"
+    fs-monkey "^1.0.4"
 
 merge-descriptors@1.0.1:
   version "1.0.1"
@@ -13145,7 +11645,7 @@ merge-stream@^2.0.0:
   resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
   integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
 
-merge2@^1.3.0:
+merge2@^1.3.0, merge2@^1.4.1:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
   integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
@@ -13165,11 +11665,6 @@ methods@~1.1.2:
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==
 
-microevent.ts@~0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/microevent.ts/-/microevent.ts-0.1.1.tgz#70b09b83f43df5172d0205a63025bce0f7357fa0"
-  integrity sha512-jo1OfR4TaEwd5HOrt5+tAZ9mqT4jmpNAusXtyfNzqVm9uiSYFZlKM1wYL4oU7azZW/PxQW53wM0S6OR1JHNa2g==
-
 micromark-extension-gfm-autolink-literal@~0.5.0:
   version "0.5.7"
   resolved "https://registry.yarnpkg.com/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-0.5.7.tgz#53866c1f0c7ef940ae7ca1f72c6faef8fed9f204"
@@ -13223,7 +11718,7 @@ micromark@^2.11.3, micromark@~2.11.0, micromark@~2.11.3:
     debug "^4.0.0"
     parse-entities "^2.0.0"
 
-micromatch@^3.1.10, micromatch@^3.1.4:
+micromatch@^3.1.4:
   version "3.1.10"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23"
   integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==
@@ -13242,40 +11737,20 @@ micromatch@^3.1.10, micromatch@^3.1.4:
     snapdragon "^0.8.1"
     to-regex "^3.0.2"
 
-micromatch@^4.0.2, micromatch@^4.0.4:
-  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9"
-  integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==
-  dependencies:
-    braces "^3.0.1"
-    picomatch "^2.2.3"
-
-miller-rabin@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/miller-rabin/-/miller-rabin-4.0.1.tgz#f080351c865b0dc562a8462966daa53543c78a4d"
-  integrity sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==
+micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5:
+  version "4.0.5"
+  resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6"
+  integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==
   dependencies:
-    bn.js "^4.0.0"
-    brorand "^1.0.1"
+    braces "^3.0.2"
+    picomatch "^2.3.1"
 
-mime-db@1.47.0, "mime-db@>= 1.43.0 < 2":
-  version "1.47.0"
-  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.47.0.tgz#8cb313e59965d3c05cfbf898915a267af46a335c"
-  integrity sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==
-
-mime-db@1.52.0:
+mime-db@1.52.0, "mime-db@>= 1.43.0 < 2":
   version "1.52.0"
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
   integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
 
-mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.17, mime-types@~2.1.19:
-  version "2.1.30"
-  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.30.tgz#6e7be8b4c479825f85ed6326695db73f9305d62d"
-  integrity sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==
-  dependencies:
-    mime-db "1.47.0"
-
-mime-types@~2.1.24, mime-types@~2.1.34:
+mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34:
   version "2.1.35"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
@@ -13287,11 +11762,6 @@ mime@1.6.0, mime@^1.4.1:
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
 
-mime@^2.4.4:
-  version "2.5.2"
-  resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe"
-  integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==
-
 mimic-fn@^1.0.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
@@ -13327,73 +11797,43 @@ mini-create-react-context@^0.4.0:
     "@babel/runtime" "^7.12.1"
     tiny-warning "^1.0.3"
 
-mini-css-extract-plugin@0.11.3:
-  version "0.11.3"
-  resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-0.11.3.tgz#15b0910a7f32e62ffde4a7430cfefbd700724ea6"
-  integrity sha512-n9BA8LonkOkW1/zn+IbLPQmovsL0wMb9yx75fMJQZf2X1Zoec9yTZtyMePcyu19wPkmFbzZZA6fLTotpFhQsOA==
+mini-css-extract-plugin@^2.4.5:
+  version "2.7.6"
+  resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-2.7.6.tgz#282a3d38863fddcd2e0c220aaed5b90bc156564d"
+  integrity sha512-Qk7HcgaPkGG6eD77mLvZS1nmxlao3j+9PkrT9Uc7HAE1id3F41+DdBRYRYkbyfNRGzm8/YWtzhw7nVPmwhqTQw==
   dependencies:
-    loader-utils "^1.1.0"
-    normalize-url "1.9.1"
-    schema-utils "^1.0.0"
-    webpack-sources "^1.1.0"
+    schema-utils "^4.0.0"
 
-minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1:
+minimalistic-assert@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz#2e194de044626d4a10e7f7fbc00ce73e83e4d5c7"
   integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==
 
-minimalistic-crypto-utils@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a"
-  integrity sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=
-
-minimatch@3.0.4, minimatch@^3.0.4:
-  version "3.0.4"
-  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
-  integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==
-  dependencies:
-    brace-expansion "^1.1.7"
-
-minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5:
-  version "1.2.7"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
-  integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==
-
-minipass-collect@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617"
-  integrity sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==
-  dependencies:
-    minipass "^3.0.0"
-
-minipass-flush@^1.0.5:
-  version "1.0.5"
-  resolved "https://registry.yarnpkg.com/minipass-flush/-/minipass-flush-1.0.5.tgz#82e7135d7e89a50ffe64610a787953c4c4cbb373"
-  integrity sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==
+minimatch@3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
+  integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==
   dependencies:
-    minipass "^3.0.0"
+    brace-expansion "^1.1.7"
 
-minipass-pipeline@^1.2.2:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz#68472f79711c084657c067c5c6ad93cddea8214c"
-  integrity sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==
+minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
+  integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
   dependencies:
-    minipass "^3.0.0"
+    brace-expansion "^1.1.7"
 
-minipass@^3.0.0, minipass@^3.1.1:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd"
-  integrity sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==
+minimatch@^5.0.1:
+  version "5.1.6"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96"
+  integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==
   dependencies:
-    yallist "^4.0.0"
+    brace-expansion "^2.0.1"
 
-minizlib@^2.1.1:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931"
-  integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==
-  dependencies:
-    minipass "^3.0.0"
-    yallist "^4.0.0"
+minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
+  integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
 
 miragejs@^0.1.0, miragejs@^0.1.41:
   version "0.1.41"
@@ -13427,21 +11867,10 @@ miragejs@^0.1.0, miragejs@^0.1.41:
     lodash.values "^4.3.0"
     pretender "^3.4.3"
 
-mississippi@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022"
-  integrity sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==
-  dependencies:
-    concat-stream "^1.5.0"
-    duplexify "^3.4.2"
-    end-of-stream "^1.1.0"
-    flush-write-stream "^1.0.0"
-    from2 "^2.1.0"
-    parallel-transform "^1.1.0"
-    pump "^3.0.0"
-    pumpify "^1.3.3"
-    stream-each "^1.1.0"
-    through2 "^2.0.0"
+mitt@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/mitt/-/mitt-2.1.0.tgz#f740577c23176c6205b121b2973514eade1b2230"
+  integrity sha512-ILj2TpLiysu2wkBbWjAmww7TkZb65aiQO+DkVdUTBpBXq+MHYiETENkKFMtsJZX1Lf4pe4QOrTSjIfUwN5lRdg==
 
 mixin-deep@^1.2.0:
   version "1.3.2"
@@ -13451,14 +11880,14 @@ mixin-deep@^1.2.0:
     for-in "^1.0.2"
     is-extendable "^1.0.1"
 
-mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5, mkdirp@~0.5.1:
+mkdirp@^0.5.1, mkdirp@~0.5.1:
   version "0.5.5"
   resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
   integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
   dependencies:
     minimist "^1.2.5"
 
-mkdirp@^1.0.3, mkdirp@^1.0.4:
+mkdirp@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
@@ -13480,12 +11909,7 @@ moment-timezone@^0.5.35:
   dependencies:
     moment ">= 2.9.0"
 
-"moment@>= 2.9.0", moment@^2.24.0:
-  version "2.29.1"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
-  integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
-
-moment@^2.29.2, moment@^2.29.4:
+"moment@>= 2.9.0", moment@^2.24.0, moment@^2.29.2, moment@^2.29.4:
   version "2.29.4"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108"
   integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==
@@ -13495,18 +11919,6 @@ monaco-editor@^0.28.1:
   resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.28.1.tgz#732788ff2172d59e6d436b206da8cac715413940"
   integrity sha512-P1vPqxB4B1ZFzTeR1ScggSp9/5NoQrLCq88fnlNUsuRAP1usEBN4TIpI2lw0AYIZNVIanHk0qwjze2uJwGOHUw==
 
-move-concurrently@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"
-  integrity sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=
-  dependencies:
-    aproba "^1.1.1"
-    copy-concurrently "^1.0.0"
-    fs-write-stream-atomic "^1.0.8"
-    mkdirp "^0.5.1"
-    rimraf "^2.5.4"
-    run-queue "^1.0.3"
-
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -13522,17 +11934,12 @@ ms@2.1.3, ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
-multicast-dns-service-types@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz#899f11d9686e5e05cb91b35d5f0e63b773cfc901"
-  integrity sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=
-
-multicast-dns@^6.0.1:
-  version "6.2.3"
-  resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-6.2.3.tgz#a0ec7bd9055c4282f790c3c82f4e28db3b31b229"
-  integrity sha512-ji6J5enbMyGRHIAkAOu3WdV8nggqviKCEKtXcOqfphZZtQrmHKycfynJ2V7eVPUA4NhJ6V7Wf4TmGbTwKE9B6g==
+multicast-dns@^7.2.5:
+  version "7.2.5"
+  resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-7.2.5.tgz#77eb46057f4d7adbd16d9290fa7299f6fa64cced"
+  integrity sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==
   dependencies:
-    dns-packet "^1.3.1"
+    dns-packet "^5.2.2"
     thunky "^1.0.2"
 
 multishift@^2.0.5:
@@ -13556,10 +11963,14 @@ mute-stream@0.0.8:
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
   integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
 
-nan@^2.12.1:
-  version "2.14.2"
-  resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
-  integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
+mz@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
+  integrity sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==
+  dependencies:
+    any-promise "^1.0.0"
+    object-assign "^4.0.1"
+    thenify-all "^1.0.0"
 
 nano-css@^5.3.1:
   version "5.3.5"
@@ -13580,10 +11991,10 @@ nanoevents@^5.1.13:
   resolved "https://registry.yarnpkg.com/nanoevents/-/nanoevents-5.1.13.tgz#0e49c30acbcf847c10f29f1d1e7147c0f88fa3d1"
   integrity sha512-JFAeG9fp0QZnRoESHjkbVFbZ9BkOXkkagUVwZVo/pkSX+Fq1VKlY+5og/8X9CYc6C7vje/CV+bwJ5M2X0+IY9Q==
 
-nanoid@^3.1.23:
-  version "3.1.23"
-  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.23.tgz#f744086ce7c2bc47ee0a8472574d5c78e4183a81"
-  integrity sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw==
+nanoid@^3.3.6:
+  version "3.3.6"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c"
+  integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==
 
 nanomatch@^1.2.9:
   version "1.2.13"
@@ -13602,43 +12013,35 @@ nanomatch@^1.2.9:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-native-request@^1.0.5:
-  version "1.0.8"
-  resolved "https://registry.yarnpkg.com/native-request/-/native-request-1.0.8.tgz#8f66bf606e0f7ea27c0e5995eb2f5d03e33ae6fb"
-  integrity sha512-vU2JojJVelUGp6jRcLwToPoWGxSx23z/0iX+I77J3Ht17rf2INGjrhOoQnjVo60nQd8wVsgzKkPfRXBiVdD2ag==
-
-native-url@^0.2.6:
-  version "0.2.6"
-  resolved "https://registry.yarnpkg.com/native-url/-/native-url-0.2.6.tgz#ca1258f5ace169c716ff44eccbddb674e10399ae"
-  integrity sha512-k4bDC87WtgrdD362gZz6zoiXQrl40kYlBmpfmSjwRO1VU0V5ccwJTlxuE72F6m3V0vc1xOf6n3UCP9QyerRqmA==
-  dependencies:
-    querystring "^0.2.0"
+natural-compare-lite@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz#17b09581988979fddafe0201e931ba933c96cbb4"
+  integrity sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==
 
 natural-compare@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
+needle@^3.1.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/needle/-/needle-3.2.0.tgz#07d240ebcabfd65c76c03afae7f6defe6469df44"
+  integrity sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==
+  dependencies:
+    debug "^3.2.6"
+    iconv-lite "^0.6.3"
+    sax "^1.2.4"
+
 negotiator@0.6.3:
   version "0.6.3"
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
   integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==
 
-neo-async@^2.5.0, neo-async@^2.6.1, neo-async@^2.6.2:
+neo-async@^2.6.2:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
   integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
 
-next-tick@~1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c"
-  integrity sha1-yobR/ogoFpsBICCOPchCS524NCw=
-
-nice-try@^1.0.4:
-  version "1.0.5"
-  resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
-  integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
-
 nise@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/nise/-/nise-5.1.0.tgz#713ef3ed138252daef20ec035ab62b7a28be645c"
@@ -13658,93 +12061,32 @@ no-case@^3.0.4:
     lower-case "^2.0.2"
     tslib "^2.0.3"
 
-node-fetch@2.6.1, node-fetch@^2.6.1:
+node-fetch@2.6.1:
   version "2.6.1"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
 
-node-fetch@2.6.7:
+node-fetch@2.6.7, node-fetch@^2.6.1:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-forge@^0.10.0:
-  version "0.10.0"
-  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"
-  integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==
+node-forge@^1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
+  integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==
 
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
   integrity sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=
 
-node-libs-browser@^2.2.1:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-2.2.1.tgz#b64f513d18338625f90346d27b0d235e631f6425"
-  integrity sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==
-  dependencies:
-    assert "^1.1.1"
-    browserify-zlib "^0.2.0"
-    buffer "^4.3.0"
-    console-browserify "^1.1.0"
-    constants-browserify "^1.0.0"
-    crypto-browserify "^3.11.0"
-    domain-browser "^1.1.1"
-    events "^3.0.0"
-    https-browserify "^1.0.0"
-    os-browserify "^0.3.0"
-    path-browserify "0.0.1"
-    process "^0.11.10"
-    punycode "^1.2.4"
-    querystring-es3 "^0.2.0"
-    readable-stream "^2.3.3"
-    stream-browserify "^2.0.1"
-    stream-http "^2.7.2"
-    string_decoder "^1.0.0"
-    timers-browserify "^2.0.4"
-    tty-browserify "0.0.0"
-    url "^0.11.0"
-    util "^0.11.0"
-    vm-browserify "^1.0.1"
-
-node-modules-regexp@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/node-modules-regexp/-/node-modules-regexp-1.0.0.tgz#8d9dbe28964a4ac5712e9131642107c71e90ec40"
-  integrity sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA=
-
-node-notifier@^8.0.0:
-  version "8.0.2"
-  resolved "https://registry.yarnpkg.com/node-notifier/-/node-notifier-8.0.2.tgz#f3167a38ef0d2c8a866a83e318c1ba0efeb702c5"
-  integrity sha512-oJP/9NAdd9+x2Q+rfphB2RJCHjod70RcRLjosiPMMu5gjIfwVnOUGq2nbTjTUbmy0DJ/tFIVT30+Qe3nzl4TJg==
-  dependencies:
-    growly "^1.3.0"
-    is-wsl "^2.2.0"
-    semver "^7.3.2"
-    shellwords "^0.1.1"
-    uuid "^8.3.0"
-    which "^2.0.2"
-
-node-releases@^1.1.61, node-releases@^1.1.71:
-  version "1.1.72"
-  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.72.tgz#14802ab6b1039a79a0c7d662b610a5bbd76eacbe"
-  integrity sha512-LLUo+PpH3dU6XizX3iVoubUNheF/owjXCZZ5yACDxNnPtgFuludV1ZL3ayK1kVep42Rmm0+R9/Y60NQbZ2bifw==
-
-node-releases@^2.0.6:
-  version "2.0.8"
-  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.8.tgz#0f349cdc8fcfa39a92ac0be9bc48b7706292b9ae"
-  integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==
-
-normalize-package-data@^2.3.2, normalize-package-data@^2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8"
-  integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==
-  dependencies:
-    hosted-git-info "^2.1.4"
-    resolve "^1.10.0"
-    semver "2 || 3 || 4 || 5"
-    validate-npm-package-license "^3.0.1"
+node-releases@^2.0.12:
+  version "2.0.12"
+  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.12.tgz#35627cc224a23bfb06fb3380f2b3afaaa7eb1039"
+  integrity sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==
 
 normalize-path@^2.1.1:
   version "2.1.1"
@@ -13763,34 +12105,17 @@ normalize-range@^0.1.2:
   resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942"
   integrity sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=
 
-normalize-url@1.9.1:
-  version "1.9.1"
-  resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-1.9.1.tgz#2cc0d66b31ea23036458436e3620d85954c66c3c"
-  integrity sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=
-  dependencies:
-    object-assign "^4.0.1"
-    prepend-http "^1.0.0"
-    query-string "^4.1.0"
-    sort-keys "^1.0.0"
-
-normalize-url@^3.0.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
-  integrity sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==
-
 normalize-url@^4.1.0:
   version "4.5.1"
   resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a"
   integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==
 
-npm-run-path@^2.0.0:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f"
-  integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=
-  dependencies:
-    path-key "^2.0.0"
+normalize-url@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a"
+  integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==
 
-npm-run-path@^4.0.0, npm-run-path@^4.0.1:
+npm-run-path@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea"
   integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==
@@ -13804,16 +12129,18 @@ nth-check@^1.0.2:
   dependencies:
     boolbase "~1.0.0"
 
+nth-check@^2.0.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d"
+  integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==
+  dependencies:
+    boolbase "^1.0.0"
+
 nullthrows@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/nullthrows/-/nullthrows-1.1.1.tgz#7818258843856ae971eae4208ad7d7eb19a431b1"
   integrity sha512-2vPPEi+Z7WqML2jZYddDIfy5Dqb0r2fze2zTxNNknZaFpVHU3mFB3R+DWeJWGVx0ecvttSGlJTI+WG+8Z4cDWw==
 
-num2fraction@^1.2.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/num2fraction/-/num2fraction-1.2.2.tgz#6f682b6a027a4e9ddfa4564cd2589d1d4e669ede"
-  integrity sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=
-
 number-is-nan@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
@@ -13843,25 +12170,17 @@ object-copy@^0.1.0:
     define-property "^0.2.5"
     kind-of "^3.0.3"
 
-object-inspect@^1.10.3:
-  version "1.10.3"
-  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.10.3.tgz#c2aa7d2d09f50c99375704f7a0adf24c5782d369"
-  integrity sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==
+object-hash@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-3.0.0.tgz#73f97f753e7baffc0e2cc9d6e079079744ac82e9"
+  integrity sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==
 
 object-inspect@^1.12.2, object-inspect@^1.9.0:
   version "1.12.2"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea"
   integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==
 
-object-is@^1.0.1:
-  version "1.1.5"
-  resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac"
-  integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==
-  dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.3"
-
-object-keys@^1.0.12, object-keys@^1.1.1:
+object-keys@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
   integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
@@ -13873,17 +12192,7 @@ object-visit@^1.0.0:
   dependencies:
     isobject "^3.0.0"
 
-object.assign@^4.1.0, object.assign@^4.1.2:
-  version "4.1.2"
-  resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940"
-  integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==
-  dependencies:
-    call-bind "^1.0.0"
-    define-properties "^1.1.3"
-    has-symbols "^1.0.1"
-    object-keys "^1.1.1"
-
-object.assign@^4.1.4:
+object.assign@^4.1.2, object.assign@^4.1.3, object.assign@^4.1.4:
   version "4.1.4"
   resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.4.tgz#9673c7c7c351ab8c4d0b516f4343ebf4dfb7799f"
   integrity sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==
@@ -13893,27 +12202,25 @@ object.assign@^4.1.4:
     has-symbols "^1.0.3"
     object-keys "^1.1.1"
 
-object.entries@^1.1.0, object.entries@^1.1.2, object.entries@^1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.3.tgz#c601c7f168b62374541a07ddbd3e2d5e4f7711a6"
-  integrity sha512-ym7h7OZebNS96hn5IJeyUmaWhaSM4SVtAPPfNLQEI2MYWCO2egsITb9nab2+i/Pwibx+R0mtn+ltKJXRSeTMGg==
+object.entries@^1.1.5, object.entries@^1.1.6:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.6.tgz#9737d0e5b8291edd340a3e3264bb8a3b00d5fa23"
+  integrity sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w==
   dependencies:
-    call-bind "^1.0.0"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.0-next.1"
-    has "^1.0.3"
+    call-bind "^1.0.2"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
 
-object.fromentries@^2.0.4:
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.4.tgz#26e1ba5c4571c5c6f0890cef4473066456a120b8"
-  integrity sha512-EsFBshs5RUUpQEY1D4q/m59kMfz4YJvxuNCJcv/jWwOJr34EaVnG11ZrZa0UHB3wnzV1wx8m58T4hQL8IuNXlQ==
+object.fromentries@^2.0.6:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.6.tgz#cdb04da08c539cffa912dcd368b886e0904bfa73"
+  integrity sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg==
   dependencies:
     call-bind "^1.0.2"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.0-next.2"
-    has "^1.0.3"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
 
-object.getownpropertydescriptors@^2.0.3, object.getownpropertydescriptors@^2.1.0:
+object.getownpropertydescriptors@^2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz#1bd63aeacf0d5d2d2f31b5e393b03a7c601a23f7"
   integrity sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==
@@ -13922,6 +12229,14 @@ object.getownpropertydescriptors@^2.0.3, object.getownpropertydescriptors@^2.1.0
     define-properties "^1.1.3"
     es-abstract "^1.18.0-next.2"
 
+object.hasown@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/object.hasown/-/object.hasown-1.1.2.tgz#f919e21fad4eb38a57bc6345b3afd496515c3f92"
+  integrity sha512-B5UIT3J1W+WuWIU55h0mjlwaqxiE5vYENJXIXZ4VFe05pNYrkKuK0U/6aFcb0pKywYJh7IhfoqUfKVmrJJHZHw==
+  dependencies:
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
+
 object.omit@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-3.0.0.tgz#0e3edc2fce2ba54df5577ff529f6d97bd8a522af"
@@ -13936,15 +12251,14 @@ object.pick@^1.3.0:
   dependencies:
     isobject "^3.0.1"
 
-object.values@^1.1.0, object.values@^1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.3.tgz#eaa8b1e17589f02f698db093f7c62ee1699742ee"
-  integrity sha512-nkF6PfDB9alkOUxpf1HNm/QlkeW3SReqL5WXeBLpEJJnlPSvRaDQpW3gQTksTN3fgJX4hL42RzKyOin6ff3tyw==
+object.values@^1.1.0, object.values@^1.1.6:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.6.tgz#4abbaa71eba47d63589d402856f908243eea9b1d"
+  integrity sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==
   dependencies:
     call-bind "^1.0.2"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.0-next.2"
-    has "^1.0.3"
+    define-properties "^1.1.4"
+    es-abstract "^1.20.4"
 
 obuf@^1.0.0, obuf@^1.1.2:
   version "1.1.2"
@@ -13984,7 +12298,7 @@ onetime@^5.1.0, onetime@^5.1.2:
   dependencies:
     mimic-fn "^2.1.0"
 
-open@^7.0.2, open@^7.3.1:
+open@^7.3.1:
   version "7.4.2"
   resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321"
   integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==
@@ -13992,12 +12306,14 @@ open@^7.0.2, open@^7.3.1:
     is-docker "^2.0.0"
     is-wsl "^2.1.1"
 
-opn@^5.5.0:
-  version "5.5.0"
-  resolved "https://registry.yarnpkg.com/opn/-/opn-5.5.0.tgz#fc7164fab56d235904c51c3b27da6758ca3b9bfc"
-  integrity sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==
+open@^8.0.9, open@^8.4.0:
+  version "8.4.2"
+  resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9"
+  integrity sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==
   dependencies:
-    is-wsl "^1.1.0"
+    define-lazy-prop "^2.0.0"
+    is-docker "^2.1.1"
+    is-wsl "^2.2.0"
 
 optimism@^0.16.0:
   version "0.16.1"
@@ -14007,14 +12323,6 @@ optimism@^0.16.0:
     "@wry/context" "^0.6.0"
     "@wry/trie" "^0.3.0"
 
-optimize-css-assets-webpack-plugin@5.0.4:
-  version "5.0.4"
-  resolved "https://registry.yarnpkg.com/optimize-css-assets-webpack-plugin/-/optimize-css-assets-webpack-plugin-5.0.4.tgz#85883c6528aaa02e30bbad9908c92926bb52dc90"
-  integrity sha512-wqd6FdI2a5/FdoiCNNkEvLeA//lHHfG24Ln2Xm2qqdIk4aOlsR18jwpyOihqQ8849W3qu2DX8fOYxpvTMj+93A==
-  dependencies:
-    cssnano "^4.1.10"
-    last-call-webpack-plugin "^3.0.0"
-
 optionator@^0.8.1:
   version "0.8.3"
   resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495"
@@ -14044,18 +12352,6 @@ orderedmap@^2.0.0:
   resolved "https://registry.yarnpkg.com/orderedmap/-/orderedmap-2.1.0.tgz#819457082fa3a06abd316d83a281a1ca467437cd"
   integrity sha512-/pIFexOm6S70EPdznemIz3BQZoJ4VTFrhqzu0ACBqBgeLsLxq8e6Jim63ImIfwW/zAD1AlXpRMlOv3aghmo4dA==
 
-original@^1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/original/-/original-1.0.2.tgz#e442a61cffe1c5fd20a65f3261c26663b303f25f"
-  integrity sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg==
-  dependencies:
-    url-parse "^1.4.3"
-
-os-browserify@^0.3.0:
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27"
-  integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=
-
 os-tmpdir@~1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
@@ -14066,16 +12362,6 @@ p-cancelable@^1.0.0:
   resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc"
   integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==
 
-p-each-series@^2.1.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a"
-  integrity sha512-ycIL2+1V32th+8scbpTvyHNaHe02z0sjgh91XXjAk+ZeXoPN4Z46DVUnzdso0aX4KckKw0FNNFHdjZ2UsZvxiA==
-
-p-finally@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
-  integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=
-
 p-limit@3.1.0, p-limit@^3.0.2:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"
@@ -14083,13 +12369,6 @@ p-limit@3.1.0, p-limit@^3.0.2:
   dependencies:
     yocto-queue "^0.1.0"
 
-p-limit@^1.1.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8"
-  integrity sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==
-  dependencies:
-    p-try "^1.0.0"
-
 p-limit@^2.0.0, p-limit@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
@@ -14097,13 +12376,6 @@ p-limit@^2.0.0, p-limit@^2.2.0:
   dependencies:
     p-try "^2.0.0"
 
-p-locate@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43"
-  integrity sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=
-  dependencies:
-    p-limit "^1.1.0"
-
 p-locate@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4"
@@ -14118,29 +12390,25 @@ p-locate@^4.1.0:
   dependencies:
     p-limit "^2.2.0"
 
+p-locate@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834"
+  integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==
+  dependencies:
+    p-limit "^3.0.2"
+
 p-map@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
   integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
 
-p-map@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b"
-  integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==
-  dependencies:
-    aggregate-error "^3.0.0"
-
-p-retry@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-3.0.1.tgz#316b4c8893e2c8dc1cfa891f406c4b422bebf328"
-  integrity sha512-XE6G4+YTTkT2a0UWb2kjZe8xNwf8bIbnqpc/IS/idOBVhyves0mK5OJgeocjx7q5pvX/6m23xuzVPYT1uGM73w==
+p-retry@^4.5.0:
+  version "4.6.2"
+  resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16"
+  integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==
   dependencies:
-    retry "^0.12.0"
-
-p-try@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"
-  integrity sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=
+    "@types/retry" "0.12.0"
+    retry "^0.13.1"
 
 p-try@^2.0.0:
   version "2.2.0"
@@ -14157,21 +12425,7 @@ package-json@^6.3.0:
     registry-url "^5.0.0"
     semver "^6.2.0"
 
-pako@~1.0.5:
-  version "1.0.11"
-  resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
-  integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
-
-parallel-transform@^1.1.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.2.0.tgz#9049ca37d6cb2182c3b1d2c720be94d14a5814fc"
-  integrity sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==
-  dependencies:
-    cyclist "^1.0.1"
-    inherits "^2.0.3"
-    readable-stream "^2.1.5"
-
-param-case@^3.0.3, param-case@^3.0.4:
+param-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/param-case/-/param-case-3.0.4.tgz#7d17fe4aa12bde34d4a77d91acfb6219caad01c5"
   integrity sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==
@@ -14191,17 +12445,6 @@ parenthesis@^3.1.8:
   resolved "https://registry.yarnpkg.com/parenthesis/-/parenthesis-3.1.8.tgz#3457fccb8f05db27572b841dad9d2630b912f125"
   integrity sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==
 
-parse-asn1@^5.0.0, parse-asn1@^5.1.5:
-  version "5.1.6"
-  resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.6.tgz#385080a3ec13cb62a62d39409cb3e88844cdaed4"
-  integrity sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==
-  dependencies:
-    asn1.js "^5.2.0"
-    browserify-aes "^1.0.0"
-    evp_bytestokey "^1.0.0"
-    pbkdf2 "^3.0.3"
-    safe-buffer "^5.1.1"
-
 parse-entities@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-2.0.0.tgz#53c6eb5b9314a1f4ec99fa0fdf7ce01ecda0cbe8"
@@ -14228,15 +12471,7 @@ parse-filepath@^1.0.2:
     map-cache "^0.2.0"
     path-root "^0.1.1"
 
-parse-json@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-4.0.0.tgz#be35f5425be1f7f6c747184f98a788cb99477ee0"
-  integrity sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=
-  dependencies:
-    error-ex "^1.3.1"
-    json-parse-better-errors "^1.0.1"
-
-parse-json@^5.0.0:
+parse-json@^5.0.0, parse-json@^5.2.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd"
   integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==
@@ -14246,6 +12481,11 @@ parse-json@^5.0.0:
     json-parse-even-better-errors "^2.3.0"
     lines-and-columns "^1.1.6"
 
+parse-node-version@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/parse-node-version/-/parse-node-version-1.0.1.tgz#e2b5dbede00e7fa9bc363607f53327e8b073189b"
+  integrity sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==
+
 parse5@6.0.1, parse5@^6.0.0:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b"
@@ -14269,11 +12509,6 @@ pascalcase@^0.1.1:
   resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14"
   integrity sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=
 
-path-browserify@0.0.1:
-  version "0.0.1"
-  resolved "https://registry.yarnpkg.com/path-browserify/-/path-browserify-0.0.1.tgz#e6c4ddd7ed3aa27c68a20cc4e50e1a4ee83bbc4a"
-  integrity sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==
-
 path-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/path-case/-/path-case-3.0.4.tgz#9168645334eb942658375c56f80b4c0cb5f82c6f"
@@ -14282,11 +12517,6 @@ path-case@^3.0.4:
     dot-case "^3.0.4"
     tslib "^2.0.3"
 
-path-dirname@^1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/path-dirname/-/path-dirname-1.0.2.tgz#cc33d24d525e099a5388c0336c6e32b9160609e0"
-  integrity sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=
-
 path-exists@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
@@ -14302,22 +12532,12 @@ path-is-absolute@^1.0.0:
   resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
   integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18=
 
-path-is-inside@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/path-is-inside/-/path-is-inside-1.0.2.tgz#365417dede44430d1c11af61027facf074bdfc53"
-  integrity sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=
-
-path-key@^2.0.0, path-key@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
-  integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=
-
 path-key@^3.0.0, path-key@^3.1.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
   integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
 
-path-parse@^1.0.6, path-parse@^1.0.7:
+path-parse@^1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
   integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
@@ -14346,13 +12566,6 @@ path-to-regexp@^1.7.0:
   dependencies:
     isarray "0.0.1"
 
-path-type@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f"
-  integrity sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==
-  dependencies:
-    pify "^3.0.0"
-
 path-type@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
@@ -14365,79 +12578,40 @@ pause-stream@0.0.11:
   dependencies:
     through "~2.3"
 
-pbkdf2@^3.0.3:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.1.2.tgz#dd822aa0887580e52f1a039dc3eda108efae3075"
-  integrity sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==
-  dependencies:
-    create-hash "^1.1.2"
-    create-hmac "^1.1.4"
-    ripemd160 "^2.0.1"
-    safe-buffer "^5.0.1"
-    sha.js "^2.4.8"
-
 performance-now@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
+picocolors@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f"
+  integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==
+
 picocolors@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
   integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
 
-picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
-  integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
+picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3, picomatch@^2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
+  integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
 
-pify@^2.0.0:
+pify@^2.3.0:
   version "2.3.0"
-  resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
-  integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw=
-
-pify@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"
-  integrity sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=
+  resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
+  integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==
 
 pify@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231"
   integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==
 
-pinkie-promise@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa"
-  integrity sha1-ITXW36ejWMBprJsXh3YogihFD/o=
-  dependencies:
-    pinkie "^2.0.0"
-
-pinkie@^2.0.0:
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
-  integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA=
-
-pirates@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87"
-  integrity sha512-WuNqLTbMI3tmfef2TKxlQmAiLHKtFhlsCZnPIpuv2Ow0RDVO8lfy1Opf4NUzlMXLjPl+Men7AuVdX6TA+s+uGA==
-  dependencies:
-    node-modules-regexp "^1.0.0"
-
-pkg-dir@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b"
-  integrity sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=
-  dependencies:
-    find-up "^2.1.0"
-
-pkg-dir@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3"
-  integrity sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==
-  dependencies:
-    find-up "^3.0.0"
+pirates@^4.0.1, pirates@^4.0.4:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9"
+  integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==
 
 pkg-dir@^4.1.0, pkg-dir@^4.2.0:
   version "4.2.0"
@@ -14446,707 +12620,569 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0:
   dependencies:
     find-up "^4.0.0"
 
-pkg-up@3.1.0:
+pkg-up@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5"
   integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==
   dependencies:
     find-up "^3.0.0"
 
-pkg-up@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-2.0.0.tgz#c819ac728059a461cab1c3889a2be3c49a004d7f"
-  integrity sha1-yBmscoBZpGHKscOImivjxJoATX8=
-  dependencies:
-    find-up "^2.1.0"
-
-pnp-webpack-plugin@1.6.4:
-  version "1.6.4"
-  resolved "https://registry.yarnpkg.com/pnp-webpack-plugin/-/pnp-webpack-plugin-1.6.4.tgz#c9711ac4dc48a685dabafc86f8b6dd9f8df84149"
-  integrity sha512-7Wjy+9E3WwLOEL30D+m8TSTF7qJJUJLONBnwQp0518siuMxUQUbgZwssaFX+QKlZkjHZcw/IpZCt/H0srrntSg==
-  dependencies:
-    ts-pnp "^1.1.6"
-
-portfinder@^1.0.26:
-  version "1.0.28"
-  resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778"
-  integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==
-  dependencies:
-    async "^2.6.2"
-    debug "^3.1.1"
-    mkdirp "^0.5.5"
-
 posix-character-classes@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
   integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=
 
-postcss-attribute-case-insensitive@^4.0.1:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-4.0.2.tgz#d93e46b504589e94ac7277b0463226c68041a880"
-  integrity sha512-clkFxk/9pcdb4Vkn0hAHq3YnxBQ2p0CGD1dy24jN+reBck+EWxMbxSUqN4Yj7t0w8csl87K6p0gxBe1utkJsYA==
+postcss-attribute-case-insensitive@^5.0.2:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-5.0.2.tgz#03d761b24afc04c09e757e92ff53716ae8ea2741"
+  integrity sha512-XIidXV8fDr0kKt28vqki84fRK8VW8eTuIa4PChv2MqKuT6C9UjmSKzen6KaWhWEoYvwxFCa7n/tC1SZ3tyq4SQ==
   dependencies:
-    postcss "^7.0.2"
-    postcss-selector-parser "^6.0.2"
+    postcss-selector-parser "^6.0.10"
 
-postcss-browser-comments@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-browser-comments/-/postcss-browser-comments-3.0.0.tgz#1248d2d935fb72053c8e1f61a84a57292d9f65e9"
-  integrity sha512-qfVjLfq7HFd2e0HW4s1dvU8X080OZdG46fFbIBFjW7US7YPDcWfRvdElvwMJr2LI6hMmD+7LnH2HcmXTs+uOig==
-  dependencies:
-    postcss "^7"
+postcss-browser-comments@^4:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/postcss-browser-comments/-/postcss-browser-comments-4.0.0.tgz#bcfc86134df5807f5d3c0eefa191d42136b5e72a"
+  integrity sha512-X9X9/WN3KIvY9+hNERUqX9gncsgBA25XaeR+jshHz2j8+sYyHktHw1JdKuMjeLpGktXidqDhA7b/qm1mrBDmgg==
 
-postcss-calc@^7.0.1:
-  version "7.0.5"
-  resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-7.0.5.tgz#f8a6e99f12e619c2ebc23cf6c486fdc15860933e"
-  integrity sha512-1tKHutbGtLtEZF6PT4JSihCHfIVldU72mZ8SdZHIYriIZ9fh9k9aWSppaT8rHsyI3dX+KSR+W+Ix9BMY3AODrg==
+postcss-calc@^8.2.3:
+  version "8.2.4"
+  resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-8.2.4.tgz#77b9c29bfcbe8a07ff6693dc87050828889739a5"
+  integrity sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q==
   dependencies:
-    postcss "^7.0.27"
-    postcss-selector-parser "^6.0.2"
-    postcss-value-parser "^4.0.2"
+    postcss-selector-parser "^6.0.9"
+    postcss-value-parser "^4.2.0"
 
-postcss-color-functional-notation@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-color-functional-notation/-/postcss-color-functional-notation-2.0.1.tgz#5efd37a88fbabeb00a2966d1e53d98ced93f74e0"
-  integrity sha512-ZBARCypjEDofW4P6IdPVTLhDNXPRn8T2s1zHbZidW6rPaaZvcnCS2soYFIQJrMZSxiePJ2XIYTlcb2ztr/eT2g==
+postcss-clamp@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-clamp/-/postcss-clamp-4.1.0.tgz#7263e95abadd8c2ba1bd911b0b5a5c9c93e02363"
+  integrity sha512-ry4b1Llo/9zz+PKC+030KUnPITTJAHeOwjfAyyB60eT0AorGLdzp52s31OsPRHRf8NchkgFoG2y6fCfn1IV1Ow==
   dependencies:
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-color-gray@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-color-gray/-/postcss-color-gray-5.0.0.tgz#532a31eb909f8da898ceffe296fdc1f864be8547"
-  integrity sha512-q6BuRnAGKM/ZRpfDascZlIZPjvwsRye7UDNalqVz3s7GDxMtqPY6+Q871liNxsonUw8oC61OG+PSaysYpl1bnw==
+postcss-color-functional-notation@^4.2.4:
+  version "4.2.4"
+  resolved "https://registry.yarnpkg.com/postcss-color-functional-notation/-/postcss-color-functional-notation-4.2.4.tgz#21a909e8d7454d3612d1659e471ce4696f28caec"
+  integrity sha512-2yrTAUZUab9s6CpxkxC4rVgFEVaR6/2Pipvi6qcgvnYiVqZcbDHEoBDhrXzyb7Efh2CCfHQNtcqWcIruDTIUeg==
   dependencies:
-    "@csstools/convert-colors" "^1.4.0"
-    postcss "^7.0.5"
-    postcss-values-parser "^2.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-color-hex-alpha@^5.0.3:
-  version "5.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-color-hex-alpha/-/postcss-color-hex-alpha-5.0.3.tgz#a8d9ca4c39d497c9661e374b9c51899ef0f87388"
-  integrity sha512-PF4GDel8q3kkreVXKLAGNpHKilXsZ6xuu+mOQMHWHLPNyjiUBOr75sp5ZKJfmv1MCus5/DWUGcK9hm6qHEnXYw==
+postcss-color-hex-alpha@^8.0.4:
+  version "8.0.4"
+  resolved "https://registry.yarnpkg.com/postcss-color-hex-alpha/-/postcss-color-hex-alpha-8.0.4.tgz#c66e2980f2fbc1a63f5b079663340ce8b55f25a5"
+  integrity sha512-nLo2DCRC9eE4w2JmuKgVA3fGL3d01kGq752pVALF68qpGLmx2Qrk91QTKkdUqqp45T1K1XV8IhQpcu1hoAQflQ==
   dependencies:
-    postcss "^7.0.14"
-    postcss-values-parser "^2.0.1"
+    postcss-value-parser "^4.2.0"
 
-postcss-color-mod-function@^3.0.3:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-color-mod-function/-/postcss-color-mod-function-3.0.3.tgz#816ba145ac11cc3cb6baa905a75a49f903e4d31d"
-  integrity sha512-YP4VG+xufxaVtzV6ZmhEtc+/aTXH3d0JLpnYfxqTvwZPbJhWqp8bSY3nfNzNRFLgB4XSaBA82OE4VjOOKpCdVQ==
+postcss-color-rebeccapurple@^7.1.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-color-rebeccapurple/-/postcss-color-rebeccapurple-7.1.1.tgz#63fdab91d878ebc4dd4b7c02619a0c3d6a56ced0"
+  integrity sha512-pGxkuVEInwLHgkNxUc4sdg4g3py7zUeCQ9sMfwyHAT+Ezk8a4OaaVZ8lIY5+oNqA/BXXgLyXv0+5wHP68R79hg==
   dependencies:
-    "@csstools/convert-colors" "^1.4.0"
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-color-rebeccapurple@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-color-rebeccapurple/-/postcss-color-rebeccapurple-4.0.1.tgz#c7a89be872bb74e45b1e3022bfe5748823e6de77"
-  integrity sha512-aAe3OhkS6qJXBbqzvZth2Au4V3KieR5sRQ4ptb2b2O8wgvB3SJBsdG+jsn2BZbbwekDG8nTfcCNKcSfe/lEy8g==
+postcss-colormin@^5.3.1:
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/postcss-colormin/-/postcss-colormin-5.3.1.tgz#86c27c26ed6ba00d96c79e08f3ffb418d1d1988f"
+  integrity sha512-UsWQG0AqTFQmpBegeLLc1+c3jIqBNB0zlDGRWR+dQ3pRKJL1oeMzyqmH3o2PIfn9MBdNrVPWhDbT769LxCTLJQ==
   dependencies:
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
+    browserslist "^4.21.4"
+    caniuse-api "^3.0.0"
+    colord "^2.9.1"
+    postcss-value-parser "^4.2.0"
 
-postcss-colormin@^4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-colormin/-/postcss-colormin-4.0.3.tgz#ae060bce93ed794ac71264f08132d550956bd381"
-  integrity sha512-WyQFAdDZpExQh32j0U0feWisZ0dmOtPl44qYmJKkq9xFWY3p+4qnRzCHeNrkeRhwPHz9bQ3mo0/yVkaply0MNw==
+postcss-convert-values@^5.1.3:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/postcss-convert-values/-/postcss-convert-values-5.1.3.tgz#04998bb9ba6b65aa31035d669a6af342c5f9d393"
+  integrity sha512-82pC1xkJZtcJEfiLw6UXnXVXScgtBrjlO5CBmuDQc+dlb88ZYheFsjTn40+zBVi3DkfF7iezO0nJUPLcJK3pvA==
   dependencies:
-    browserslist "^4.0.0"
-    color "^3.0.0"
-    has "^1.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    browserslist "^4.21.4"
+    postcss-value-parser "^4.2.0"
 
-postcss-convert-values@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-convert-values/-/postcss-convert-values-4.0.1.tgz#ca3813ed4da0f812f9d43703584e449ebe189a7f"
-  integrity sha512-Kisdo1y77KUC0Jmn0OXU/COOJbzM8cImvw1ZFsBgBgMgb1iL23Zs/LXRe3r+EZqM3vGYKdQ2YJVQ5VkJI+zEJQ==
+postcss-custom-media@^8.0.2:
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/postcss-custom-media/-/postcss-custom-media-8.0.2.tgz#c8f9637edf45fef761b014c024cee013f80529ea"
+  integrity sha512-7yi25vDAoHAkbhAzX9dHx2yc6ntS4jQvejrNcC+csQJAXjj15e7VcWfMgLqBNAbOvqi5uIa9huOVwdHbf+sKqg==
   dependencies:
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-custom-media@^7.0.8:
-  version "7.0.8"
-  resolved "https://registry.yarnpkg.com/postcss-custom-media/-/postcss-custom-media-7.0.8.tgz#fffd13ffeffad73621be5f387076a28b00294e0c"
-  integrity sha512-c9s5iX0Ge15o00HKbuRuTqNndsJUbaXdiNsksnVH8H4gdc+zbLzr/UasOwNG6CTDpLFekVY4672eWdiiWu2GUg==
+postcss-custom-properties@^12.1.10:
+  version "12.1.11"
+  resolved "https://registry.yarnpkg.com/postcss-custom-properties/-/postcss-custom-properties-12.1.11.tgz#d14bb9b3989ac4d40aaa0e110b43be67ac7845cf"
+  integrity sha512-0IDJYhgU8xDv1KY6+VgUwuQkVtmYzRwu+dMjnmdMafXYv86SWqfxkc7qdDvWS38vsjaEtv8e0vGOUQrAiMBLpQ==
   dependencies:
-    postcss "^7.0.14"
+    postcss-value-parser "^4.2.0"
 
-postcss-custom-properties@^8.0.11:
-  version "8.0.11"
-  resolved "https://registry.yarnpkg.com/postcss-custom-properties/-/postcss-custom-properties-8.0.11.tgz#2d61772d6e92f22f5e0d52602df8fae46fa30d97"
-  integrity sha512-nm+o0eLdYqdnJ5abAJeXp4CEU1c1k+eB2yMCvhgzsds/e0umabFrN6HoTy/8Q4K5ilxERdl/JD1LO5ANoYBeMA==
+postcss-custom-selectors@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/postcss-custom-selectors/-/postcss-custom-selectors-6.0.3.tgz#1ab4684d65f30fed175520f82d223db0337239d9"
+  integrity sha512-fgVkmyiWDwmD3JbpCmB45SvvlCD6z9CG6Ie6Iere22W5aHea6oWa7EM2bpnv2Fj3I94L3VbtvX9KqwSi5aFzSg==
   dependencies:
-    postcss "^7.0.17"
-    postcss-values-parser "^2.0.1"
+    postcss-selector-parser "^6.0.4"
 
-postcss-custom-selectors@^5.1.2:
-  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/postcss-custom-selectors/-/postcss-custom-selectors-5.1.2.tgz#64858c6eb2ecff2fb41d0b28c9dd7b3db4de7fba"
-  integrity sha512-DSGDhqinCqXqlS4R7KGxL1OSycd1lydugJ1ky4iRXPHdBRiozyMHrdu0H3o7qNOCiZwySZTUI5MV0T8QhCLu+w==
+postcss-dir-pseudo-class@^6.0.5:
+  version "6.0.5"
+  resolved "https://registry.yarnpkg.com/postcss-dir-pseudo-class/-/postcss-dir-pseudo-class-6.0.5.tgz#2bf31de5de76added44e0a25ecf60ae9f7c7c26c"
+  integrity sha512-eqn4m70P031PF7ZQIvSgy9RSJ5uI2171O/OO/zcRNYpJbvaeKFUlar1aJ7rmgiQtbm0FSPsRewjpdS0Oew7MPA==
   dependencies:
-    postcss "^7.0.2"
-    postcss-selector-parser "^5.0.0-rc.3"
+    postcss-selector-parser "^6.0.10"
 
-postcss-dir-pseudo-class@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-dir-pseudo-class/-/postcss-dir-pseudo-class-5.0.0.tgz#6e3a4177d0edb3abcc85fdb6fbb1c26dabaeaba2"
-  integrity sha512-3pm4oq8HYWMZePJY+5ANriPs3P07q+LW6FAdTlkFH2XqDdP4HeeJYMOzn0HYLhRSjBO3fhiqSwwU9xEULSrPgw==
-  dependencies:
-    postcss "^7.0.2"
-    postcss-selector-parser "^5.0.0-rc.3"
+postcss-discard-comments@^5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/postcss-discard-comments/-/postcss-discard-comments-5.1.2.tgz#8df5e81d2925af2780075840c1526f0660e53696"
+  integrity sha512-+L8208OVbHVF2UQf1iDmRcbdjJkuBF6IS29yBDSiWUIzpYaAhtNl6JYnYm12FnkeCwQqF5LeklOu6rAqgfBZqQ==
 
-postcss-discard-comments@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-discard-comments/-/postcss-discard-comments-4.0.2.tgz#1fbabd2c246bff6aaad7997b2b0918f4d7af4033"
-  integrity sha512-RJutN259iuRf3IW7GZyLM5Sw4GLTOH8FmsXBnv8Ab/Tc2k4SR4qbV4DNbyyY4+Sjo362SyDmW2DQ7lBSChrpkg==
-  dependencies:
-    postcss "^7.0.0"
+postcss-discard-duplicates@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-discard-duplicates/-/postcss-discard-duplicates-5.1.0.tgz#9eb4fe8456706a4eebd6d3b7b777d07bad03e848"
+  integrity sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw==
 
-postcss-discard-duplicates@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-discard-duplicates/-/postcss-discard-duplicates-4.0.2.tgz#3fe133cd3c82282e550fc9b239176a9207b784eb"
-  integrity sha512-ZNQfR1gPNAiXZhgENFfEglF93pciw0WxMkJeVmw8eF+JZBbMD7jp6C67GqJAXVZP2BWbOztKfbsdmMp/k8c6oQ==
-  dependencies:
-    postcss "^7.0.0"
+postcss-discard-empty@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-discard-empty/-/postcss-discard-empty-5.1.1.tgz#e57762343ff7f503fe53fca553d18d7f0c369c6c"
+  integrity sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A==
 
-postcss-discard-empty@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-discard-empty/-/postcss-discard-empty-4.0.1.tgz#c8c951e9f73ed9428019458444a02ad90bb9f765"
-  integrity sha512-B9miTzbznhDjTfjvipfHoqbWKwd0Mj+/fL5s1QOz06wufguil+Xheo4XpOnc4NqKYBCNqqEzgPv2aPBIJLox0w==
-  dependencies:
-    postcss "^7.0.0"
+postcss-discard-overridden@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz#7e8c5b53325747e9d90131bb88635282fb4a276e"
+  integrity sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw==
 
-postcss-discard-overridden@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-discard-overridden/-/postcss-discard-overridden-4.0.1.tgz#652aef8a96726f029f5e3e00146ee7a4e755ff57"
-  integrity sha512-IYY2bEDD7g1XM1IDEsUT4//iEYCxAmP5oDSFMVU/JVvT7gh+l4fmjciLqGgwjdWpQIdb0Che2VX00QObS5+cTg==
+postcss-double-position-gradients@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/postcss-double-position-gradients/-/postcss-double-position-gradients-3.1.2.tgz#b96318fdb477be95997e86edd29c6e3557a49b91"
+  integrity sha512-GX+FuE/uBR6eskOK+4vkXgT6pDkexLokPaz/AbJna9s5Kzp/yl488pKPjhy0obB475ovfT1Wv8ho7U/cHNaRgQ==
   dependencies:
-    postcss "^7.0.0"
+    "@csstools/postcss-progressive-custom-properties" "^1.1.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-double-position-gradients@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-double-position-gradients/-/postcss-double-position-gradients-1.0.0.tgz#fc927d52fddc896cb3a2812ebc5df147e110522e"
-  integrity sha512-G+nV8EnQq25fOI8CH/B6krEohGWnF5+3A6H/+JEpOncu5dCnkS1QQ6+ct3Jkaepw1NGVqqOZH6lqrm244mCftA==
+postcss-env-function@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/postcss-env-function/-/postcss-env-function-4.0.6.tgz#7b2d24c812f540ed6eda4c81f6090416722a8e7a"
+  integrity sha512-kpA6FsLra+NqcFnL81TnsU+Z7orGtDTxcOhl6pwXeEq1yFPpRMkCDpHhrz8CFQDr/Wfm0jLiNQ1OsGGPjlqPwA==
   dependencies:
-    postcss "^7.0.5"
-    postcss-values-parser "^2.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-env-function@^2.0.2:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-env-function/-/postcss-env-function-2.0.2.tgz#0f3e3d3c57f094a92c2baf4b6241f0b0da5365d7"
-  integrity sha512-rwac4BuZlITeUbiBq60h/xbLzXY43qOsIErngWa4l7Mt+RaSkT7QBjXVGTcBHupykkblHMDrBFh30zchYPaOUw==
-  dependencies:
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
+postcss-flexbugs-fixes@^5.0.2:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/postcss-flexbugs-fixes/-/postcss-flexbugs-fixes-5.0.2.tgz#2028e145313074fc9abe276cb7ca14e5401eb49d"
+  integrity sha512-18f9voByak7bTktR2QgDveglpn9DTbBWPUzSOe9g0N4WR/2eSt6Vrcbf0hmspvMI6YWGywz6B9f7jzpFNJJgnQ==
 
-postcss-flexbugs-fixes@4.2.1:
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/postcss-flexbugs-fixes/-/postcss-flexbugs-fixes-4.2.1.tgz#9218a65249f30897deab1033aced8578562a6690"
-  integrity sha512-9SiofaZ9CWpQWxOwRh1b/r85KD5y7GgvsNt1056k6OYLvWUun0czCvogfJgylC22uJTwW1KzY3Gz65NZRlvoiQ==
+postcss-focus-visible@^6.0.4:
+  version "6.0.4"
+  resolved "https://registry.yarnpkg.com/postcss-focus-visible/-/postcss-focus-visible-6.0.4.tgz#50c9ea9afa0ee657fb75635fabad25e18d76bf9e"
+  integrity sha512-QcKuUU/dgNsstIK6HELFRT5Y3lbrMLEOwG+A4s5cA+fx3A3y/JTq3X9LaOj3OC3ALH0XqyrgQIgey/MIZ8Wczw==
   dependencies:
-    postcss "^7.0.26"
+    postcss-selector-parser "^6.0.9"
 
-postcss-focus-visible@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-focus-visible/-/postcss-focus-visible-4.0.0.tgz#477d107113ade6024b14128317ade2bd1e17046e"
-  integrity sha512-Z5CkWBw0+idJHSV6+Bgf2peDOFf/x4o+vX/pwcNYrWpXFrSfTkQ3JQ1ojrq9yS+upnAlNRHeg8uEwFTgorjI8g==
+postcss-focus-within@^5.0.4:
+  version "5.0.4"
+  resolved "https://registry.yarnpkg.com/postcss-focus-within/-/postcss-focus-within-5.0.4.tgz#5b1d2ec603195f3344b716c0b75f61e44e8d2e20"
+  integrity sha512-vvjDN++C0mu8jz4af5d52CB184ogg/sSxAFS+oUJQq2SuCe7T5U2iIsVJtsCp2d6R4j0jr5+q3rPkBVZkXD9fQ==
   dependencies:
-    postcss "^7.0.2"
+    postcss-selector-parser "^6.0.9"
 
-postcss-focus-within@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-focus-within/-/postcss-focus-within-3.0.0.tgz#763b8788596cee9b874c999201cdde80659ef680"
-  integrity sha512-W0APui8jQeBKbCGZudW37EeMCjDeVxKgiYfIIEo8Bdh5SpB9sxds/Iq8SEuzS0Q4YFOlG7EPFulbbxujpkrV2w==
-  dependencies:
-    postcss "^7.0.2"
+postcss-font-variant@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/postcss-font-variant/-/postcss-font-variant-5.0.0.tgz#efd59b4b7ea8bb06127f2d031bfbb7f24d32fa66"
+  integrity sha512-1fmkBaCALD72CK2a9i468mA/+tr9/1cBxRRMXOUaZqO43oWPR5imcyPjXwuv7PXbCid4ndlP5zWhidQVVa3hmA==
 
-postcss-font-variant@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-font-variant/-/postcss-font-variant-4.0.1.tgz#42d4c0ab30894f60f98b17561eb5c0321f502641"
-  integrity sha512-I3ADQSTNtLTTd8uxZhtSOrTCQ9G4qUVKPjHiDk0bV75QSxXjVWiJVJ2VLdspGUi9fbW9BcjKJoRvxAH1pckqmA==
-  dependencies:
-    postcss "^7.0.2"
+postcss-gap-properties@^3.0.5:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/postcss-gap-properties/-/postcss-gap-properties-3.0.5.tgz#f7e3cddcf73ee19e94ccf7cb77773f9560aa2fff"
+  integrity sha512-IuE6gKSdoUNcvkGIqdtjtcMtZIFyXZhmFd5RUlg97iVEvp1BZKV5ngsAjCjrVy+14uhGBQl9tzmi1Qwq4kqVOg==
 
-postcss-gap-properties@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-gap-properties/-/postcss-gap-properties-2.0.0.tgz#431c192ab3ed96a3c3d09f2ff615960f902c1715"
-  integrity sha512-QZSqDaMgXCHuHTEzMsS2KfVDOq7ZFiknSpkrPJY6jmxbugUPTuSzs/vuE5I3zv0WAS+3vhrlqhijiprnuQfzmg==
+postcss-image-set-function@^4.0.7:
+  version "4.0.7"
+  resolved "https://registry.yarnpkg.com/postcss-image-set-function/-/postcss-image-set-function-4.0.7.tgz#08353bd756f1cbfb3b6e93182c7829879114481f"
+  integrity sha512-9T2r9rsvYzm5ndsBE8WgtrMlIT7VbtTfE7b3BQnudUqnBcBo7L758oc+o+pdj/dUV0l5wjwSdjeOH2DZtfv8qw==
   dependencies:
-    postcss "^7.0.2"
+    postcss-value-parser "^4.2.0"
 
-postcss-image-set-function@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-image-set-function/-/postcss-image-set-function-3.0.1.tgz#28920a2f29945bed4c3198d7df6496d410d3f288"
-  integrity sha512-oPTcFFip5LZy8Y/whto91L9xdRHCWEMs3e1MdJxhgt4jy2WYXfhkng59fH5qLXSCPN8k4n94p1Czrfe5IOkKUw==
+postcss-import@^15.1.0:
+  version "15.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-import/-/postcss-import-15.1.0.tgz#41c64ed8cc0e23735a9698b3249ffdbf704adc70"
+  integrity sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==
   dependencies:
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
+    postcss-value-parser "^4.0.0"
+    read-cache "^1.0.0"
+    resolve "^1.1.7"
 
-postcss-initial@^3.0.0:
-  version "3.0.4"
-  resolved "https://registry.yarnpkg.com/postcss-initial/-/postcss-initial-3.0.4.tgz#9d32069a10531fe2ecafa0b6ac750ee0bc7efc53"
-  integrity sha512-3RLn6DIpMsK1l5UUy9jxQvoDeUN4gP939tDcKUHD/kM8SGSKbFAnvkpFpj3Bhtz3HGk1jWY5ZNWX6mPta5M9fg==
-  dependencies:
-    postcss "^7.0.2"
+postcss-initial@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-initial/-/postcss-initial-4.0.1.tgz#529f735f72c5724a0fb30527df6fb7ac54d7de42"
+  integrity sha512-0ueD7rPqX8Pn1xJIjay0AZeIuDoF+V+VvMt/uOnn+4ezUKhZM/NokDeP6DwMNyIoYByuN/94IQnt5FEkaN59xQ==
 
-postcss-lab-function@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-lab-function/-/postcss-lab-function-2.0.1.tgz#bb51a6856cd12289ab4ae20db1e3821ef13d7d2e"
-  integrity sha512-whLy1IeZKY+3fYdqQFuDBf8Auw+qFuVnChWjmxm/UhHWqNHZx+B99EwxTvGYmUBqe3Fjxs4L1BoZTJmPu6usVg==
+postcss-js@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-js/-/postcss-js-4.0.1.tgz#61598186f3703bab052f1c4f7d805f3991bee9d2"
+  integrity sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==
   dependencies:
-    "@csstools/convert-colors" "^1.4.0"
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
+    camelcase-css "^2.0.1"
 
-postcss-load-config@^2.0.0:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-2.1.2.tgz#c5ea504f2c4aef33c7359a34de3573772ad7502a"
-  integrity sha512-/rDeGV6vMUo3mwJZmeHfEDvwnTKKqQ0S7OHUi/kJvvtx3aWtyWG2/0ZWnzCt2keEclwN6Tf0DST2v9kITdOKYw==
+postcss-lab-function@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/postcss-lab-function/-/postcss-lab-function-4.2.1.tgz#6fe4c015102ff7cd27d1bd5385582f67ebdbdc98"
+  integrity sha512-xuXll4isR03CrQsmxyz92LJB2xX9n+pZJ5jE9JgcnmsCammLyKdlzrBin+25dy6wIjfhJpKBAN80gsTlCgRk2w==
   dependencies:
-    cosmiconfig "^5.0.0"
-    import-cwd "^2.0.0"
+    "@csstools/postcss-progressive-custom-properties" "^1.1.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-loader@3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-loader/-/postcss-loader-3.0.0.tgz#6b97943e47c72d845fa9e03f273773d4e8dd6c2d"
-  integrity sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==
+postcss-load-config@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-4.0.1.tgz#152383f481c2758274404e4962743191d73875bd"
+  integrity sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==
   dependencies:
-    loader-utils "^1.1.0"
-    postcss "^7.0.0"
-    postcss-load-config "^2.0.0"
-    schema-utils "^1.0.0"
+    lilconfig "^2.0.5"
+    yaml "^2.1.1"
 
-postcss-logical@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-logical/-/postcss-logical-3.0.0.tgz#2495d0f8b82e9f262725f75f9401b34e7b45d5b5"
-  integrity sha512-1SUKdJc2vuMOmeItqGuNaC+N8MzBWFWEkAnRnLpFYj1tGGa7NqyVBujfRtgNa2gXR+6RkGUiB2O5Vmh7E2RmiA==
+postcss-loader@^6.2.1:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/postcss-loader/-/postcss-loader-6.2.1.tgz#0895f7346b1702103d30fdc66e4d494a93c008ef"
+  integrity sha512-WbbYpmAaKcux/P66bZ40bpWsBucjx/TTgVVzRZ9yUO8yQfVBlameJ0ZGVaPfH64hNSBh63a+ICP5nqOpBA0w+Q==
   dependencies:
-    postcss "^7.0.2"
+    cosmiconfig "^7.0.0"
+    klona "^2.0.5"
+    semver "^7.3.5"
 
-postcss-media-minmax@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-media-minmax/-/postcss-media-minmax-4.0.0.tgz#b75bb6cbc217c8ac49433e12f22048814a4f5ed5"
-  integrity sha512-fo9moya6qyxsjbFAYl97qKO9gyre3qvbMnkOZeZwlsW6XYFsvs2DMGDlchVLfAd8LHPZDxivu/+qW2SMQeTHBw==
-  dependencies:
-    postcss "^7.0.2"
+postcss-logical@^5.0.4:
+  version "5.0.4"
+  resolved "https://registry.yarnpkg.com/postcss-logical/-/postcss-logical-5.0.4.tgz#ec75b1ee54421acc04d5921576b7d8db6b0e6f73"
+  integrity sha512-RHXxplCeLh9VjinvMrZONq7im4wjWGlRJAqmAVLXyZaXwfDWP73/oq4NdIp+OZwhQUMj0zjqDfM5Fj7qby+B4g==
 
-postcss-merge-longhand@^4.0.11:
-  version "4.0.11"
-  resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-4.0.11.tgz#62f49a13e4a0ee04e7b98f42bb16062ca2549e24"
-  integrity sha512-alx/zmoeXvJjp7L4mxEMjh8lxVlDFX1gqWHzaaQewwMZiVhLo42TEClKaeHbRf6J7j82ZOdTJ808RtN0ZOZwvw==
+postcss-media-minmax@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/postcss-media-minmax/-/postcss-media-minmax-5.0.0.tgz#7140bddec173e2d6d657edbd8554a55794e2a5b5"
+  integrity sha512-yDUvFf9QdFZTuCUg0g0uNSHVlJ5X1lSzDZjPSFaiCWvjgsvu8vEVxtahPrLMinIDEEGnx6cBe6iqdx5YWz08wQ==
+
+postcss-merge-longhand@^5.1.7:
+  version "5.1.7"
+  resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-5.1.7.tgz#24a1bdf402d9ef0e70f568f39bdc0344d568fb16"
+  integrity sha512-YCI9gZB+PLNskrK0BB3/2OzPnGhPkBEwmwhfYk1ilBHYVAZB7/tkTHFBAnCrvBBOmeYyMYw3DMjT55SyxMBzjQ==
   dependencies:
-    css-color-names "0.0.4"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
-    stylehacks "^4.0.0"
+    postcss-value-parser "^4.2.0"
+    stylehacks "^5.1.1"
 
-postcss-merge-rules@^4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-merge-rules/-/postcss-merge-rules-4.0.3.tgz#362bea4ff5a1f98e4075a713c6cb25aefef9a650"
-  integrity sha512-U7e3r1SbvYzO0Jr3UT/zKBVgYYyhAz0aitvGIYOYK5CPmkNih+WDSsS5tvPrJ8YMQYlEMvsZIiqmn7HdFUaeEQ==
+postcss-merge-rules@^5.1.4:
+  version "5.1.4"
+  resolved "https://registry.yarnpkg.com/postcss-merge-rules/-/postcss-merge-rules-5.1.4.tgz#2f26fa5cacb75b1402e213789f6766ae5e40313c"
+  integrity sha512-0R2IuYpgU93y9lhVbO/OylTtKMVcHb67zjWIfCiKR9rWL3GUk1677LAqD/BcHizukdZEjT8Ru3oHRoAYoJy44g==
   dependencies:
-    browserslist "^4.0.0"
+    browserslist "^4.21.4"
     caniuse-api "^3.0.0"
-    cssnano-util-same-parent "^4.0.0"
-    postcss "^7.0.0"
-    postcss-selector-parser "^3.0.0"
-    vendors "^1.0.0"
+    cssnano-utils "^3.1.0"
+    postcss-selector-parser "^6.0.5"
 
-postcss-minify-font-values@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-minify-font-values/-/postcss-minify-font-values-4.0.2.tgz#cd4c344cce474343fac5d82206ab2cbcb8afd5a6"
-  integrity sha512-j85oO6OnRU9zPf04+PZv1LYIYOprWm6IA6zkXkrJXyRveDEuQggG6tvoy8ir8ZwjLxLuGfNkCZEQG7zan+Hbtg==
+postcss-minify-font-values@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-minify-font-values/-/postcss-minify-font-values-5.1.0.tgz#f1df0014a726083d260d3bd85d7385fb89d1f01b"
+  integrity sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA==
   dependencies:
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-minify-gradients@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-4.0.2.tgz#93b29c2ff5099c535eecda56c4aa6e665a663471"
-  integrity sha512-qKPfwlONdcf/AndP1U8SJ/uzIJtowHlMaSioKzebAXSG4iJthlWC9iSWznQcX4f66gIWX44RSA841HTHj3wK+Q==
+postcss-minify-gradients@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-5.1.1.tgz#f1fe1b4f498134a5068240c2f25d46fcd236ba2c"
+  integrity sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw==
   dependencies:
-    cssnano-util-get-arguments "^4.0.0"
-    is-color-stop "^1.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    colord "^2.9.1"
+    cssnano-utils "^3.1.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-minify-params@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-4.0.2.tgz#6b9cef030c11e35261f95f618c90036d680db874"
-  integrity sha512-G7eWyzEx0xL4/wiBBJxJOz48zAKV2WG3iZOqVhPet/9geefm/Px5uo1fzlHu+DOjT+m0Mmiz3jkQzVHe6wxAWg==
+postcss-minify-params@^5.1.4:
+  version "5.1.4"
+  resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-5.1.4.tgz#c06a6c787128b3208b38c9364cfc40c8aa5d7352"
+  integrity sha512-+mePA3MgdmVmv6g+30rn57USjOGSAyuxUmkfiWpzalZ8aiBkdPYjXWtHuwJGm1v5Ojy0Z0LaSYhHaLJQB0P8Jw==
   dependencies:
-    alphanum-sort "^1.0.0"
-    browserslist "^4.0.0"
-    cssnano-util-get-arguments "^4.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
-    uniqs "^2.0.0"
+    browserslist "^4.21.4"
+    cssnano-utils "^3.1.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-minify-selectors@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-minify-selectors/-/postcss-minify-selectors-4.0.2.tgz#e2e5eb40bfee500d0cd9243500f5f8ea4262fbd8"
-  integrity sha512-D5S1iViljXBj9kflQo4YutWnJmwm8VvIsU1GeXJGiG9j8CIg9zs4voPMdQDUmIxetUOh60VilsNzCiAFTOqu3g==
+postcss-minify-selectors@^5.2.1:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/postcss-minify-selectors/-/postcss-minify-selectors-5.2.1.tgz#d4e7e6b46147b8117ea9325a915a801d5fe656c6"
+  integrity sha512-nPJu7OjZJTsVUmPdm2TcaiohIwxP+v8ha9NehQ2ye9szv4orirRU3SDdtUmKH+10nzn0bAyOXZ0UEr7OpvLehg==
   dependencies:
-    alphanum-sort "^1.0.0"
-    has "^1.0.0"
-    postcss "^7.0.0"
-    postcss-selector-parser "^3.0.0"
+    postcss-selector-parser "^6.0.5"
 
-postcss-modules-extract-imports@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-2.0.0.tgz#818719a1ae1da325f9832446b01136eeb493cd7e"
-  integrity sha512-LaYLDNS4SG8Q5WAWqIJgdHPJrDDr/Lv775rMBFUbgjTz6j34lUznACHcdRWroPvXANP2Vj7yNK57vp9eFqzLWQ==
-  dependencies:
-    postcss "^7.0.5"
+postcss-modules-extract-imports@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz#cda1f047c0ae80c97dbe28c3e76a43b88025741d"
+  integrity sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw==
 
-postcss-modules-local-by-default@^3.0.3:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-3.0.3.tgz#bb14e0cc78279d504dbdcbfd7e0ca28993ffbbb0"
-  integrity sha512-e3xDq+LotiGesympRlKNgaJ0PCzoUIdpH0dj47iWAui/kyTgh3CiAr1qP54uodmJhl6p9rN6BoNcdEDVJx9RDw==
+postcss-modules-local-by-default@^4.0.3:
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.3.tgz#b08eb4f083050708998ba2c6061b50c2870ca524"
+  integrity sha512-2/u2zraspoACtrbFRnTijMiQtb4GW4BvatjaG/bCjYQo8kLTdevCUlwuBHx2sCnSyrI3x3qj4ZK1j5LQBgzmwA==
   dependencies:
-    icss-utils "^4.1.1"
-    postcss "^7.0.32"
+    icss-utils "^5.0.0"
     postcss-selector-parser "^6.0.2"
     postcss-value-parser "^4.1.0"
 
-postcss-modules-scope@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-2.2.0.tgz#385cae013cc7743f5a7d7602d1073a89eaae62ee"
-  integrity sha512-YyEgsTMRpNd+HmyC7H/mh3y+MeFWevy7V1evVhJWewmMbjDHIbZbOXICC2y+m1xI1UVfIT1HMW/O04Hxyu9oXQ==
-  dependencies:
-    postcss "^7.0.6"
-    postcss-selector-parser "^6.0.0"
-
-postcss-modules-values@^3.0.0:
+postcss-modules-scope@^3.0.0:
   version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-3.0.0.tgz#5b5000d6ebae29b4255301b4a3a54574423e7f10"
-  integrity sha512-1//E5jCBrZ9DmRX+zCtmQtRSV6PV42Ix7Bzj9GbwJceduuf7IqP8MgeTXuRDHOWj2m0VzZD5+roFWDuU8RQjcg==
-  dependencies:
-    icss-utils "^4.0.0"
-    postcss "^7.0.6"
-
-postcss-nesting@^7.0.0:
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-nesting/-/postcss-nesting-7.0.1.tgz#b50ad7b7f0173e5b5e3880c3501344703e04c052"
-  integrity sha512-FrorPb0H3nuVq0Sff7W2rnc3SmIcruVC6YwpcS+k687VxyxO33iE1amna7wHuRVzM8vfiYofXSBHNAZ3QhLvYg==
+  resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz#9ef3151456d3bbfa120ca44898dfca6f2fa01f06"
+  integrity sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg==
   dependencies:
-    postcss "^7.0.2"
+    postcss-selector-parser "^6.0.4"
 
-postcss-normalize-charset@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-charset/-/postcss-normalize-charset-4.0.1.tgz#8b35add3aee83a136b0471e0d59be58a50285dd4"
-  integrity sha512-gMXCrrlWh6G27U0hF3vNvR3w8I1s2wOBILvA87iNXaPvSNo5uZAMYsZG7XjCUf1eVxuPfyL4TJ7++SGZLc9A3g==
+postcss-modules-values@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz#d7c5e7e68c3bb3c9b27cbf48ca0bb3ffb4602c9c"
+  integrity sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==
   dependencies:
-    postcss "^7.0.0"
+    icss-utils "^5.0.0"
 
-postcss-normalize-display-values@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-display-values/-/postcss-normalize-display-values-4.0.2.tgz#0dbe04a4ce9063d4667ed2be476bb830c825935a"
-  integrity sha512-3F2jcsaMW7+VtRMAqf/3m4cPFhPD3EFRgNs18u+k3lTJJlVe7d0YPO+bnwqo2xg8YiRpDXJI2u8A0wqJxMsQuQ==
+postcss-nested@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-nested/-/postcss-nested-6.0.1.tgz#f83dc9846ca16d2f4fa864f16e9d9f7d0961662c"
+  integrity sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==
   dependencies:
-    cssnano-util-get-match "^4.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-selector-parser "^6.0.11"
 
-postcss-normalize-positions@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-positions/-/postcss-normalize-positions-4.0.2.tgz#05f757f84f260437378368a91f8932d4b102917f"
-  integrity sha512-Dlf3/9AxpxE+NF1fJxYDeggi5WwV35MXGFnnoccP/9qDtFrTArZ0D0R+iKcg5WsUd8nUYMIl8yXDCtcrT8JrdA==
+postcss-nesting@^10.2.0:
+  version "10.2.0"
+  resolved "https://registry.yarnpkg.com/postcss-nesting/-/postcss-nesting-10.2.0.tgz#0b12ce0db8edfd2d8ae0aaf86427370b898890be"
+  integrity sha512-EwMkYchxiDiKUhlJGzWsD9b2zvq/r2SSubcRrgP+jujMXFzqvANLt16lJANC+5uZ6hjI7lpRmI6O8JIl+8l1KA==
   dependencies:
-    cssnano-util-get-arguments "^4.0.0"
-    has "^1.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    "@csstools/selector-specificity" "^2.0.0"
+    postcss-selector-parser "^6.0.10"
 
-postcss-normalize-repeat-style@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-4.0.2.tgz#c4ebbc289f3991a028d44751cbdd11918b17910c"
-  integrity sha512-qvigdYYMpSuoFs3Is/f5nHdRLJN/ITA7huIoCyqqENJe9PvPmLhNLMu7QTjPdtnVf6OcYYO5SHonx4+fbJE1+Q==
-  dependencies:
-    cssnano-util-get-arguments "^4.0.0"
-    cssnano-util-get-match "^4.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+postcss-normalize-charset@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-charset/-/postcss-normalize-charset-5.1.0.tgz#9302de0b29094b52c259e9b2cf8dc0879879f0ed"
+  integrity sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg==
 
-postcss-normalize-string@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-string/-/postcss-normalize-string-4.0.2.tgz#cd44c40ab07a0c7a36dc5e99aace1eca4ec2690c"
-  integrity sha512-RrERod97Dnwqq49WNz8qo66ps0swYZDSb6rM57kN2J+aoyEAJfZ6bMx0sx/F9TIEX0xthPGCmeyiam/jXif0eA==
+postcss-normalize-display-values@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-display-values/-/postcss-normalize-display-values-5.1.0.tgz#72abbae58081960e9edd7200fcf21ab8325c3da8"
+  integrity sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA==
   dependencies:
-    has "^1.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-normalize-timing-functions@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-4.0.2.tgz#8e009ca2a3949cdaf8ad23e6b6ab99cb5e7d28d9"
-  integrity sha512-acwJY95edP762e++00Ehq9L4sZCEcOPyaHwoaFOhIwWCDfik6YvqsYNxckee65JHLKzuNSSmAdxwD2Cud1Z54A==
+postcss-normalize-positions@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-positions/-/postcss-normalize-positions-5.1.1.tgz#ef97279d894087b59325b45c47f1e863daefbb92"
+  integrity sha512-6UpCb0G4eofTCQLFVuI3EVNZzBNPiIKcA1AKVka+31fTVySphr3VUgAIULBhxZkKgwLImhzMR2Bw1ORK+37INg==
   dependencies:
-    cssnano-util-get-match "^4.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-normalize-unicode@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-unicode/-/postcss-normalize-unicode-4.0.1.tgz#841bd48fdcf3019ad4baa7493a3d363b52ae1cfb"
-  integrity sha512-od18Uq2wCYn+vZ/qCOeutvHjB5jm57ToxRaMeNuf0nWVHaP9Hua56QyMF6fs/4FSUnVIw0CBPsU0K4LnBPwYwg==
+postcss-normalize-repeat-style@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-5.1.1.tgz#e9eb96805204f4766df66fd09ed2e13545420fb2"
+  integrity sha512-mFpLspGWkQtBcWIRFLmewo8aC3ImN2i/J3v8YCFUwDnPu3Xz4rLohDO26lGjwNsQxB3YF0KKRwspGzE2JEuS0g==
   dependencies:
-    browserslist "^4.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-normalize-url@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-4.0.1.tgz#10e437f86bc7c7e58f7b9652ed878daaa95faae1"
-  integrity sha512-p5oVaF4+IHwu7VpMan/SSpmpYxcJMtkGppYf0VbdH5B6hN8YNmVyJLuY9FmLQTzY3fag5ESUUHDqM+heid0UVA==
+postcss-normalize-string@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-string/-/postcss-normalize-string-5.1.0.tgz#411961169e07308c82c1f8c55f3e8a337757e228"
+  integrity sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w==
   dependencies:
-    is-absolute-url "^2.0.0"
-    normalize-url "^3.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-normalize-whitespace@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-4.0.2.tgz#bf1d4070fe4fcea87d1348e825d8cc0c5faa7d82"
-  integrity sha512-tO8QIgrsI3p95r8fyqKV+ufKlSHh9hMJqACqbv2XknufqEDhDvbguXGBBqxw9nsQoXWf0qOqppziKJKHMD4GtA==
+postcss-normalize-timing-functions@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-5.1.0.tgz#d5614410f8f0b2388e9f240aa6011ba6f52dafbb"
+  integrity sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg==
   dependencies:
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-normalize@8.0.1:
-  version "8.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-normalize/-/postcss-normalize-8.0.1.tgz#90e80a7763d7fdf2da6f2f0f82be832ce4f66776"
-  integrity sha512-rt9JMS/m9FHIRroDDBGSMsyW1c0fkvOJPy62ggxSHUldJO7B195TqFMqIf+lY5ezpDcYOV4j86aUp3/XbxzCCQ==
+postcss-normalize-unicode@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-unicode/-/postcss-normalize-unicode-5.1.1.tgz#f67297fca3fea7f17e0d2caa40769afc487aa030"
+  integrity sha512-qnCL5jzkNUmKVhZoENp1mJiGNPcsJCs1aaRmURmeJGES23Z/ajaln+EPTD+rBeNkSryI+2WTdW+lwcVdOikrpA==
   dependencies:
-    "@csstools/normalize.css" "^10.1.0"
-    browserslist "^4.6.2"
-    postcss "^7.0.17"
-    postcss-browser-comments "^3.0.0"
-    sanitize.css "^10.0.0"
+    browserslist "^4.21.4"
+    postcss-value-parser "^4.2.0"
 
-postcss-ordered-values@^4.1.2:
-  version "4.1.2"
-  resolved "https://registry.yarnpkg.com/postcss-ordered-values/-/postcss-ordered-values-4.1.2.tgz#0cf75c820ec7d5c4d280189559e0b571ebac0eee"
-  integrity sha512-2fCObh5UanxvSxeXrtLtlwVThBvHn6MQcu4ksNT2tsaV2Fg76R2CV98W7wNSlX+5/pFwEyaDwKLLoEV7uRybAw==
+postcss-normalize-url@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-5.1.0.tgz#ed9d88ca82e21abef99f743457d3729a042adcdc"
+  integrity sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew==
   dependencies:
-    cssnano-util-get-arguments "^4.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    normalize-url "^6.0.1"
+    postcss-value-parser "^4.2.0"
 
-postcss-overflow-shorthand@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-overflow-shorthand/-/postcss-overflow-shorthand-2.0.0.tgz#31ecf350e9c6f6ddc250a78f0c3e111f32dd4c30"
-  integrity sha512-aK0fHc9CBNx8jbzMYhshZcEv8LtYnBIRYQD5i7w/K/wS9c2+0NSR6B3OVMu5y0hBHYLcMGjfU+dmWYNKH0I85g==
+postcss-normalize-whitespace@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.1.1.tgz#08a1a0d1ffa17a7cc6efe1e6c9da969cc4493cfa"
+  integrity sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA==
   dependencies:
-    postcss "^7.0.2"
+    postcss-value-parser "^4.2.0"
 
-postcss-page-break@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-page-break/-/postcss-page-break-2.0.0.tgz#add52d0e0a528cabe6afee8b46e2abb277df46bf"
-  integrity sha512-tkpTSrLpfLfD9HvgOlJuigLuk39wVTbbd8RKcy8/ugV2bNBUW3xU+AIqyxhDrQr1VUj1RmyJrBn1YWrqUm9zAQ==
+postcss-normalize@^10.0.1:
+  version "10.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-normalize/-/postcss-normalize-10.0.1.tgz#464692676b52792a06b06880a176279216540dd7"
+  integrity sha512-+5w18/rDev5mqERcG3W5GZNMJa1eoYYNGo8gB7tEwaos0ajk3ZXAI4mHGcNT47NE+ZnZD1pEpUOFLvltIwmeJA==
   dependencies:
-    postcss "^7.0.2"
+    "@csstools/normalize.css" "*"
+    postcss-browser-comments "^4"
+    sanitize.css "*"
 
-postcss-place@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-place/-/postcss-place-4.0.1.tgz#e9f39d33d2dc584e46ee1db45adb77ca9d1dcc62"
-  integrity sha512-Zb6byCSLkgRKLODj/5mQugyuj9bvAAw9LqJJjgwz5cYryGeXfFZfSXoP1UfveccFmeq0b/2xxwcTEVScnqGxBg==
-  dependencies:
-    postcss "^7.0.2"
-    postcss-values-parser "^2.0.0"
-
-postcss-preset-env@6.7.0:
-  version "6.7.0"
-  resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-6.7.0.tgz#c34ddacf8f902383b35ad1e030f178f4cdf118a5"
-  integrity sha512-eU4/K5xzSFwUFJ8hTdTQzo2RBLbDVt83QZrAvI07TULOkmyQlnYlpwep+2yIK+K+0KlZO4BvFcleOCCcUtwchg==
-  dependencies:
-    autoprefixer "^9.6.1"
-    browserslist "^4.6.4"
-    caniuse-lite "^1.0.30000981"
-    css-blank-pseudo "^0.1.4"
-    css-has-pseudo "^0.10.0"
-    css-prefers-color-scheme "^3.1.1"
-    cssdb "^4.4.0"
-    postcss "^7.0.17"
-    postcss-attribute-case-insensitive "^4.0.1"
-    postcss-color-functional-notation "^2.0.1"
-    postcss-color-gray "^5.0.0"
-    postcss-color-hex-alpha "^5.0.3"
-    postcss-color-mod-function "^3.0.3"
-    postcss-color-rebeccapurple "^4.0.1"
-    postcss-custom-media "^7.0.8"
-    postcss-custom-properties "^8.0.11"
-    postcss-custom-selectors "^5.1.2"
-    postcss-dir-pseudo-class "^5.0.0"
-    postcss-double-position-gradients "^1.0.0"
-    postcss-env-function "^2.0.2"
-    postcss-focus-visible "^4.0.0"
-    postcss-focus-within "^3.0.0"
-    postcss-font-variant "^4.0.0"
-    postcss-gap-properties "^2.0.0"
-    postcss-image-set-function "^3.0.1"
-    postcss-initial "^3.0.0"
-    postcss-lab-function "^2.0.1"
-    postcss-logical "^3.0.0"
-    postcss-media-minmax "^4.0.0"
-    postcss-nesting "^7.0.0"
-    postcss-overflow-shorthand "^2.0.0"
-    postcss-page-break "^2.0.0"
-    postcss-place "^4.0.1"
-    postcss-pseudo-class-any-link "^6.0.0"
-    postcss-replace-overflow-wrap "^3.0.0"
-    postcss-selector-matches "^4.0.0"
-    postcss-selector-not "^4.0.0"
-
-postcss-pseudo-class-any-link@^6.0.0:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-pseudo-class-any-link/-/postcss-pseudo-class-any-link-6.0.0.tgz#2ed3eed393b3702879dec4a87032b210daeb04d1"
-  integrity sha512-lgXW9sYJdLqtmw23otOzrtbDXofUdfYzNm4PIpNE322/swES3VU9XlXHeJS46zT2onFO7V1QFdD4Q9LiZj8mew==
-  dependencies:
-    postcss "^7.0.2"
-    postcss-selector-parser "^5.0.0-rc.3"
+postcss-opacity-percentage@^1.1.2:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/postcss-opacity-percentage/-/postcss-opacity-percentage-1.1.3.tgz#5b89b35551a556e20c5d23eb5260fbfcf5245da6"
+  integrity sha512-An6Ba4pHBiDtyVpSLymUUERMo2cU7s+Obz6BTrS+gxkbnSBNKSuD0AVUc+CpBMrpVPKKfoVz0WQCX+Tnst0i4A==
 
-postcss-reduce-initial@^4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-4.0.3.tgz#7fd42ebea5e9c814609639e2c2e84ae270ba48df"
-  integrity sha512-gKWmR5aUulSjbzOfD9AlJiHCGH6AEVLaM0AV+aSioxUDd16qXP1PCh8d1/BGVvpdWn8k/HiK7n6TjeoXN1F7DA==
+postcss-ordered-values@^5.1.3:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/postcss-ordered-values/-/postcss-ordered-values-5.1.3.tgz#b6fd2bd10f937b23d86bc829c69e7732ce76ea38"
+  integrity sha512-9UO79VUhPwEkzbb3RNpqqghc6lcYej1aveQteWY+4POIwlqkYE21HKWaLDF6lWNuqCobEAyTovVhtI32Rbv2RQ==
   dependencies:
-    browserslist "^4.0.0"
-    caniuse-api "^3.0.0"
-    has "^1.0.0"
-    postcss "^7.0.0"
+    cssnano-utils "^3.1.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-reduce-transforms@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-reduce-transforms/-/postcss-reduce-transforms-4.0.2.tgz#17efa405eacc6e07be3414a5ca2d1074681d4e29"
-  integrity sha512-EEVig1Q2QJ4ELpJXMZR8Vt5DQx8/mo+dGWSR7vWXqcob2gQLyQGsionYcGKATXvQzMPn6DSN1vTN7yFximdIAg==
+postcss-overflow-shorthand@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/postcss-overflow-shorthand/-/postcss-overflow-shorthand-3.0.4.tgz#7ed6486fec44b76f0eab15aa4866cda5d55d893e"
+  integrity sha512-otYl/ylHK8Y9bcBnPLo3foYFLL6a6Ak+3EQBPOTR7luMYCOsiVTUk1iLvNf6tVPNGXcoL9Hoz37kpfriRIFb4A==
   dependencies:
-    cssnano-util-get-match "^4.0.0"
-    has "^1.0.0"
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-replace-overflow-wrap@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-replace-overflow-wrap/-/postcss-replace-overflow-wrap-3.0.0.tgz#61b360ffdaedca84c7c918d2b0f0d0ea559ab01c"
-  integrity sha512-2T5hcEHArDT6X9+9dVSPQdo7QHzG4XKclFT8rU5TzJPDN7RIRTbO9c4drUISOVemLj03aezStHCR2AIcr8XLpw==
-  dependencies:
-    postcss "^7.0.2"
+postcss-page-break@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/postcss-page-break/-/postcss-page-break-3.0.4.tgz#7fbf741c233621622b68d435babfb70dd8c1ee5f"
+  integrity sha512-1JGu8oCjVXLa9q9rFTo4MbeeA5FMe00/9C7lN4va606Rdb+HkxXtXsmEDrIraQ11fGz/WvKWa8gMuCKkrXpTsQ==
 
-postcss-safe-parser@5.0.2:
-  version "5.0.2"
-  resolved "https://registry.yarnpkg.com/postcss-safe-parser/-/postcss-safe-parser-5.0.2.tgz#459dd27df6bc2ba64608824ba39e45dacf5e852d"
-  integrity sha512-jDUfCPJbKOABhwpUKcqCVbbXiloe/QXMcbJ6Iipf3sDIihEzTqRCeMBfRaOHxhBuTYqtASrI1KJWxzztZU4qUQ==
+postcss-place@^7.0.5:
+  version "7.0.5"
+  resolved "https://registry.yarnpkg.com/postcss-place/-/postcss-place-7.0.5.tgz#95dbf85fd9656a3a6e60e832b5809914236986c4"
+  integrity sha512-wR8igaZROA6Z4pv0d+bvVrvGY4GVHihBCBQieXFY3kuSuMyOmEnnfFzHl/tQuqHZkfkIVBEbDvYcFfHmpSet9g==
   dependencies:
-    postcss "^8.1.0"
+    postcss-value-parser "^4.2.0"
 
-postcss-selector-matches@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-selector-matches/-/postcss-selector-matches-4.0.0.tgz#71c8248f917ba2cc93037c9637ee09c64436fcff"
-  integrity sha512-LgsHwQR/EsRYSqlwdGzeaPKVT0Ml7LAT6E75T8W8xLJY62CE4S/l03BWIt3jT8Taq22kXP08s2SfTSzaraoPww==
+postcss-preset-env@^7.0.1:
+  version "7.8.3"
+  resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-7.8.3.tgz#2a50f5e612c3149cc7af75634e202a5b2ad4f1e2"
+  integrity sha512-T1LgRm5uEVFSEF83vHZJV2z19lHg4yJuZ6gXZZkqVsqv63nlr6zabMH3l4Pc01FQCyfWVrh2GaUeCVy9Po+Aag==
+  dependencies:
+    "@csstools/postcss-cascade-layers" "^1.1.1"
+    "@csstools/postcss-color-function" "^1.1.1"
+    "@csstools/postcss-font-format-keywords" "^1.0.1"
+    "@csstools/postcss-hwb-function" "^1.0.2"
+    "@csstools/postcss-ic-unit" "^1.0.1"
+    "@csstools/postcss-is-pseudo-class" "^2.0.7"
+    "@csstools/postcss-nested-calc" "^1.0.0"
+    "@csstools/postcss-normalize-display-values" "^1.0.1"
+    "@csstools/postcss-oklab-function" "^1.1.1"
+    "@csstools/postcss-progressive-custom-properties" "^1.3.0"
+    "@csstools/postcss-stepped-value-functions" "^1.0.1"
+    "@csstools/postcss-text-decoration-shorthand" "^1.0.0"
+    "@csstools/postcss-trigonometric-functions" "^1.0.2"
+    "@csstools/postcss-unset-value" "^1.0.2"
+    autoprefixer "^10.4.13"
+    browserslist "^4.21.4"
+    css-blank-pseudo "^3.0.3"
+    css-has-pseudo "^3.0.4"
+    css-prefers-color-scheme "^6.0.3"
+    cssdb "^7.1.0"
+    postcss-attribute-case-insensitive "^5.0.2"
+    postcss-clamp "^4.1.0"
+    postcss-color-functional-notation "^4.2.4"
+    postcss-color-hex-alpha "^8.0.4"
+    postcss-color-rebeccapurple "^7.1.1"
+    postcss-custom-media "^8.0.2"
+    postcss-custom-properties "^12.1.10"
+    postcss-custom-selectors "^6.0.3"
+    postcss-dir-pseudo-class "^6.0.5"
+    postcss-double-position-gradients "^3.1.2"
+    postcss-env-function "^4.0.6"
+    postcss-focus-visible "^6.0.4"
+    postcss-focus-within "^5.0.4"
+    postcss-font-variant "^5.0.0"
+    postcss-gap-properties "^3.0.5"
+    postcss-image-set-function "^4.0.7"
+    postcss-initial "^4.0.1"
+    postcss-lab-function "^4.2.1"
+    postcss-logical "^5.0.4"
+    postcss-media-minmax "^5.0.0"
+    postcss-nesting "^10.2.0"
+    postcss-opacity-percentage "^1.1.2"
+    postcss-overflow-shorthand "^3.0.4"
+    postcss-page-break "^3.0.4"
+    postcss-place "^7.0.5"
+    postcss-pseudo-class-any-link "^7.1.6"
+    postcss-replace-overflow-wrap "^4.0.0"
+    postcss-selector-not "^6.0.1"
+    postcss-value-parser "^4.2.0"
+
+postcss-pseudo-class-any-link@^7.1.6:
+  version "7.1.6"
+  resolved "https://registry.yarnpkg.com/postcss-pseudo-class-any-link/-/postcss-pseudo-class-any-link-7.1.6.tgz#2693b221902da772c278def85a4d9a64b6e617ab"
+  integrity sha512-9sCtZkO6f/5ML9WcTLcIyV1yz9D1rf0tWc+ulKcvV30s0iZKS/ONyETvoWsr6vnrmW+X+KmuK3gV/w5EWnT37w==
+  dependencies:
+    postcss-selector-parser "^6.0.10"
+
+postcss-reduce-initial@^5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-5.1.2.tgz#798cd77b3e033eae7105c18c9d371d989e1382d6"
+  integrity sha512-dE/y2XRaqAi6OvjzD22pjTUQ8eOfc6m/natGHgKFBK9DxFmIm69YmaRVQrGgFlEfc1HePIurY0TmDeROK05rIg==
   dependencies:
-    balanced-match "^1.0.0"
-    postcss "^7.0.2"
+    browserslist "^4.21.4"
+    caniuse-api "^3.0.0"
 
-postcss-selector-not@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-selector-not/-/postcss-selector-not-4.0.1.tgz#263016eef1cf219e0ade9a913780fc1f48204cbf"
-  integrity sha512-YolvBgInEK5/79C+bdFMyzqTg6pkYqDbzZIST/PDMqa/o3qtXenD05apBG2jLgT0/BQ77d4U2UK12jWpilqMAQ==
+postcss-reduce-transforms@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-reduce-transforms/-/postcss-reduce-transforms-5.1.0.tgz#333b70e7758b802f3dd0ddfe98bb1ccfef96b6e9"
+  integrity sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ==
   dependencies:
-    balanced-match "^1.0.0"
-    postcss "^7.0.2"
+    postcss-value-parser "^4.2.0"
 
-postcss-selector-parser@^3.0.0:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz#b310f5c4c0fdaf76f94902bbaa30db6aa84f5270"
-  integrity sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==
-  dependencies:
-    dot-prop "^5.2.0"
-    indexes-of "^1.0.1"
-    uniq "^1.0.1"
+postcss-replace-overflow-wrap@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/postcss-replace-overflow-wrap/-/postcss-replace-overflow-wrap-4.0.0.tgz#d2df6bed10b477bf9c52fab28c568b4b29ca4319"
+  integrity sha512-KmF7SBPphT4gPPcKZc7aDkweHiKEEO8cla/GjcBK+ckKxiZslIu3C4GCRW3DNfL0o7yW7kMQu9xlZ1kXRXLXtw==
 
-postcss-selector-parser@^5.0.0-rc.3, postcss-selector-parser@^5.0.0-rc.4:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-5.0.0.tgz#249044356697b33b64f1a8f7c80922dddee7195c"
-  integrity sha512-w+zLE5Jhg6Liz8+rQOWEAwtwkyqpfnmsinXjXg6cY7YIONZZtgvE0v2O0uhQBs0peNomOJwWRKt6JBfTdTd3OQ==
+postcss-selector-not@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-selector-not/-/postcss-selector-not-6.0.1.tgz#8f0a709bf7d4b45222793fc34409be407537556d"
+  integrity sha512-1i9affjAe9xu/y9uqWH+tD4r6/hDaXJruk8xn2x1vzxC2U3J3LKO3zJW4CyxlNhA56pADJ/djpEwpH1RClI2rQ==
   dependencies:
-    cssesc "^2.0.0"
-    indexes-of "^1.0.1"
-    uniq "^1.0.1"
+    postcss-selector-parser "^6.0.10"
 
-postcss-selector-parser@^6.0.0, postcss-selector-parser@^6.0.2:
-  version "6.0.6"
-  resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.6.tgz#2c5bba8174ac2f6981ab631a42ab0ee54af332ea"
-  integrity sha512-9LXrvaaX3+mcv5xkg5kFwqSzSH1JIObIx51PrndZwlmznwXRfxMddDvo9gve3gVR8ZTKgoFDdWkbRFmEhT4PMg==
+postcss-selector-parser@^6.0.10, postcss-selector-parser@^6.0.11, postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.9:
+  version "6.0.13"
+  resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.13.tgz#d05d8d76b1e8e173257ef9d60b706a8e5e99bf1b"
+  integrity sha512-EaV1Gl4mUEV4ddhDnv/xtj7sxwrwxdetHdWUGnT4VJQf+4d05v6lHYZr8N573k5Z0BViss7BDhfWtKS3+sfAqQ==
   dependencies:
     cssesc "^3.0.0"
     util-deprecate "^1.0.2"
 
-postcss-svgo@^4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-svgo/-/postcss-svgo-4.0.3.tgz#343a2cdbac9505d416243d496f724f38894c941e"
-  integrity sha512-NoRbrcMWTtUghzuKSoIm6XV+sJdvZ7GZSc3wdBN0W19FTtp2ko8NqLsgoh/m9CzNhU3KLPvQmjIwtaNFkaFTvw==
-  dependencies:
-    postcss "^7.0.0"
-    postcss-value-parser "^3.0.0"
-    svgo "^1.0.0"
-
-postcss-unique-selectors@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-4.0.1.tgz#9446911f3289bfd64c6d680f073c03b1f9ee4bac"
-  integrity sha512-+JanVaryLo9QwZjKrmJgkI4Fn8SBgRO6WXQBJi7KiAVPlmxikB5Jzc4EvXMT2H0/m0RjrVVm9rGNhZddm/8Spg==
+postcss-svgo@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-svgo/-/postcss-svgo-5.1.0.tgz#0a317400ced789f233a28826e77523f15857d80d"
+  integrity sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA==
   dependencies:
-    alphanum-sort "^1.0.0"
-    postcss "^7.0.0"
-    uniqs "^2.0.0"
-
-postcss-value-parser@^3.0.0:
-  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz#9ff822547e2893213cf1c30efa51ac5fd1ba8281"
-  integrity sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==
+    postcss-value-parser "^4.2.0"
+    svgo "^2.7.0"
 
-postcss-value-parser@^4.0.2, postcss-value-parser@^4.1.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz#443f6a20ced6481a2bda4fa8532a6e55d789a2cb"
-  integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==
-
-postcss-values-parser@^2.0.0, postcss-values-parser@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/postcss-values-parser/-/postcss-values-parser-2.0.1.tgz#da8b472d901da1e205b47bdc98637b9e9e550e5f"
-  integrity sha512-2tLuBsA6P4rYTNKCXYG/71C7j1pU6pK503suYOmn4xYrQIzW+opD+7FAFNuGSdZC/3Qfy334QbeMu7MEb8gOxg==
+postcss-unique-selectors@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-5.1.1.tgz#a9f273d1eacd09e9aa6088f4b0507b18b1b541b6"
+  integrity sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA==
   dependencies:
-    flatten "^1.0.2"
-    indexes-of "^1.0.1"
-    uniq "^1.0.1"
+    postcss-selector-parser "^6.0.5"
 
-postcss@7.0.21:
-  version "7.0.21"
-  resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.21.tgz#06bb07824c19c2021c5d056d5b10c35b989f7e17"
-  integrity sha512-uIFtJElxJo29QC753JzhidoAhvp/e/Exezkdhfmt8AymWT6/5B7W1WmponYWkHk2eg6sONyTch0A3nkMPun3SQ==
-  dependencies:
-    chalk "^2.4.2"
-    source-map "^0.6.1"
-    supports-color "^6.1.0"
+postcss-value-parser@^4.0.0, postcss-value-parser@^4.0.2, postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514"
+  integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==
 
-postcss@^7, postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.17, postcss@^7.0.2, postcss@^7.0.26, postcss@^7.0.27, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6:
-  version "7.0.35"
-  resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.35.tgz#d2be00b998f7f211d8a276974079f2e92b970e24"
-  integrity sha512-3QT8bBJeX/S5zKTTjTCIjRF3If4avAT6kqxcASlTWEtAFCb9NH0OUxNDfgZSWdP5fJnBYCMEWkIFfWeugjzYMg==
+postcss@^7.0.35:
+  version "7.0.39"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309"
+  integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA==
   dependencies:
-    chalk "^2.4.2"
+    picocolors "^0.2.1"
     source-map "^0.6.1"
-    supports-color "^6.1.0"
 
-postcss@^8.1.0:
-  version "8.3.0"
-  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.0.tgz#b1a713f6172ca427e3f05ef1303de8b65683325f"
-  integrity sha512-+ogXpdAjWGa+fdYY5BQ96V/6tAo+TdSSIMP5huJBIygdWwKtVoB5JWZ7yUd4xZ8r+8Kvvx4nyg/PQ071H4UtcQ==
+postcss@^8.3.5, postcss@^8.4.21, postcss@^8.4.23, postcss@^8.4.4:
+  version "8.4.24"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.24.tgz#f714dba9b2284be3cc07dbd2fc57ee4dc972d2df"
+  integrity sha512-M0RzbcI0sO/XJNucsGjvWU9ERWxb/ytp1w6dKtxTKgixdtQDq4rmx/g8W1hnaheq9jgwL/oyEdH5Bc4WwJKMqg==
   dependencies:
-    colorette "^1.2.2"
-    nanoid "^3.1.23"
-    source-map-js "^0.6.2"
+    nanoid "^3.3.6"
+    picocolors "^1.0.0"
+    source-map-js "^1.0.2"
 
 precision@~1.0.0:
   version "1.0.1"
@@ -15166,11 +13202,6 @@ prelude-ls@~1.1.2:
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
   integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=
 
-prepend-http@^1.0.0:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc"
-  integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=
-
 prepend-http@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
@@ -15184,32 +13215,25 @@ pretender@^3.4.3:
     fake-xml-http-request "^2.1.1"
     route-recognizer "^0.3.3"
 
-prettier-linter-helpers@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz#d23d41fe1375646de2d0104d3454a3008802cf7b"
-  integrity sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==
-  dependencies:
-    fast-diff "^1.1.2"
-
-prettier@^2.3.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.0.tgz#b6a5bf1284026ae640f17f7ff5658a7567fc0d18"
-  integrity sha512-kXtO4s0Lz/DW/IJ9QdWhAf7/NmPWQXkFr/r/WkR3vyI+0v8amTDxiaQSLzs8NBlytfLWX/7uQUMIW677yLKl4w==
+prettier@^2.8.8:
+  version "2.8.8"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da"
+  integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==
 
-pretty-bytes@^5.3.0:
+pretty-bytes@^5.3.0, pretty-bytes@^5.4.1:
   version "5.6.0"
-  resolved "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb"
+  resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb"
   integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==
 
-pretty-error@^2.1.1:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-2.1.2.tgz#be89f82d81b1c86ec8fdfbc385045882727f93b6"
-  integrity sha512-EY5oDzmsX5wvuynAByrmY0P0hcp+QpnAKbJng2A2MPjVKXCxrDSUkzghVJ4ZGPIv+JC4gX8fPUWscC0RtjsWGw==
+pretty-error@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-4.0.0.tgz#90a703f46dd7234adb46d0f84823e9d1cb8f10d6"
+  integrity sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==
   dependencies:
     lodash "^4.17.20"
-    renderkid "^2.0.4"
+    renderkid "^3.0.0"
 
-pretty-format@^26.0.0, pretty-format@^26.6.0, pretty-format@^26.6.2:
+pretty-format@^26.0.0, pretty-format@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.6.2.tgz#e35c2705f14cb7fe2fe94fa078345b444120fc93"
   integrity sha512-7AeGuCYNGmycyQbCqd/3PWH4eOoX/OiCa0uphp57NVTeAGdJGaAliecxwBDHYQCIvrW7aDBZCYeNTP/WX69mkg==
@@ -15219,6 +13243,25 @@ pretty-format@^26.0.0, pretty-format@^26.6.0, pretty-format@^26.6.2:
     ansi-styles "^4.0.0"
     react-is "^17.0.1"
 
+pretty-format@^27.5.1:
+  version "27.5.1"
+  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e"
+  integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==
+  dependencies:
+    ansi-regex "^5.0.1"
+    ansi-styles "^5.0.0"
+    react-is "^17.0.1"
+
+pretty-format@^28.1.3:
+  version "28.1.3"
+  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-28.1.3.tgz#c9fba8cedf99ce50963a11b27d982a9ae90970d5"
+  integrity sha512-8gFb/To0OmxHR9+ZTb14Df2vNxdGCX8g1xWGUTqUw5TiZvcQf5sHKObd5UcPyLLyowNwDAMTF3XWOG1B6mxl1Q==
+  dependencies:
+    "@jest/schemas" "^28.1.3"
+    ansi-regex "^5.0.1"
+    ansi-styles "^5.0.0"
+    react-is "^18.0.0"
+
 prismjs@^1.22.0:
   version "1.24.1"
   resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.24.1.tgz#c4d7895c4d6500289482fa8936d9cdd192684036"
@@ -15236,21 +13279,6 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
-process@^0.11.10:
-  version "0.11.10"
-  resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
-  integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI=
-
-progress@^2.0.0:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
-  integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
-
-promise-inflight@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"
-  integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM=
-
 promise@^7.1.1:
   version "7.3.1"
   resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf"
@@ -15265,23 +13293,15 @@ promise@^8.1.0:
   dependencies:
     asap "~2.0.6"
 
-prompts@2.4.0:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.0.tgz#4aa5de0723a231d1ee9121c40fdf663df73f61d7"
-  integrity sha512-awZAKrk3vN6CroQukBL+R9051a4R3zCZBlJm/HBfrSZ8iTpYix3VX1vU4mveiLpiwmOJT4wokTF9m6HUk4KqWQ==
-  dependencies:
-    kleur "^3.0.3"
-    sisteransi "^1.0.5"
-
-prompts@^2.0.1:
-  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.1.tgz#befd3b1195ba052f9fd2fde8a486c4e82ee77f61"
-  integrity sha512-EQyfIuO2hPDsX1L/blblV+H7I0knhgAd82cVneCwcdND9B8AuCDuRcBH6yIcG4dFzlOUqbazQqwGjx5xmsNLuQ==
+prompts@^2.0.1, prompts@^2.4.2:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"
+  integrity sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==
   dependencies:
     kleur "^3.0.3"
     sisteransi "^1.0.5"
 
-prop-types@15.7.2, prop-types@^15.5.10, prop-types@^15.5.7, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2:
+prop-types@15.7.2:
   version "15.7.2"
   resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.7.2.tgz#52c41e75b8c87e72b9d9360e0206b99dcbffa6c5"
   integrity sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==
@@ -15290,7 +13310,7 @@ prop-types@15.7.2, prop-types@^15.5.10, prop-types@^15.5.7, prop-types@^15.6.0,
     object-assign "^4.1.1"
     react-is "^16.8.1"
 
-prop-types@^15.8.1:
+prop-types@^15.5.10, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1:
   version "15.8.1"
   resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
   integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
@@ -15494,26 +13514,6 @@ psl@^1.1.28, psl@^1.1.33:
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
   integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
 
-public-encrypt@^4.0.0:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/public-encrypt/-/public-encrypt-4.0.3.tgz#4fcc9d77a07e48ba7527e7cbe0de33d0701331e0"
-  integrity sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==
-  dependencies:
-    bn.js "^4.1.0"
-    browserify-rsa "^4.0.0"
-    create-hash "^1.1.0"
-    parse-asn1 "^5.0.0"
-    randombytes "^2.0.1"
-    safe-buffer "^5.1.2"
-
-pump@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/pump/-/pump-2.0.1.tgz#12399add6e4cf7526d973cbc8b5ce2e2908b3909"
-  integrity sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==
-  dependencies:
-    end-of-stream "^1.1.0"
-    once "^1.3.1"
-
 pump@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
@@ -15522,25 +13522,6 @@ pump@^3.0.0:
     end-of-stream "^1.1.0"
     once "^1.3.1"
 
-pumpify@^1.3.3:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/pumpify/-/pumpify-1.5.1.tgz#36513be246ab27570b1a374a5ce278bfd74370ce"
-  integrity sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==
-  dependencies:
-    duplexify "^3.6.0"
-    inherits "^2.0.3"
-    pump "^2.0.0"
-
-punycode@1.3.2:
-  version "1.3.2"
-  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
-  integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
-
-punycode@^1.2.4:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
-  integrity sha1-wNWmOycYgArY4esPpSachN1BhF4=
-
 punycode@^2.1.0, punycode@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
@@ -15573,14 +13554,6 @@ query-string@*:
     split-on-first "^1.0.0"
     strict-uri-encode "^2.0.0"
 
-query-string@^4.1.0:
-  version "4.3.4"
-  resolved "https://registry.yarnpkg.com/query-string/-/query-string-4.3.4.tgz#bbb693b9ca915c232515b228b1a02b609043dbeb"
-  integrity sha1-u7aTucqRXCMlFbIosaArYJBD2+s=
-  dependencies:
-    object-assign "^4.1.0"
-    strict-uri-encode "^1.0.0"
-
 query-string@^6.13.8:
   version "6.14.1"
   resolved "https://registry.yarnpkg.com/query-string/-/query-string-6.14.1.tgz#7ac2dca46da7f309449ba0f86b1fd28255b0c86a"
@@ -15591,22 +13564,7 @@ query-string@^6.13.8:
     split-on-first "^1.0.0"
     strict-uri-encode "^2.0.0"
 
-querystring-es3@^0.2.0:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73"
-  integrity sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=
-
-querystring@0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
-  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
-
-querystring@^0.2.0:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.1.tgz#40d77615bb09d16902a85c3e38aa8b5ed761c2dd"
-  integrity sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg==
-
-querystringify@^2.1.1, querystringify@^2.2.0:
+querystringify@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6"
   integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==
@@ -15628,21 +13586,13 @@ ramda@0.26.1:
   resolved "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz#8d41351eb8111c55353617fc3bbffad8e4d35d06"
   integrity sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ==
 
-randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0:
+randombytes@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a"
   integrity sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==
   dependencies:
     safe-buffer "^5.1.0"
 
-randomfill@^1.0.3:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/randomfill/-/randomfill-1.0.4.tgz#c92196fc86ab42be983f1bf31778224931d61458"
-  integrity sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==
-  dependencies:
-    randombytes "^2.0.5"
-    safe-buffer "^5.1.0"
-
 range-parser@^1.2.1, range-parser@~1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031"
@@ -15796,16 +13746,7 @@ rc-menu@~9.8.0:
     rc-util "^5.12.0"
     shallowequal "^1.1.0"
 
-rc-motion@^2.0.0, rc-motion@^2.0.1, rc-motion@^2.2.0, rc-motion@^2.3.0, rc-motion@^2.3.4, rc-motion@^2.4.3:
-  version "2.4.3"
-  resolved "https://registry.yarnpkg.com/rc-motion/-/rc-motion-2.4.3.tgz#2afd129da8764ee0372ba83442949d8ecb1c7ad2"
-  integrity sha512-GZLLFXHl/VqTfI7bSZNNZozcblNmDka1AAoQig7EZ6s0rWg5y0RlgrcHWO+W+nrOVbYfJDxoaQUoP2fEmvCWmA==
-  dependencies:
-    "@babel/runtime" "^7.11.1"
-    classnames "^2.2.1"
-    rc-util "^5.2.1"
-
-rc-motion@^2.4.4, rc-motion@^2.6.1, rc-motion@^2.6.2:
+rc-motion@^2.0.0, rc-motion@^2.0.1, rc-motion@^2.2.0, rc-motion@^2.3.0, rc-motion@^2.3.4, rc-motion@^2.4.3, rc-motion@^2.4.4, rc-motion@^2.6.1, rc-motion@^2.6.2:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/rc-motion/-/rc-motion-2.6.2.tgz#3d31f97e41fb8e4f91a4a4189b6a98ac63342869"
   integrity sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==
@@ -15824,17 +13765,7 @@ rc-notification@~4.6.0:
     rc-motion "^2.2.0"
     rc-util "^5.20.1"
 
-rc-overflow@^1.0.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/rc-overflow/-/rc-overflow-1.2.1.tgz#ea13f1e2ec152d8adcfc3bf1e53fa2b62a1d2cff"
-  integrity sha512-TCB8QiEnmNUbsJZX1GU8ZvIgVxk42eu3yaRdDZc2HFjVeT3HBSfscVaCVzBuH3NR5IWrQLodI0p6MZEGFn+KiA==
-  dependencies:
-    "@babel/runtime" "^7.11.1"
-    classnames "^2.2.1"
-    rc-resize-observer "^1.0.0"
-    rc-util "^5.5.1"
-
-rc-overflow@^1.2.8:
+rc-overflow@^1.0.0, rc-overflow@^1.2.8:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/rc-overflow/-/rc-overflow-1.2.8.tgz#40f140fabc244118543e627cdd1ef750d9481a88"
   integrity sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==
@@ -15884,17 +13815,7 @@ rc-rate@~2.9.0:
     classnames "^2.2.5"
     rc-util "^5.0.1"
 
-rc-resize-observer@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/rc-resize-observer/-/rc-resize-observer-1.0.0.tgz#97fb89856f62fec32ab6e40933935cf58e2e102d"
-  integrity sha512-RgKGukg1mlzyGdvzF7o/LGFC8AeoMH9aGzXTUdp6m+OApvmRdUuOscq/Y2O45cJA+rXt1ApWlpFoOIioXL3AGg==
-  dependencies:
-    "@babel/runtime" "^7.10.1"
-    classnames "^2.2.1"
-    rc-util "^5.0.0"
-    resize-observer-polyfill "^1.5.1"
-
-rc-resize-observer@^1.1.0, rc-resize-observer@^1.2.0:
+rc-resize-observer@^1.0.0, rc-resize-observer@^1.1.0, rc-resize-observer@^1.2.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/rc-resize-observer/-/rc-resize-observer-1.2.1.tgz#7f9715b5d1afe126ade3c107aafd2cebf8a57a99"
   integrity sha512-g53PnWLeVOmt4XWkt2x+QlIdf/PhJSd7JqHhtMrUY370e7wJ+kxbgXicYqvENUcgFiiOiMCd07YsC2GNsoSbnA==
@@ -15955,18 +13876,7 @@ rc-switch@~3.2.0:
     classnames "^2.2.1"
     rc-util "^5.0.1"
 
-rc-table@^7.13.1:
-  version "7.15.1"
-  resolved "https://registry.yarnpkg.com/rc-table/-/rc-table-7.15.1.tgz#e5dde23289f55c9e0d4b217a61b9487bb4325b2a"
-  integrity sha512-RJ+BkBrrHt9/M4Dz5zIXYyAzrrqB/FjTGDGAfJv7P8LHczpLEt1yRTlz2wjX1ktxMpj16gZlssIchbKTqdKjlw==
-  dependencies:
-    "@babel/runtime" "^7.10.1"
-    classnames "^2.2.5"
-    rc-resize-observer "^1.0.0"
-    rc-util "^5.13.0"
-    shallowequal "^1.1.0"
-
-rc-table@~7.26.0:
+rc-table@^7.13.1, rc-table@~7.26.0:
   version "7.26.0"
   resolved "https://registry.yarnpkg.com/rc-table/-/rc-table-7.26.0.tgz#9d517e7fa512e7571fdcc453eb1bf19edfac6fbc"
   integrity sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==
@@ -16032,29 +13942,7 @@ rc-tree@~5.7.0:
     rc-util "^5.16.1"
     rc-virtual-list "^3.4.8"
 
-rc-trigger@^5.0.0, rc-trigger@^5.0.4, rc-trigger@^5.1.2:
-  version "5.2.8"
-  resolved "https://registry.yarnpkg.com/rc-trigger/-/rc-trigger-5.2.8.tgz#27c8291c24518b8f11d76c848f5424e0c429e94a"
-  integrity sha512-Tn84oGmvNBLXI+ptpzxyJx4ArKTduuB6l74ShDLhDaJaF9f5JAMizfx31L30ELVIzRr3Ze4sekG7rzwPGwVOdw==
-  dependencies:
-    "@babel/runtime" "^7.11.2"
-    classnames "^2.2.6"
-    rc-align "^4.0.0"
-    rc-motion "^2.0.0"
-    rc-util "^5.5.0"
-
-rc-trigger@^5.2.10:
-  version "5.3.3"
-  resolved "https://registry.yarnpkg.com/rc-trigger/-/rc-trigger-5.3.3.tgz#166013df79e6a4ce64515391bd6d4f8386839761"
-  integrity sha512-IC4nuTSAME7RJSgwvHCNDQrIzhvGMKf6NDu5veX+zk1MG7i1UnwTWWthcP9WHw3+FZfP3oZGvkrHFPu/EGkFKw==
-  dependencies:
-    "@babel/runtime" "^7.18.3"
-    classnames "^2.2.6"
-    rc-align "^4.0.0"
-    rc-motion "^2.0.0"
-    rc-util "^5.19.2"
-
-rc-trigger@^5.3.1:
+rc-trigger@^5.0.0, rc-trigger@^5.0.4, rc-trigger@^5.1.2, rc-trigger@^5.2.10, rc-trigger@^5.3.1:
   version "5.3.4"
   resolved "https://registry.yarnpkg.com/rc-trigger/-/rc-trigger-5.3.4.tgz#6b4b26e32825677c837d1eb4d7085035eecf9a61"
   integrity sha512-mQv+vas0TwKcjAO2izNPkqR4j86OemLRmvL2nOzdP9OWNWA1ivoTt5hzFqYNW9zACwmTezRiN8bttrC7cZzYSw==
@@ -16074,25 +13962,7 @@ rc-upload@~4.3.0:
     classnames "^2.2.5"
     rc-util "^5.2.0"
 
-rc-util@^5.0.0, rc-util@^5.0.1, rc-util@^5.0.6, rc-util@^5.0.7, rc-util@^5.12.0, rc-util@^5.13.0, rc-util@^5.2.0, rc-util@^5.2.1, rc-util@^5.3.0, rc-util@^5.4.0, rc-util@^5.5.0, rc-util@^5.5.1, rc-util@^5.6.1, rc-util@^5.8.0, rc-util@^5.9.4:
-  version "5.13.1"
-  resolved "https://registry.yarnpkg.com/rc-util/-/rc-util-5.13.1.tgz#03e74955b5c46a58cbc6236e4d30dd462c755290"
-  integrity sha512-Dws2tjXBBihfjVQFlG5JzZ/5O3Wutctm0W94Wb1+M7GD2roWJPrQdSa4AkWm2pn0Ms32zoVPPkWodFeAYZPLfA==
-  dependencies:
-    "@babel/runtime" "^7.12.5"
-    react-is "^16.12.0"
-    shallowequal "^1.1.0"
-
-rc-util@^5.15.0, rc-util@^5.16.1, rc-util@^5.17.0, rc-util@^5.19.2, rc-util@^5.21.0, rc-util@^5.23.0:
-  version "5.24.4"
-  resolved "https://registry.yarnpkg.com/rc-util/-/rc-util-5.24.4.tgz#a4126f01358c86f17c1bf380a1d83d6c9155ae65"
-  integrity sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==
-  dependencies:
-    "@babel/runtime" "^7.18.3"
-    react-is "^16.12.0"
-    shallowequal "^1.1.0"
-
-rc-util@^5.16.0, rc-util@^5.18.1, rc-util@^5.20.1, rc-util@^5.21.2, rc-util@^5.22.5, rc-util@^5.24.4, rc-util@^5.27.0:
+rc-util@^5.0.1, rc-util@^5.0.6, rc-util@^5.12.0, rc-util@^5.15.0, rc-util@^5.16.0, rc-util@^5.16.1, rc-util@^5.17.0, rc-util@^5.18.1, rc-util@^5.19.2, rc-util@^5.2.0, rc-util@^5.2.1, rc-util@^5.20.1, rc-util@^5.21.0, rc-util@^5.21.2, rc-util@^5.22.5, rc-util@^5.23.0, rc-util@^5.24.4, rc-util@^5.27.0, rc-util@^5.3.0, rc-util@^5.4.0, rc-util@^5.6.1, rc-util@^5.8.0, rc-util@^5.9.4:
   version "5.27.1"
   resolved "https://registry.yarnpkg.com/rc-util/-/rc-util-5.27.1.tgz#d12f02b9577b04299c0f1a235c8acbcf56e2824b"
   integrity sha512-PsjHA+f+KBCz+YTZxrl3ukJU5RoNKoe3KSNMh0xGiISbR67NaM9E9BiMjCwxa3AcCUOg/rZ+V0ZKLSimAA+e3w==
@@ -16100,16 +13970,7 @@ rc-util@^5.16.0, rc-util@^5.18.1, rc-util@^5.20.1, rc-util@^5.21.2, rc-util@^5.2
     "@babel/runtime" "^7.18.3"
     react-is "^16.12.0"
 
-rc-virtual-list@^3.2.0:
-  version "3.2.6"
-  resolved "https://registry.yarnpkg.com/rc-virtual-list/-/rc-virtual-list-3.2.6.tgz#2c92a40f4425e19881b38134d6bd286a11137d2d"
-  integrity sha512-8FiQLDzm3c/tMX0d62SQtKDhLH7zFlSI6pWBAPt+TUntEqd3Lz9zFAmpvTu8gkvUom/HCsDSZs4wfV4wDPWC0Q==
-  dependencies:
-    classnames "^2.2.6"
-    rc-resize-observer "^1.0.0"
-    rc-util "^5.0.7"
-
-rc-virtual-list@^3.4.8:
+rc-virtual-list@^3.2.0, rc-virtual-list@^3.4.8:
   version "3.4.13"
   resolved "https://registry.yarnpkg.com/rc-virtual-list/-/rc-virtual-list-3.4.13.tgz#20acc934b263abcf7b7c161f50ef82281b2f7e8d"
   integrity sha512-cPOVDmcNM7rH6ANotanMDilW/55XnFPw0Jh/GQYtrzZSy3AmWvCnqVNyNC/pgg3lfVmX2994dlzAhuUrd4jG7w==
@@ -16129,17 +13990,17 @@ rc@^1.2.8:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
-react-app-polyfill@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/react-app-polyfill/-/react-app-polyfill-2.0.0.tgz#a0bea50f078b8a082970a9d853dc34b6dcc6a3cf"
-  integrity sha512-0sF4ny9v/B7s6aoehwze9vJNWcmCemAUYBVasscVr92+UYiEqDXOxfKjXN685mDaMRNF3WdhHQs76oTODMocFA==
+react-app-polyfill@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/react-app-polyfill/-/react-app-polyfill-3.0.0.tgz#95221e0a9bd259e5ca6b177c7bb1cb6768f68fd7"
+  integrity sha512-sZ41cxiU5llIB003yxxQBYrARBqe0repqPTTYBTmMqTz9szeBbE37BehCE891NZsmdZqqP+xWKdT3eo3vOzN8w==
   dependencies:
-    core-js "^3.6.5"
+    core-js "^3.19.2"
     object-assign "^4.1.1"
     promise "^8.1.0"
     raf "^3.4.1"
-    regenerator-runtime "^0.13.7"
-    whatwg-fetch "^3.4.1"
+    regenerator-runtime "^0.13.9"
+    whatwg-fetch "^3.6.2"
 
 react-clientside-effect@^1.2.5:
   version "1.2.6"
@@ -16161,35 +14022,35 @@ react-color@^2.19.3:
     reactcss "^1.2.0"
     tinycolor2 "^1.4.1"
 
-react-dev-utils@^11.0.3:
-  version "11.0.4"
-  resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-11.0.4.tgz#a7ccb60257a1ca2e0efe7a83e38e6700d17aa37a"
-  integrity sha512-dx0LvIGHcOPtKbeiSUM4jqpBl3TcY7CDjZdfOIcKeznE7BWr9dg0iPG90G5yfVQ+p/rGNMXdbfStvzQZEVEi4A==
-  dependencies:
-    "@babel/code-frame" "7.10.4"
-    address "1.1.2"
-    browserslist "4.14.2"
-    chalk "2.4.2"
-    cross-spawn "7.0.3"
-    detect-port-alt "1.1.6"
-    escape-string-regexp "2.0.0"
-    filesize "6.1.0"
-    find-up "4.1.0"
-    fork-ts-checker-webpack-plugin "4.1.6"
-    global-modules "2.0.0"
-    globby "11.0.1"
-    gzip-size "5.1.1"
-    immer "8.0.1"
-    is-root "2.1.0"
-    loader-utils "2.0.0"
-    open "^7.0.2"
-    pkg-up "3.1.0"
-    prompts "2.4.0"
-    react-error-overlay "^6.0.9"
-    recursive-readdir "2.2.2"
-    shell-quote "1.7.2"
-    strip-ansi "6.0.0"
-    text-table "0.2.0"
+react-dev-utils@^12.0.1:
+  version "12.0.1"
+  resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-12.0.1.tgz#ba92edb4a1f379bd46ccd6bcd4e7bc398df33e73"
+  integrity sha512-84Ivxmr17KjUupyqzFode6xKhjwuEJDROWKJy/BthkL7Wn6NJ8h4WE6k/exAv6ImS+0oZLRRW5j/aINMHyeGeQ==
+  dependencies:
+    "@babel/code-frame" "^7.16.0"
+    address "^1.1.2"
+    browserslist "^4.18.1"
+    chalk "^4.1.2"
+    cross-spawn "^7.0.3"
+    detect-port-alt "^1.1.6"
+    escape-string-regexp "^4.0.0"
+    filesize "^8.0.6"
+    find-up "^5.0.0"
+    fork-ts-checker-webpack-plugin "^6.5.0"
+    global-modules "^2.0.0"
+    globby "^11.0.4"
+    gzip-size "^6.0.0"
+    immer "^9.0.7"
+    is-root "^2.1.0"
+    loader-utils "^3.2.0"
+    open "^8.4.0"
+    pkg-up "^3.1.0"
+    prompts "^2.4.2"
+    react-error-overlay "^6.0.11"
+    recursive-readdir "^2.2.2"
+    shell-quote "^1.7.3"
+    strip-ansi "^6.0.1"
+    text-table "^0.2.0"
 
 react-dom@^17.0.0:
   version "17.0.2"
@@ -16200,17 +14061,10 @@ react-dom@^17.0.0:
     object-assign "^4.1.1"
     scheduler "^0.20.2"
 
-react-email-share-link@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/react-email-share-link/-/react-email-share-link-1.0.3.tgz#745d09fa9bca8a1c3820f7a8bc43237bb4e5c05e"
-  integrity sha512-MNWNN4OHbrE3ia3l3a5EHds4dXbu9o7j1F4P+UkQ4y15AdkU/laMDD2EXE3onVRePtk2Et3TUHKyEleocUfN2g==
-  dependencies:
-    prop-types "^15.7.2"
-
-react-error-overlay@^6.0.9:
-  version "6.0.9"
-  resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.9.tgz#3c743010c9359608c375ecd6bc76f35d93995b0a"
-  integrity sha512-nQTTcUu+ATDbrSD1BZHr5kgSD4oF8OFjxun8uAaL8RwPBacGBNPf/yAuVVdx17N8XNzRDMrZ9XcKZHCjPW+9ew==
+react-error-overlay@^6.0.11:
+  version "6.0.11"
+  resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.11.tgz#92835de5841c5cf08ba00ddd2d677b6d17ff9adb"
+  integrity sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg==
 
 react-fast-compare@^3.2.0:
   version "3.2.1"
@@ -16265,7 +14119,7 @@ react-is@^17.0.0, react-is@^17.0.1:
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
   integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==
 
-react-is@^18.2.0:
+react-is@^18.0.0, react-is@^18.2.0:
   version "18.2.0"
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b"
   integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==
@@ -16294,10 +14148,10 @@ react-markdown@6.0.2:
     unist-util-visit "^2.0.0"
     vfile "^4.0.0"
 
-react-refresh@^0.8.3:
-  version "0.8.3"
-  resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.8.3.tgz#721d4657672d400c5e3c75d063c4a85fb2d5d68f"
-  integrity sha512-X8jZHc7nCMjaCqoU+V2I0cOhNW+QMBwSUkeXnTi8IPe6zaRWfn60ZzvFDZqWPfmSJfjub7dDW1SP0jaHWLu/hg==
+react-refresh@^0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.11.0.tgz#77198b944733f0f1f1a90e791de4541f9f074046"
+  integrity sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==
 
 react-router-dom@^5.1.6:
   version "5.2.0"
@@ -16328,71 +14182,60 @@ react-router@5.2.0, react-router@^5.2.0:
     tiny-invariant "^1.0.2"
     tiny-warning "^1.0.0"
 
-react-scripts@4.0.3:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-4.0.3.tgz#b1cafed7c3fa603e7628ba0f187787964cb5d345"
-  integrity sha512-S5eO4vjUzUisvkIPB7jVsKtuH2HhWcASREYWHAQ1FP5HyCv3xgn+wpILAEWkmy+A+tTNbSZClhxjT3qz6g4L1A==
-  dependencies:
-    "@babel/core" "7.12.3"
-    "@pmmmwh/react-refresh-webpack-plugin" "0.4.3"
-    "@svgr/webpack" "5.5.0"
-    "@typescript-eslint/eslint-plugin" "^4.5.0"
-    "@typescript-eslint/parser" "^4.5.0"
-    babel-eslint "^10.1.0"
-    babel-jest "^26.6.0"
-    babel-loader "8.1.0"
-    babel-plugin-named-asset-import "^0.3.7"
-    babel-preset-react-app "^10.0.0"
+react-scripts@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-5.0.1.tgz#6285dbd65a8ba6e49ca8d651ce30645a6d980003"
+  integrity sha512-8VAmEm/ZAwQzJ+GOMLbBsTdDKOpuZh7RPs0UymvBR2vRk4iZWCskjbFnxqjrzoIvlNNRZ3QJFx6/qDSi6zSnaQ==
+  dependencies:
+    "@babel/core" "^7.16.0"
+    "@pmmmwh/react-refresh-webpack-plugin" "^0.5.3"
+    "@svgr/webpack" "^5.5.0"
+    babel-jest "^27.4.2"
+    babel-loader "^8.2.3"
+    babel-plugin-named-asset-import "^0.3.8"
+    babel-preset-react-app "^10.0.1"
     bfj "^7.0.2"
-    camelcase "^6.1.0"
-    case-sensitive-paths-webpack-plugin "2.3.0"
-    css-loader "4.3.0"
-    dotenv "8.2.0"
-    dotenv-expand "5.1.0"
-    eslint "^7.11.0"
-    eslint-config-react-app "^6.0.0"
-    eslint-plugin-flowtype "^5.2.0"
-    eslint-plugin-import "^2.22.1"
-    eslint-plugin-jest "^24.1.0"
-    eslint-plugin-jsx-a11y "^6.3.1"
-    eslint-plugin-react "^7.21.5"
-    eslint-plugin-react-hooks "^4.2.0"
-    eslint-plugin-testing-library "^3.9.2"
-    eslint-webpack-plugin "^2.5.2"
-    file-loader "6.1.1"
-    fs-extra "^9.0.1"
-    html-webpack-plugin "4.5.0"
-    identity-obj-proxy "3.0.0"
-    jest "26.6.0"
-    jest-circus "26.6.0"
-    jest-resolve "26.6.0"
-    jest-watch-typeahead "0.6.1"
-    mini-css-extract-plugin "0.11.3"
-    optimize-css-assets-webpack-plugin "5.0.4"
-    pnp-webpack-plugin "1.6.4"
-    postcss-flexbugs-fixes "4.2.1"
-    postcss-loader "3.0.0"
-    postcss-normalize "8.0.1"
-    postcss-preset-env "6.7.0"
-    postcss-safe-parser "5.0.2"
-    prompts "2.4.0"
-    react-app-polyfill "^2.0.0"
-    react-dev-utils "^11.0.3"
-    react-refresh "^0.8.3"
-    resolve "1.18.1"
-    resolve-url-loader "^3.1.2"
-    sass-loader "^10.0.5"
-    semver "7.3.2"
-    style-loader "1.3.0"
-    terser-webpack-plugin "4.2.3"
-    ts-pnp "1.2.0"
-    url-loader "4.1.1"
-    webpack "4.44.2"
-    webpack-dev-server "3.11.1"
-    webpack-manifest-plugin "2.2.0"
-    workbox-webpack-plugin "5.1.4"
+    browserslist "^4.18.1"
+    camelcase "^6.2.1"
+    case-sensitive-paths-webpack-plugin "^2.4.0"
+    css-loader "^6.5.1"
+    css-minimizer-webpack-plugin "^3.2.0"
+    dotenv "^10.0.0"
+    dotenv-expand "^5.1.0"
+    eslint "^8.3.0"
+    eslint-config-react-app "^7.0.1"
+    eslint-webpack-plugin "^3.1.1"
+    file-loader "^6.2.0"
+    fs-extra "^10.0.0"
+    html-webpack-plugin "^5.5.0"
+    identity-obj-proxy "^3.0.0"
+    jest "^27.4.3"
+    jest-resolve "^27.4.2"
+    jest-watch-typeahead "^1.0.0"
+    mini-css-extract-plugin "^2.4.5"
+    postcss "^8.4.4"
+    postcss-flexbugs-fixes "^5.0.2"
+    postcss-loader "^6.2.1"
+    postcss-normalize "^10.0.1"
+    postcss-preset-env "^7.0.1"
+    prompts "^2.4.2"
+    react-app-polyfill "^3.0.0"
+    react-dev-utils "^12.0.1"
+    react-refresh "^0.11.0"
+    resolve "^1.20.0"
+    resolve-url-loader "^4.0.0"
+    sass-loader "^12.3.0"
+    semver "^7.3.5"
+    source-map-loader "^3.0.0"
+    style-loader "^3.3.1"
+    tailwindcss "^3.0.2"
+    terser-webpack-plugin "^5.2.5"
+    webpack "^5.64.4"
+    webpack-dev-server "^4.6.0"
+    webpack-manifest-plugin "^4.0.2"
+    workbox-webpack-plugin "^6.4.1"
   optionalDependencies:
-    fsevents "^2.1.3"
+    fsevents "^2.3.2"
 
 react-syntax-highlighter@^15.4.4:
   version "15.4.4"
@@ -16420,12 +14263,12 @@ react-universal-interface@^0.6.2:
   resolved "https://registry.yarnpkg.com/react-universal-interface/-/react-universal-interface-0.6.2.tgz#5e8d438a01729a4dbbcbeeceb0b86be146fe2b3b"
   integrity sha512-dg8yXdcQmvgR13RIlZbTRQOoUrDciFVoSBZILwjE2LFISxZZ8loVJKAkuzswl5js8BHda79bIb2b84ehU8IjXw==
 
-react-use-measure@2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/react-use-measure/-/react-use-measure-2.0.1.tgz#4f23f94c832cd4512da55acb300d1915dcbf3ae8"
-  integrity sha512-lFfHiqcXbJ2/6aUkZwt8g5YYM7EGqNVxJhMqMPqv1BVXRKp8D7jYLlmma0SvhRY4WYxxkZpCdbJvhDylb5gcEA==
+react-use-measure@^2.0.4:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/react-use-measure/-/react-use-measure-2.1.1.tgz#5824537f4ee01c9469c45d5f7a8446177c6cc4ba"
+  integrity sha512-nocZhN26cproIiIduswYpV5y5lQpSQS1y/4KuvUCjSKmw7ZWIS/+g3aFnX3WdBkyuGUtTLif3UTqnLLhbDoQig==
   dependencies:
-    debounce "^1.2.0"
+    debounce "^1.2.1"
 
 react-use@^17.3.2:
   version "17.4.0"
@@ -16482,45 +14325,16 @@ reactour@1.18.7:
     prop-types "15.7.2"
     react-focus-lock "2.5.2"
     scroll-smooth "1.1.1"
-    scrollparent "2.0.1"
-
-read-pkg-up@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-3.0.0.tgz#3ed496685dba0f8fe118d0691dc51f4a1ff96f07"
-  integrity sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=
-  dependencies:
-    find-up "^2.0.0"
-    read-pkg "^3.0.0"
-
-read-pkg-up@^7.0.1:
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507"
-  integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==
-  dependencies:
-    find-up "^4.1.0"
-    read-pkg "^5.2.0"
-    type-fest "^0.8.1"
-
-read-pkg@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-3.0.0.tgz#9cbc686978fee65d16c00e2b19c237fcf6e38389"
-  integrity sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=
-  dependencies:
-    load-json-file "^4.0.0"
-    normalize-package-data "^2.3.2"
-    path-type "^3.0.0"
+    scrollparent "2.0.1"
 
-read-pkg@^5.2.0:
-  version "5.2.0"
-  resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc"
-  integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==
+read-cache@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/read-cache/-/read-cache-1.0.0.tgz#e664ef31161166c9751cdbe8dbcf86b5fb58f774"
+  integrity sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==
   dependencies:
-    "@types/normalize-package-data" "^2.4.0"
-    normalize-package-data "^2.5.0"
-    parse-json "^5.0.0"
-    type-fest "^0.6.0"
+    pify "^2.3.0"
 
-"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6:
+readable-stream@^2.0.1:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -16533,7 +14347,7 @@ read-pkg@^5.2.0:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.6.0:
+readable-stream@^3.0.6:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -16542,28 +14356,19 @@ readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.6.0:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
-readdirp@^2.2.1:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525"
-  integrity sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==
-  dependencies:
-    graceful-fs "^4.1.11"
-    micromatch "^3.1.10"
-    readable-stream "^2.0.2"
-
-readdirp@~3.5.0:
-  version "3.5.0"
-  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.5.0.tgz#9ba74c019b15d365278d2e91bb8c48d7b4d42c9e"
-  integrity sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==
+readdirp@~3.6.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7"
+  integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==
   dependencies:
     picomatch "^2.2.1"
 
-recursive-readdir@2.2.2:
-  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.2.tgz#9946fb3274e1628de6e36b2f6714953b4845094f"
-  integrity sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==
+recursive-readdir@^2.2.2:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.3.tgz#e726f328c0d69153bcabd5c322d3195252379372"
+  integrity sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==
   dependencies:
-    minimatch "3.0.4"
+    minimatch "^3.0.5"
 
 redent@^3.0.0:
   version "3.0.0"
@@ -16598,42 +14403,27 @@ refractor@3.3.1, refractor@^3.2.0, refractor@^3.3.1:
     parse-entities "^2.0.0"
     prismjs "~1.23.0"
 
-regenerate-unicode-properties@^8.2.0:
-  version "8.2.0"
-  resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec"
-  integrity sha512-F9DjY1vKLo/tPePDycuH3dn9H1OTPIkVD9Kz4LODu+F2C75mgjAJ7x/gwy6ZcSNRAAkhNlJSOHRe8k3p+K9WhA==
+regenerate-unicode-properties@^10.1.0:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.0.tgz#7c3192cab6dd24e21cb4461e5ddd7dd24fa8374c"
+  integrity sha512-d1VudCLoIGitcU/hEg2QqvyGZQmdC0Lf8BqdOMXGFSvJP4bNV1+XqbPQeHHLD51Jh4QJJ225dlIFvY4Ly6MXmQ==
   dependencies:
-    regenerate "^1.4.0"
+    regenerate "^1.4.2"
 
-regenerate@^1.4.0:
+regenerate@^1.4.2:
   version "1.4.2"
   resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a"
   integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==
 
-regenerator-runtime@^0.11.0:
-  version "0.11.1"
-  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9"
-  integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==
-
-regenerator-runtime@^0.13.10:
-  version "0.13.10"
-  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz#ed07b19616bcbec5da6274ebc75ae95634bfc2ee"
-  integrity sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==
-
-regenerator-runtime@^0.13.11:
+regenerator-runtime@^0.13.11, regenerator-runtime@^0.13.4, regenerator-runtime@^0.13.9:
   version "0.13.11"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9"
   integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==
 
-regenerator-runtime@^0.13.4, regenerator-runtime@^0.13.7:
-  version "0.13.7"
-  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55"
-  integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==
-
-regenerator-transform@^0.14.2:
-  version "0.14.5"
-  resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.14.5.tgz#c98da154683671c9c4dcb16ece736517e1b7feb4"
-  integrity sha512-eOf6vka5IO151Jfsw2NO9WpGX58W6wWmefK3I1zEGr0lOD0u8rwPaNqQL1aRxUaxLeKO3ArNh3VYg1KbaD+FFw==
+regenerator-transform@^0.15.1:
+  version "0.15.1"
+  resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.15.1.tgz#f6c4e99fc1b4591f780db2586328e4d9a9d8dc56"
+  integrity sha512-knzmNAcuyxV+gQCufkYcvOqX/qIIfHLv0u5x79kRxuGojfYVky1f15TzZEu2Avte8QGepvUNTnLskf8E6X6Vyg==
   dependencies:
     "@babel/runtime" "^7.8.4"
 
@@ -16650,14 +14440,6 @@ regex-parser@^2.2.11:
   resolved "https://registry.yarnpkg.com/regex-parser/-/regex-parser-2.2.11.tgz#3b37ec9049e19479806e878cabe7c1ca83ccfe58"
   integrity sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==
 
-regexp.prototype.flags@^1.2.0, regexp.prototype.flags@^1.3.1:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz#7ef352ae8d159e758c0eadca6f8fcb4eef07be26"
-  integrity sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==
-  dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.3"
-
 regexp.prototype.flags@^1.4.3:
   version "1.4.3"
   resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz#87cab30f80f66660181a3bb7bf5981a872b367ac"
@@ -16667,22 +14449,17 @@ regexp.prototype.flags@^1.4.3:
     define-properties "^1.1.3"
     functions-have-names "^1.2.2"
 
-regexpp@^3.0.0, regexpp@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2"
-  integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==
-
-regexpu-core@^4.7.1:
-  version "4.7.1"
-  resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.1.tgz#2dea5a9a07233298fbf0db91fa9abc4c6e0f8ad6"
-  integrity sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ==
+regexpu-core@^5.3.1:
+  version "5.3.2"
+  resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.2.tgz#11a2b06884f3527aec3e93dbbf4a3b958a95546b"
+  integrity sha512-RAM5FlZz+Lhmo7db9L298p2vHP5ZywrVXmVXpmAD9GuL5MPH6t9ROw1iA/wfHkQ76Qe7AaPF0nGuim96/IrQMQ==
   dependencies:
-    regenerate "^1.4.0"
-    regenerate-unicode-properties "^8.2.0"
-    regjsgen "^0.5.1"
-    regjsparser "^0.6.4"
-    unicode-match-property-ecmascript "^1.0.4"
-    unicode-match-property-value-ecmascript "^1.2.0"
+    "@babel/regjsgen" "^0.8.0"
+    regenerate "^1.4.2"
+    regenerate-unicode-properties "^10.1.0"
+    regjsparser "^0.9.1"
+    unicode-match-property-ecmascript "^2.0.0"
+    unicode-match-property-value-ecmascript "^2.1.0"
 
 registry-auth-token@^4.0.0:
   version "4.2.1"
@@ -16698,15 +14475,10 @@ registry-url@^5.0.0:
   dependencies:
     rc "^1.2.8"
 
-regjsgen@^0.5.1:
-  version "0.5.2"
-  resolved "https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.5.2.tgz#92ff295fb1deecbf6ecdab2543d207e91aa33733"
-  integrity sha512-OFFT3MfrH90xIW8OOSyUrk6QHD5E9JOTeGodiJeBS3J6IwlgzJMNE/1bZklWz5oTg+9dCMyEetclvCVXOPoN3A==
-
-regjsparser@^0.6.4:
-  version "0.6.9"
-  resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.9.tgz#b489eef7c9a2ce43727627011429cf833a7183e6"
-  integrity sha512-ZqbNRz1SNjLAiYuwY0zoXW8Ne675IX5q+YHioAGbCw4X96Mjl2+dcX9B2ciaeyYjViDAfvIjFpQjJgLttTEERQ==
+regjsparser@^0.9.1:
+  version "0.9.1"
+  resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.9.1.tgz#272d05aa10c7c1f67095b1ff0addae8442fc5709"
+  integrity sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ==
   dependencies:
     jsesc "~0.5.0"
 
@@ -16870,16 +14642,16 @@ remove-trailing-spaces@^1.0.6:
   resolved "https://registry.yarnpkg.com/remove-trailing-spaces/-/remove-trailing-spaces-1.0.8.tgz#4354d22f3236374702f58ee373168f6d6887ada7"
   integrity sha512-O3vsMYfWighyFbTd8hk8VaSj9UAGENxAtX+//ugIst2RMk5e03h6RoIS+0ylsFxY1gvmPuAY/PO4It+gPEeySA==
 
-renderkid@^2.0.4:
-  version "2.0.5"
-  resolved "https://registry.yarnpkg.com/renderkid/-/renderkid-2.0.5.tgz#483b1ac59c6601ab30a7a596a5965cabccfdd0a5"
-  integrity sha512-ccqoLg+HLOHq1vdfYNm4TBeaCDIi1FLt3wGojTDSvdewUv65oTmI3cnT2E4hRjl1gzKZIPK+KZrXzlUYKnR+vQ==
+renderkid@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/renderkid/-/renderkid-3.0.0.tgz#5fd823e4d6951d37358ecc9a58b1f06836b6268a"
+  integrity sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==
   dependencies:
-    css-select "^2.0.2"
-    dom-converter "^0.2"
-    htmlparser2 "^3.10.1"
-    lodash "^4.17.20"
-    strip-ansi "^3.0.0"
+    css-select "^4.1.3"
+    dom-converter "^0.2.0"
+    htmlparser2 "^6.1.0"
+    lodash "^4.17.21"
+    strip-ansi "^6.0.1"
 
 repeat-element@^1.1.2:
   version "1.1.4"
@@ -16937,33 +14709,16 @@ require-main-filename@^2.0.0:
   resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b"
   integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==
 
-require-package-name@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/require-package-name/-/require-package-name-2.0.1.tgz#c11e97276b65b8e2923f75dabf5fb2ef0c3841b9"
-  integrity sha1-wR6XJ2tluOKSP3Xav1+y7ww4Qbk=
-
 requires-port@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
   integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=
 
-resize-observer-polyfill@1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.0.tgz#660ff1d9712a2382baa2cad450a4716209f9ca69"
-  integrity sha512-M2AelyJDVR/oLnToJLtuDJRBBWUGUvvGigj1411hXhAdyFWqMaqHp7TixW3FpiLuVaikIcR1QL+zqoJoZlOgpg==
-
 resize-observer-polyfill@^1.5.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464"
   integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==
 
-resolve-cwd@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a"
-  integrity sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=
-  dependencies:
-    resolve-from "^3.0.0"
-
 resolve-cwd@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d"
@@ -16976,11 +14731,6 @@ resolve-from@5.0.0, resolve-from@^5.0.0:
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69"
   integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==
 
-resolve-from@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-3.0.0.tgz#b22c7af7d9d6881bc8b6e653335eebcb0a188748"
-  integrity sha1-six699nWiBvItuZTM17rywoYh0g=
-
 resolve-from@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
@@ -16991,20 +14741,15 @@ resolve-pathname@^3.0.0:
   resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-3.0.0.tgz#99d02224d3cf263689becbb393bc560313025dcd"
   integrity sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==
 
-resolve-url-loader@^3.1.2:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/resolve-url-loader/-/resolve-url-loader-3.1.3.tgz#49ec68340f67d8d2ab6b401948d5def3ab2d0367"
-  integrity sha512-WbDSNFiKPPLem1ln+EVTE+bFUBdTTytfQZWbmghroaFNFaAVmGq0Saqw6F/306CwgPXsGwXVxbODE+3xAo/YbA==
-  dependencies:
-    adjust-sourcemap-loader "3.0.0"
-    camelcase "5.3.1"
-    compose-function "3.0.3"
-    convert-source-map "1.7.0"
-    es6-iterator "2.0.3"
-    loader-utils "1.2.3"
-    postcss "7.0.21"
-    rework "1.0.1"
-    rework-visit "1.0.0"
+resolve-url-loader@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/resolve-url-loader/-/resolve-url-loader-4.0.0.tgz#d50d4ddc746bb10468443167acf800dcd6c3ad57"
+  integrity sha512-05VEMczVREcbtT7Bz+C+96eUO5HDNvdthIiMB34t7FcF8ehcu4wC0sSgPUubs3XW2Q3CNLJk/BJrCU9wVRymiA==
+  dependencies:
+    adjust-sourcemap-loader "^4.0.0"
+    convert-source-map "^1.7.0"
+    loader-utils "^2.0.0"
+    postcss "^7.0.35"
     source-map "0.6.1"
 
 resolve-url@^0.2.1:
@@ -17012,39 +14757,29 @@ resolve-url@^0.2.1:
   resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"
   integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=
 
-resolve@1.18.1:
-  version "1.18.1"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130"
-  integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==
-  dependencies:
-    is-core-module "^2.0.0"
-    path-parse "^1.0.6"
+resolve.exports@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-1.1.1.tgz#05cfd5b3edf641571fd46fa608b610dda9ead999"
+  integrity sha512-/NtpHNDN7jWhAaQ9BvBUYZ6YTXsRBgfqWFWP7BZBaoMJO/I3G5OFzvTuWNlZC3aPjins1F+TNrLKsGbH4rfsRQ==
 
-resolve@^1.10.0, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.14.2, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.20.0, resolve@^1.3.2, resolve@^1.8.1:
-  version "1.20.0"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975"
-  integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==
+resolve@^1.1.7, resolve@^1.12.0, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22.1, resolve@^1.22.2:
+  version "1.22.2"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.2.tgz#0ed0943d4e301867955766c9f3e1ae6d01c6845f"
+  integrity sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==
   dependencies:
-    is-core-module "^2.2.0"
-    path-parse "^1.0.6"
+    is-core-module "^2.11.0"
+    path-parse "^1.0.7"
+    supports-preserve-symlinks-flag "^1.0.0"
 
-resolve@^1.19.0:
-  version "1.22.1"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177"
-  integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==
+resolve@^2.0.0-next.4:
+  version "2.0.0-next.4"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660"
+  integrity sha512-iMDbmAWtfU+MHpxt/I5iWI7cY6YVEZUQ3MBgPQ++XD1PELuJHIl82xBmObyP2KyQmkNB2dsqF7seoQQiAn5yDQ==
   dependencies:
     is-core-module "^2.9.0"
     path-parse "^1.0.7"
     supports-preserve-symlinks-flag "^1.0.0"
 
-resolve@^2.0.0-next.3:
-  version "2.0.0-next.3"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.3.tgz#d41016293d4a8586a39ca5d9b5f15cbea1f55e46"
-  integrity sha512-W8LucSynKUIDu9ylraa7ueVZ7hc0uAgJBxVsQSKOXOyle8a93qXhcz+XAXZ8bIq2d6i4Ehddn6Evt+0/UwKk6Q==
-  dependencies:
-    is-core-module "^2.2.0"
-    path-parse "^1.0.6"
-
 responselike@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7"
@@ -17073,46 +14808,16 @@ ret@~0.1.10:
   resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
   integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==
 
-retry@^0.12.0:
-  version "0.12.0"
-  resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b"
-  integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=
+retry@^0.13.1:
+  version "0.13.1"
+  resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
+  integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==
 
 reusify@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
   integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
 
-rework-visit@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/rework-visit/-/rework-visit-1.0.0.tgz#9945b2803f219e2f7aca00adb8bc9f640f842c9a"
-  integrity sha1-mUWygD8hni96ygCtuLyfZA+ELJo=
-
-rework@1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/rework/-/rework-1.0.1.tgz#30806a841342b54510aa4110850cd48534144aa7"
-  integrity sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=
-  dependencies:
-    convert-source-map "^0.3.3"
-    css "^2.0.0"
-
-rgb-regex@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/rgb-regex/-/rgb-regex-1.0.1.tgz#c0e0d6882df0e23be254a475e8edd41915feaeb1"
-  integrity sha1-wODWiC3w4jviVKR16O3UGRX+rrE=
-
-rgba-regex@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3"
-  integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM=
-
-rimraf@^2.5.4, rimraf@^2.6.3:
-  version "2.7.1"
-  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
-  integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
-  dependencies:
-    glob "^7.1.3"
-
 rimraf@^3.0.0, rimraf@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
@@ -17127,48 +14832,22 @@ rimraf@~2.6.2:
   dependencies:
     glob "^7.1.3"
 
-ripemd160@^2.0.0, ripemd160@^2.0.1:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c"
-  integrity sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==
-  dependencies:
-    hash-base "^3.0.0"
-    inherits "^2.0.1"
-
-rollup-plugin-babel@^4.3.3:
-  version "4.4.0"
-  resolved "https://registry.yarnpkg.com/rollup-plugin-babel/-/rollup-plugin-babel-4.4.0.tgz#d15bd259466a9d1accbdb2fe2fff17c52d030acb"
-  integrity sha512-Lek/TYp1+7g7I+uMfJnnSJ7YWoD58ajo6Oarhlex7lvUce+RCKRuGRSgztDO3/MF/PuGKmUL5iTHKf208UNszw==
-  dependencies:
-    "@babel/helper-module-imports" "^7.0.0"
-    rollup-pluginutils "^2.8.1"
-
-rollup-plugin-terser@^5.3.1:
-  version "5.3.1"
-  resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-5.3.1.tgz#8c650062c22a8426c64268548957463bf981b413"
-  integrity sha512-1pkwkervMJQGFYvM9nscrUoncPwiKR/K+bHdjv6PFgRo3cgPHoRT83y2Aa3GvINj4539S15t/tpFPb775TDs6w==
+rollup-plugin-terser@^7.0.0:
+  version "7.0.2"
+  resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz#e8fbba4869981b2dc35ae7e8a502d5c6c04d324d"
+  integrity sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ==
   dependencies:
-    "@babel/code-frame" "^7.5.5"
-    jest-worker "^24.9.0"
-    rollup-pluginutils "^2.8.2"
+    "@babel/code-frame" "^7.10.4"
+    jest-worker "^26.2.1"
     serialize-javascript "^4.0.0"
-    terser "^4.6.2"
-
-rollup-pluginutils@^2.8.1, rollup-pluginutils@^2.8.2:
-  version "2.8.2"
-  resolved "https://registry.yarnpkg.com/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e"
-  integrity sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==
-  dependencies:
-    estree-walker "^0.6.1"
+    terser "^5.0.0"
 
-rollup@^1.31.1:
-  version "1.32.1"
-  resolved "https://registry.yarnpkg.com/rollup/-/rollup-1.32.1.tgz#4480e52d9d9e2ae4b46ba0d9ddeaf3163940f9c4"
-  integrity sha512-/2HA0Ec70TvQnXdzynFffkjA6XN+1e2pEv/uKS5Ulca40g2L7KuOE3riasHoNVHOsFD5KKZgDsMk1CP3Tw9s+A==
-  dependencies:
-    "@types/estree" "*"
-    "@types/node" "*"
-    acorn "^7.1.0"
+rollup@^2.43.1:
+  version "2.79.1"
+  resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7"
+  integrity sha512-uKxbd0IhMZOhjAiD5oAFp7BqvkA4Dv47qpOCtaNvng4HBwdbWtdOh8f5nZNuk2rp51PMGk3bzfWu5oayNEuYnw==
+  optionalDependencies:
+    fsevents "~2.3.2"
 
 rope-sequence@^1.3.0:
   version "1.3.3"
@@ -17196,11 +14875,6 @@ route-recognizer@^0.3.3:
   resolved "https://registry.npmjs.org/route-recognizer/-/route-recognizer-0.3.4.tgz#39ab1ffbce1c59e6d2bdca416f0932611e4f3ca3"
   integrity sha512-2+MhsfPhvauN1O8KaXpXAOfR/fwe8dnUXVM+xw7yt40lJRfPVQxV6yryZm0cgRvAj5fMF/mdRZbL2ptwbs5i2g==
 
-rsvp@^4.8.4:
-  version "4.8.5"
-  resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734"
-  integrity sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==
-
 rtl-css-js@^1.14.0:
   version "1.16.0"
   resolved "https://registry.yarnpkg.com/rtl-css-js/-/rtl-css-js-1.16.0.tgz#e8d682982441aadb63cabcb2f7385f3fb78ff26e"
@@ -17220,13 +14894,6 @@ run-parallel@^1.1.9:
   dependencies:
     queue-microtask "^1.2.2"
 
-run-queue@^1.0.0, run-queue@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/run-queue/-/run-queue-1.0.3.tgz#e848396f057d223f24386924618e25694161ec47"
-  integrity sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=
-  dependencies:
-    aproba "^1.1.1"
-
 rxjs@^6.3.3, rxjs@^6.6.0, rxjs@^6.6.3:
   version "6.6.7"
   resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9"
@@ -17244,7 +14911,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
-safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0:
+safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@@ -17265,43 +14932,25 @@ safe-regex@^1.1.0:
   dependencies:
     ret "~0.1.10"
 
-"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
+"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
 
-sane@^4.0.3:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/sane/-/sane-4.1.0.tgz#ed881fd922733a6c461bc189dc2b6c006f3ffded"
-  integrity sha512-hhbzAgTIX8O7SHfp2c8/kREfEn4qO/9q8C9beyY6+tvZ87EpoZ3i1RIEvp27YBswnNbY9mWd6paKVmKbAgLfZA==
-  dependencies:
-    "@cnakazawa/watch" "^1.0.3"
-    anymatch "^2.0.0"
-    capture-exit "^2.0.0"
-    exec-sh "^0.3.2"
-    execa "^1.0.0"
-    fb-watchman "^2.0.0"
-    micromatch "^3.1.4"
-    minimist "^1.1.1"
-    walker "~1.0.5"
-
-sanitize.css@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/sanitize.css/-/sanitize.css-10.0.0.tgz#b5cb2547e96d8629a60947544665243b1dc3657a"
-  integrity sha512-vTxrZz4dX5W86M6oVWVdOVe72ZiPs41Oi7Z6Km4W5Turyz28mrXSJhhEBZoRtzJWIv3833WKVwLSDWWkEfupMg==
+sanitize.css@*:
+  version "13.0.0"
+  resolved "https://registry.yarnpkg.com/sanitize.css/-/sanitize.css-13.0.0.tgz#2675553974b27964c75562ade3bd85d79879f173"
+  integrity sha512-ZRwKbh/eQ6w9vmTjkuG0Ioi3HBwPFce0O+v//ve+aOq1oeCy7jMV2qzzAlpsNuqpqCBjjriM1lbtZbF/Q8jVyA==
 
-sass-loader@^10.0.5:
-  version "10.2.0"
-  resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-10.2.0.tgz#3d64c1590f911013b3fa48a0b22a83d5e1494716"
-  integrity sha512-kUceLzC1gIHz0zNJPpqRsJyisWatGYNFRmv2CKZK2/ngMJgLqxTbXwe/hJ85luyvZkgqU3VlJ33UVF2T/0g6mw==
+sass-loader@^12.3.0:
+  version "12.6.0"
+  resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-12.6.0.tgz#5148362c8e2cdd4b950f3c63ac5d16dbfed37bcb"
+  integrity sha512-oLTaH0YCtX4cfnJZxKSLAyglED0naiYfNG1iXfU5w1LNZ+ukoA5DtyDIN5zmKVZwYNJP4KRc5Y3hkWga+7tYfA==
   dependencies:
     klona "^2.0.4"
-    loader-utils "^2.0.0"
     neo-async "^2.6.2"
-    schema-utils "^3.0.0"
-    semver "^7.3.2"
 
-sax@~1.2.4:
+sax@^1.2.4, sax@~1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -17321,16 +14970,16 @@ scheduler@^0.20.2:
     loose-envify "^1.1.0"
     object-assign "^4.1.1"
 
-schema-utils@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770"
-  integrity sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==
+schema-utils@2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.0.tgz#17151f76d8eae67fbbf77960c33c676ad9f4efc7"
+  integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A==
   dependencies:
-    ajv "^6.1.0"
-    ajv-errors "^1.0.0"
-    ajv-keywords "^3.1.0"
+    "@types/json-schema" "^7.0.4"
+    ajv "^6.12.2"
+    ajv-keywords "^3.4.1"
 
-schema-utils@^2.6.5, schema-utils@^2.7.0, schema-utils@^2.7.1:
+schema-utils@^2.6.5:
   version "2.7.1"
   resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.1.tgz#1ca4f32d1b24c590c203b8e7a50bf0ea4cd394d7"
   integrity sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==
@@ -17339,15 +14988,25 @@ schema-utils@^2.6.5, schema-utils@^2.7.0, schema-utils@^2.7.1:
     ajv "^6.12.4"
     ajv-keywords "^3.5.2"
 
-schema-utils@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef"
-  integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==
+schema-utils@^3.0.0, schema-utils@^3.1.1, schema-utils@^3.2.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe"
+  integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==
   dependencies:
-    "@types/json-schema" "^7.0.6"
+    "@types/json-schema" "^7.0.8"
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
+schema-utils@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-4.2.0.tgz#70d7c93e153a273a805801882ebd3bff20d89c8b"
+  integrity sha512-L0jRsrPpjdckP3oPug3/VxNKt2trR8TcabrM6FOAAlvC/9Phcmm+cuAgTlxBqdBR1WJx7Naj9WHw+aOmheSVbw==
+  dependencies:
+    "@types/json-schema" "^7.0.9"
+    ajv "^8.9.0"
+    ajv-formats "^2.1.1"
+    ajv-keywords "^5.1.0"
+
 screenfull@^5.1.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/screenfull/-/screenfull-5.2.0.tgz#6533d524d30621fc1283b9692146f3f13a93d1ba"
@@ -17385,37 +15044,27 @@ select@^1.1.2:
   resolved "https://registry.yarnpkg.com/select/-/select-1.1.2.tgz#0e7350acdec80b1108528786ec1d4418d11b396d"
   integrity sha1-DnNQrN7ICxEIUoeG7B1EGNEbOW0=
 
-selfsigned@^1.10.8:
-  version "1.10.11"
-  resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.11.tgz#24929cd906fe0f44b6d01fb23999a739537acbe9"
-  integrity sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA==
+selfsigned@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-2.1.1.tgz#18a7613d714c0cd3385c48af0075abf3f266af61"
+  integrity sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ==
   dependencies:
-    node-forge "^0.10.0"
+    node-forge "^1"
 
-"semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.5.1, semver@^5.6.0:
+semver@^5.6.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
 
-semver@7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"
-  integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==
-
-semver@7.3.2:
-  version "7.3.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
-  integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
-
 semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
   integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
 
-semver@^7.2.1, semver@^7.3.2:
-  version "7.3.5"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7"
-  integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==
+semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8:
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
+  integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
@@ -17454,10 +15103,10 @@ serialize-javascript@^4.0.0:
   dependencies:
     randombytes "^2.1.0"
 
-serialize-javascript@^5.0.1:
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-5.0.1.tgz#7886ec848049a462467a97d3d918ebb2aaf934f4"
-  integrity sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==
+serialize-javascript@^6.0.0, serialize-javascript@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.1.tgz#b206efb27c3da0b0ab6b52f48d170b7996458e5c"
+  integrity sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w==
   dependencies:
     randombytes "^2.1.0"
 
@@ -17504,7 +15153,7 @@ set-value@^2.0.0, set-value@^2.0.1:
     is-plain-object "^2.0.3"
     split-string "^3.0.1"
 
-setimmediate@^1.0.4, setimmediate@^1.0.5:
+setimmediate@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
   integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=
@@ -17519,26 +15168,18 @@ setprototypeof@1.2.0:
   resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424"
   integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==
 
-sha.js@^2.4.0, sha.js@^2.4.8:
-  version "2.4.11"
-  resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7"
-  integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==
+shallow-clone@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3"
+  integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==
   dependencies:
-    inherits "^2.0.1"
-    safe-buffer "^5.0.1"
+    kind-of "^6.0.2"
 
 shallowequal@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/shallowequal/-/shallowequal-1.1.0.tgz#188d521de95b9087404fd4dcb68b13df0ae4e7f8"
   integrity sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==
 
-shebang-command@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea"
-  integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=
-  dependencies:
-    shebang-regex "^1.0.0"
-
 shebang-command@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
@@ -17546,25 +15187,15 @@ shebang-command@^2.0.0:
   dependencies:
     shebang-regex "^3.0.0"
 
-shebang-regex@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
-  integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=
-
 shebang-regex@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
   integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
 
-shell-quote@1.7.2:
-  version "1.7.2"
-  resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2"
-  integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==
-
-shellwords@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"
-  integrity sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==
+shell-quote@^1.7.3:
+  version "1.8.1"
+  resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680"
+  integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==
 
 side-channel@^1.0.4:
   version "1.0.4"
@@ -17575,7 +15206,7 @@ side-channel@^1.0.4:
     get-intrinsic "^1.0.2"
     object-inspect "^1.9.0"
 
-signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3:
+signal-exit@^3.0.2, signal-exit@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c"
   integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==
@@ -17585,13 +15216,6 @@ signedsource@^1.0.0:
   resolved "https://registry.yarnpkg.com/signedsource/-/signedsource-1.0.0.tgz#1ddace4981798f93bd833973803d80d52e93ad6a"
   integrity sha512-6+eerH9fEnNmi/hyM1DXcRK3pWdoMQtlkQ+ns0ntzunjKqp5i3sKCc80ym8Fib3iaYhdJUOPdhlJWj1tvge2Ww==
 
-simple-swizzle@^0.2.2:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a"
-  integrity sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=
-  dependencies:
-    is-arrayish "^0.3.1"
-
 sinon@^11.1.1:
   version "11.1.1"
   resolved "https://registry.yarnpkg.com/sinon/-/sinon-11.1.1.tgz#99a295a8b6f0fadbbb7e004076f3ae54fc6eab91"
@@ -17614,20 +15238,16 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
+slash@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7"
+  integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==
+
 slice-ansi@0.0.4:
   version "0.0.4"
   resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35"
   integrity sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU=
 
-slice-ansi@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b"
-  integrity sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==
-  dependencies:
-    ansi-styles "^4.0.0"
-    astral-regex "^2.0.0"
-    is-fullwidth-code-point "^3.0.0"
-
 snake-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/snake-case/-/snake-case-3.0.4.tgz#4f2bbd568e9935abdfd593f34c691dadb49c452c"
@@ -17666,35 +15286,16 @@ snapdragon@^0.8.1:
     source-map-resolve "^0.5.0"
     use "^3.1.0"
 
-sockjs-client@^1.5.0:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.5.1.tgz#256908f6d5adfb94dabbdbd02c66362cca0f9ea6"
-  integrity sha512-VnVAb663fosipI/m6pqRXakEOw7nvd7TUgdr3PlR/8V2I95QIdwT8L4nMxhyU8SmDBHYXU1TOElaKOmKLfYzeQ==
-  dependencies:
-    debug "^3.2.6"
-    eventsource "^1.0.7"
-    faye-websocket "^0.11.3"
-    inherits "^2.0.4"
-    json3 "^3.3.3"
-    url-parse "^1.5.1"
-
-sockjs@^0.3.21:
-  version "0.3.21"
-  resolved "https://registry.yarnpkg.com/sockjs/-/sockjs-0.3.21.tgz#b34ffb98e796930b60a0cfa11904d6a339a7d417"
-  integrity sha512-DhbPFGpxjc6Z3I+uX07Id5ZO2XwYsWOrYjaSeieES78cq+JaJvVe5q/m1uvjIQhXinhIeCFRH6JgXe+mvVMyXw==
+sockjs@^0.3.24:
+  version "0.3.24"
+  resolved "https://registry.yarnpkg.com/sockjs/-/sockjs-0.3.24.tgz#c9bc8995f33a111bea0395ec30aa3206bdb5ccce"
+  integrity sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==
   dependencies:
     faye-websocket "^0.11.3"
-    uuid "^3.4.0"
+    uuid "^8.3.2"
     websocket-driver "^0.7.4"
 
-sort-keys@^1.0.0:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-1.1.2.tgz#441b6d4d346798f1b4e49e8920adfba0e543f9ad"
-  integrity sha1-RBttTTRnmPG05J6JIK37oOVD+a0=
-  dependencies:
-    is-plain-obj "^1.0.0"
-
-source-list-map@^2.0.0:
+source-list-map@^2.0.0, source-list-map@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34"
   integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==
@@ -17717,12 +15318,21 @@ source-map-explorer@^2.5.2:
     temp "^0.9.4"
     yargs "^16.2.0"
 
-source-map-js@^0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-0.6.2.tgz#0bb5de631b41cfbda6cfba8bd05a80efdfd2385e"
-  integrity sha512-/3GptzWzu0+0MBQFrDKzw/DvvMTUORvgY6k6jd/VS6iCR4RDTKWH6v6WPwQoUO8667uQEf9Oe38DxAYWY5F/Ug==
+source-map-js@^1.0.1, source-map-js@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
+  integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
+
+source-map-loader@^3.0.0:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/source-map-loader/-/source-map-loader-3.0.2.tgz#af23192f9b344daa729f6772933194cc5fa54fee"
+  integrity sha512-BokxPoLjyl3iOrgkWaakaxqnelAJSS+0V+De0kKIq6lyWrXuiPgYTGp6z3iHmqljKAaLXwZa+ctD8GccRJeVvg==
+  dependencies:
+    abab "^2.0.5"
+    iconv-lite "^0.6.3"
+    source-map-js "^1.0.1"
 
-source-map-resolve@^0.5.0, source-map-resolve@^0.5.2:
+source-map-resolve@^0.5.0:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a"
   integrity sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==
@@ -17741,7 +15351,7 @@ source-map-resolve@^0.6.0:
     atob "^2.1.2"
     decode-uri-component "^0.2.0"
 
-source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.12, source-map-support@~0.5.19:
+source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.20:
   version "0.5.21"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
   integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==
@@ -17764,16 +15374,23 @@ source-map@0.6.1, source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, sourc
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
-source-map@^0.5.0, source-map@^0.5.6, source-map@^0.5.7:
+source-map@^0.5.6, source-map@^0.5.7:
   version "0.5.7"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
 
-source-map@^0.7.3, source-map@~0.7.2:
+source-map@^0.7.3:
   version "0.7.3"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383"
   integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==
 
+source-map@^0.8.0-beta.0:
+  version "0.8.0-beta.0"
+  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.8.0-beta.0.tgz#d4c1bb42c3f7ee925f005927ba10709e0d1d1f11"
+  integrity sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==
+  dependencies:
+    whatwg-url "^7.0.0"
+
 sourcemap-codec@^1.4.4, sourcemap-codec@^1.4.8:
   version "1.4.8"
   resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4"
@@ -17784,32 +15401,6 @@ space-separated-tokens@^1.0.0, space-separated-tokens@^1.1.0:
   resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz#85f32c3d10d9682007e917414ddc5c26d1aa6899"
   integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==
 
-spdx-correct@^3.0.0:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9"
-  integrity sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==
-  dependencies:
-    spdx-expression-parse "^3.0.0"
-    spdx-license-ids "^3.0.0"
-
-spdx-exceptions@^2.1.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz#3f28ce1a77a00372683eade4a433183527a2163d"
-  integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==
-
-spdx-expression-parse@^3.0.0:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz#cf70f50482eefdc98e3ce0a6833e4a53ceeba679"
-  integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==
-  dependencies:
-    spdx-exceptions "^2.1.0"
-    spdx-license-ids "^3.0.0"
-
-spdx-license-ids@^3.0.0:
-  version "3.0.9"
-  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.9.tgz#8a595135def9592bda69709474f1cbeea7c2467f"
-  integrity sha512-Ki212dKK4ogX+xDo4CtOZBVIwhsKBEfsEEcwmJfLQzirgc2jIWdzg40Unxz/HzEUqM1WFzVlQSMF9kZZ2HboLQ==
-
 spdy-transport@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/spdy-transport/-/spdy-transport-3.0.0.tgz#00d4863a6400ad75df93361a1608605e5dcdcf31"
@@ -17879,20 +15470,6 @@ sshpk@^1.7.0:
     safer-buffer "^2.0.2"
     tweetnacl "~0.14.0"
 
-ssri@^6.0.1:
-  version "6.0.2"
-  resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.2.tgz#157939134f20464e7301ddba3e90ffa8f7728ac5"
-  integrity sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==
-  dependencies:
-    figgy-pudding "^3.5.1"
-
-ssri@^8.0.1:
-  version "8.0.1"
-  resolved "https://registry.yarnpkg.com/ssri/-/ssri-8.0.1.tgz#638e4e439e2ffbd2cd289776d5ca457c4f51a2af"
-  integrity sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==
-  dependencies:
-    minipass "^3.1.1"
-
 stable@^0.1.8:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf"
@@ -17905,19 +15482,14 @@ stack-generator@^2.0.5:
   dependencies:
     stackframe "^1.3.4"
 
-stack-utils@^2.0.2:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.3.tgz#cd5f030126ff116b78ccb3c027fe302713b61277"
-  integrity sha512-gL//fkxfWUsIlFL2Tl42Cl6+HFALEaB1FU76I/Fy+oZjRreP7OPMXFlGbxM7NQsI0ZpUfw76sHnv0WNYuTb7Iw==
+stack-utils@^2.0.3:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f"
+  integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==
   dependencies:
     escape-string-regexp "^2.0.0"
 
-stackframe@^1.1.1:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/stackframe/-/stackframe-1.2.0.tgz#52429492d63c62eb989804c11552e3d22e779303"
-  integrity sha512-GrdeshiRmS1YLMYgzF16olf2jJ/IzxXY9lhKOskuVziubpTYcYqyOwYeJKzQkwy7uN0fYSsbsC4RQaXf9LCrYA==
-
-stackframe@^1.3.4:
+stackframe@^1.1.1, stackframe@^1.3.4:
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/stackframe/-/stackframe-1.3.4.tgz#b881a004c8c149a5e8efef37d51b16e412943310"
   integrity sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==
@@ -17975,14 +15547,6 @@ statuses@2.0.1:
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
   integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=
 
-stream-browserify@^2.0.1:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b"
-  integrity sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==
-  dependencies:
-    inherits "~2.0.1"
-    readable-stream "^2.0.2"
-
 stream-combiner@~0.0.4:
   version "0.0.4"
   resolved "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz#4d5e433c185261dde623ca3f44c586bcf5c4ad14"
@@ -17990,35 +15554,6 @@ stream-combiner@~0.0.4:
   dependencies:
     duplexer "~0.1.1"
 
-stream-each@^1.1.0:
-  version "1.2.3"
-  resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae"
-  integrity sha512-vlMC2f8I2u/bZGqkdfLQW/13Zihpej/7PmSiMQsbYddxuTsJp8vRe2x2FvVExZg7FaOds43ROAuFJwPR4MTZLw==
-  dependencies:
-    end-of-stream "^1.1.0"
-    stream-shift "^1.0.0"
-
-stream-http@^2.7.2:
-  version "2.8.3"
-  resolved "https://registry.yarnpkg.com/stream-http/-/stream-http-2.8.3.tgz#b2d242469288a5a27ec4fe8933acf623de6514fc"
-  integrity sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==
-  dependencies:
-    builtin-status-codes "^3.0.0"
-    inherits "^2.0.1"
-    readable-stream "^2.3.6"
-    to-arraybuffer "^1.0.0"
-    xtend "^4.0.0"
-
-stream-shift@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
-  integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
-
-strict-uri-encode@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
-  integrity sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=
-
 strict-uri-encode@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546"
@@ -18042,6 +15577,14 @@ string-length@^4.0.1:
     char-regex "^1.0.2"
     strip-ansi "^6.0.0"
 
+string-length@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/string-length/-/string-length-5.0.1.tgz#3d647f497b6e8e8d41e422f7e0b23bc536c8381e"
+  integrity sha512-9Ep08KAMUn0OadnVaBuRdE2l615CQ508kr0XMadjClfYpdCyvrbFp6Taebo8yyxokQ4viUd/xPPUA4FGgUa0ow==
+  dependencies:
+    char-regex "^2.0.0"
+    strip-ansi "^7.0.1"
+
 string-natural-compare@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4"
@@ -18064,15 +15607,6 @@ string-width@^2.1.1:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^4.0.0"
 
-string-width@^3.0.0, string-width@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961"
-  integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==
-  dependencies:
-    emoji-regex "^7.0.1"
-    is-fullwidth-code-point "^2.0.0"
-    strip-ansi "^5.1.0"
-
 string-width@^4.1.0, string-width@^4.2.0:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5"
@@ -18082,21 +15616,7 @@ string-width@^4.1.0, string-width@^4.2.0:
     is-fullwidth-code-point "^3.0.0"
     strip-ansi "^6.0.0"
 
-string.prototype.matchall@^4.0.4:
-  version "4.0.5"
-  resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.5.tgz#59370644e1db7e4c0c045277690cf7b01203c4da"
-  integrity sha512-Z5ZaXO0svs0M2xd/6By3qpeKpLKd9mO4v4q3oMEQrk8Ck4xOD5d5XeBOOjGrmVZZ/AHB1S0CgG4N5r1G9N3E2Q==
-  dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.3"
-    es-abstract "^1.18.2"
-    get-intrinsic "^1.1.1"
-    has-symbols "^1.0.2"
-    internal-slot "^1.0.3"
-    regexp.prototype.flags "^1.3.1"
-    side-channel "^1.0.4"
-
-string.prototype.matchall@^4.0.7:
+string.prototype.matchall@^4.0.6, string.prototype.matchall@^4.0.7, string.prototype.matchall@^4.0.8:
   version "4.0.8"
   resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.8.tgz#3bf85722021816dcd1bf38bb714915887ca79fd3"
   integrity sha512-6zOCOcJ+RJAQshcTvXPHoxoQGONa3e/Lqx90wUA+wEzX78sg5Bo+1tQo4N0pohS0erG9qtCqJDjNCQBjeWVxyg==
@@ -18110,14 +15630,6 @@ string.prototype.matchall@^4.0.7:
     regexp.prototype.flags "^1.4.3"
     side-channel "^1.0.4"
 
-string.prototype.trimend@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80"
-  integrity sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==
-  dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.3"
-
 string.prototype.trimend@^1.0.5:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz#c4a27fa026d979d79c04f17397f250a462944533"
@@ -18127,14 +15639,6 @@ string.prototype.trimend@^1.0.5:
     define-properties "^1.1.4"
     es-abstract "^1.20.4"
 
-string.prototype.trimstart@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz#b36399af4ab2999b4c9c648bd7a3fb2bb26feeed"
-  integrity sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==
-  dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.3"
-
 string.prototype.trimstart@^1.0.5:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz#e90ab66aa8e4007d92ef591bbf3cd422c56bdcf4"
@@ -18144,7 +15648,7 @@ string.prototype.trimstart@^1.0.5:
     define-properties "^1.1.4"
     es-abstract "^1.20.4"
 
-string_decoder@^1.0.0, string_decoder@^1.1.1:
+string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
   integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
@@ -18176,13 +15680,6 @@ stringify-object@^3.3.0:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-strip-ansi@6.0.0, strip-ansi@^6.0.0:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
-  integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
-  dependencies:
-    ansi-regex "^5.0.0"
-
 strip-ansi@^3.0.0, strip-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
@@ -18197,12 +15694,19 @@ strip-ansi@^4.0.0:
   dependencies:
     ansi-regex "^3.0.0"
 
-strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
-  version "5.2.0"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae"
-  integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
+strip-ansi@^7.0.1:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45"
+  integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==
   dependencies:
-    ansi-regex "^4.1.0"
+    ansi-regex "^6.0.1"
 
 strip-bom@^3.0.0:
   version "3.0.0"
@@ -18214,18 +15718,10 @@ strip-bom@^4.0.0:
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878"
   integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==
 
-strip-comments@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/strip-comments/-/strip-comments-1.0.2.tgz#82b9c45e7f05873bee53f37168af930aa368679d"
-  integrity sha512-kL97alc47hoyIQSV165tTt9rG5dn4w1dNnBhOQ3bOU1Nc1hel09jnXANaHJ7vzHLd4Ju8kseDGzlev96pghLFw==
-  dependencies:
-    babel-extract-comments "^1.0.0"
-    babel-plugin-transform-object-rest-spread "^6.26.0"
-
-strip-eof@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
-  integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=
+strip-comments@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/strip-comments/-/strip-comments-2.0.1.tgz#4ad11c3fbcac177a67a40ac224ca339ca1c1ba9b"
+  integrity sha512-ZprKx+bBLXv067WTCALv8SSz5l2+XhpYCsVtSqlMnkAXMWDq+/ekVbl1ghqP9rUHTzv6sm/DwCOiYutU/yp1fw==
 
 strip-final-newline@^2.0.0:
   version "2.0.0"
@@ -18249,13 +15745,10 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1:
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
   integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
 
-style-loader@1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-1.3.0.tgz#828b4a3b3b7e7aa5847ce7bae9e874512114249e"
-  integrity sha512-V7TCORko8rs9rIqkSrlMfkqA63DfoGBBJmK1kKGCcSi+BWb4cqz0SRsnp4l6rU5iwOEd0/2ePv68SV22VXon4Q==
-  dependencies:
-    loader-utils "^2.0.0"
-    schema-utils "^2.7.0"
+style-loader@^3.3.1:
+  version "3.3.3"
+  resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-3.3.3.tgz#bba8daac19930169c0c9c96706749a597ae3acff"
+  integrity sha512-53BiGLXAcll9maCYtZi2RCQZKa8NQQai5C4horqKyRmHj9H7QmcUyucrH+4KW/gBQbXM2AsB0axoEcFZPlfPcw==
 
 style-to-object@^0.3.0:
   version "0.3.0"
@@ -18280,14 +15773,13 @@ styled-components@^5.2.1:
     shallowequal "^1.1.0"
     supports-color "^5.5.0"
 
-stylehacks@^4.0.0:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/stylehacks/-/stylehacks-4.0.3.tgz#6718fcaf4d1e07d8a1318690881e8d96726a71d5"
-  integrity sha512-7GlLk9JwlElY4Y6a/rmbH2MhVlTyVmiJd1PfTCqFaIBEGMYNsrO/v3SeGTdhBThLg4Z+NbOk/qFMwCa+J+3p/g==
+stylehacks@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/stylehacks/-/stylehacks-5.1.1.tgz#7934a34eb59d7152149fa69d6e9e56f2fc34bcc9"
+  integrity sha512-sBpcd5Hx7G6seo7b1LkpttvTz7ikD0LlH5RmdcBNb6fFR0Fl7LQwHDFr300q4cwUqi+IYrFGmsIHieMBfnN/Bw==
   dependencies:
-    browserslist "^4.0.0"
-    postcss "^7.0.0"
-    postcss-selector-parser "^3.0.0"
+    browserslist "^4.21.4"
+    postcss-selector-parser "^6.0.4"
 
 stylis@4.1.3, stylis@^4.0.6:
   version "4.1.3"
@@ -18305,6 +15797,19 @@ subscriptions-transport-ws@^0.9.18:
     symbol-observable "^1.0.4"
     ws "^5.2.0"
 
+sucrase@^3.32.0:
+  version "3.32.0"
+  resolved "https://registry.yarnpkg.com/sucrase/-/sucrase-3.32.0.tgz#c4a95e0f1e18b6847127258a75cf360bc568d4a7"
+  integrity sha512-ydQOU34rpSyj2TGyz4D2p8rbktIOZ8QY9s+DGLvFU1i5pWJE8vkpruCjGCMHsdXwnD7JDcS+noSwM/a7zyNFDQ==
+  dependencies:
+    "@jridgewell/gen-mapping" "^0.3.2"
+    commander "^4.0.0"
+    glob "7.1.6"
+    lines-and-columns "^1.1.6"
+    mz "^2.7.0"
+    pirates "^4.0.1"
+    ts-interface-checker "^0.1.9"
+
 supports-color@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"
@@ -18317,13 +15822,6 @@ supports-color@^5.3.0, supports-color@^5.5.0:
   dependencies:
     has-flag "^3.0.0"
 
-supports-color@^6.1.0:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3"
-  integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==
-  dependencies:
-    has-flag "^3.0.0"
-
 supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
@@ -18331,6 +15829,13 @@ supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0:
   dependencies:
     has-flag "^4.0.0"
 
+supports-color@^8.0.0:
+  version "8.1.1"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c"
+  integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==
+  dependencies:
+    has-flag "^4.0.0"
+
 supports-hyperlinks@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.2.0.tgz#4f77b42488765891774b70c79babd87f9bd594bb"
@@ -18361,7 +15866,7 @@ svgmoji@^3.2.0:
     "@svgmoji/openmoji" "^3.2.0"
     "@svgmoji/twemoji" "^3.2.0"
 
-svgo@^1.0.0, svgo@^1.2.2:
+svgo@^1.2.2:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/svgo/-/svgo-1.3.2.tgz#b6dc511c063346c9e415b81e43401145b96d4167"
   integrity sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw==
@@ -18380,6 +15885,19 @@ svgo@^1.0.0, svgo@^1.2.2:
     unquote "~1.1.1"
     util.promisify "~1.0.0"
 
+svgo@^2.7.0:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.8.0.tgz#4ff80cce6710dc2795f0c7c74101e6764cfccd24"
+  integrity sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg==
+  dependencies:
+    "@trysound/sax" "0.2.0"
+    commander "^7.2.0"
+    css-select "^4.1.3"
+    css-tree "^1.1.3"
+    csso "^4.2.0"
+    picocolors "^1.0.0"
+    stable "^0.1.8"
+
 swap-case@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/swap-case/-/swap-case-2.0.2.tgz#671aedb3c9c137e2985ef51c51f9e98445bf70d9"
@@ -18410,39 +15928,49 @@ sync-fetch@0.3.0:
     buffer "^5.7.0"
     node-fetch "^2.6.1"
 
-table@^6.0.9:
-  version "6.7.1"
-  resolved "https://registry.yarnpkg.com/table/-/table-6.7.1.tgz#ee05592b7143831a8c94f3cee6aae4c1ccef33e2"
-  integrity sha512-ZGum47Yi6KOOFDE8m223td53ath2enHcYLgOCjGr5ngu8bdIARQk6mN/wRMv4yMRcHnCSnHbCEha4sobQx5yWg==
-  dependencies:
-    ajv "^8.0.1"
-    lodash.clonedeep "^4.5.0"
-    lodash.truncate "^4.4.2"
-    slice-ansi "^4.0.0"
-    string-width "^4.2.0"
-    strip-ansi "^6.0.0"
-
-tapable@^1.0.0, tapable@^1.1.3:
+tailwindcss@^3.0.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.3.2.tgz#2f9e35d715fdf0bbf674d90147a0684d7054a2d3"
+  integrity sha512-9jPkMiIBXvPc2KywkraqsUfbfj+dHDb+JPWtSJa9MLFdrPyazI7q6WX2sUrm7R9eVR7qqv3Pas7EvQFzxKnI6w==
+  dependencies:
+    "@alloc/quick-lru" "^5.2.0"
+    arg "^5.0.2"
+    chokidar "^3.5.3"
+    didyoumean "^1.2.2"
+    dlv "^1.1.3"
+    fast-glob "^3.2.12"
+    glob-parent "^6.0.2"
+    is-glob "^4.0.3"
+    jiti "^1.18.2"
+    lilconfig "^2.1.0"
+    micromatch "^4.0.5"
+    normalize-path "^3.0.0"
+    object-hash "^3.0.0"
+    picocolors "^1.0.0"
+    postcss "^8.4.23"
+    postcss-import "^15.1.0"
+    postcss-js "^4.0.1"
+    postcss-load-config "^4.0.1"
+    postcss-nested "^6.0.1"
+    postcss-selector-parser "^6.0.11"
+    postcss-value-parser "^4.2.0"
+    resolve "^1.22.2"
+    sucrase "^3.32.0"
+
+tapable@^1.0.0:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2"
   integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==
 
-tar@^6.0.2:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83"
-  integrity sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==
-  dependencies:
-    chownr "^2.0.0"
-    fs-minipass "^2.0.0"
-    minipass "^3.0.0"
-    minizlib "^2.1.1"
-    mkdirp "^1.0.3"
-    yallist "^4.0.0"
+tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0:
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0"
+  integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==
 
-temp-dir@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-1.0.0.tgz#0a7c0ea26d3a39afa7e0ebea9c1fc0bc4daa011d"
-  integrity sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=
+temp-dir@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-2.0.0.tgz#bde92b05bdfeb1516e804c9c00ad45177f31321e"
+  integrity sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==
 
 temp@^0.9.4:
   version "0.9.4"
@@ -18452,14 +15980,15 @@ temp@^0.9.4:
     mkdirp "^0.5.1"
     rimraf "~2.6.2"
 
-tempy@^0.3.0:
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/tempy/-/tempy-0.3.0.tgz#6f6c5b295695a16130996ad5ab01a8bd726e8bf8"
-  integrity sha512-WrH/pui8YCwmeiAoxV+lpRH9HpRtgBhSR2ViBPgpGb/wnYDzp21R4MN45fsCGvLROvY67o3byhJRYRONJyImVQ==
+tempy@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/tempy/-/tempy-0.6.0.tgz#65e2c35abc06f1124a97f387b08303442bde59f3"
+  integrity sha512-G13vtMYPT/J8A4X2SjdtBTphZlrp1gKv6hZiOjw14RCWg6GbHuQBGtjlx75xLbYV/wEc0D7G5K4rxKP/cXk8Bw==
   dependencies:
-    temp-dir "^1.0.0"
-    type-fest "^0.3.1"
-    unique-string "^1.0.0"
+    is-stream "^2.0.0"
+    temp-dir "^2.0.0"
+    type-fest "^0.16.0"
+    unique-string "^2.0.0"
 
 terminal-link@^2.0.0:
   version "2.1.1"
@@ -18469,53 +15998,26 @@ terminal-link@^2.0.0:
     ansi-escapes "^4.2.1"
     supports-hyperlinks "^2.0.0"
 
-terser-webpack-plugin@4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-4.2.3.tgz#28daef4a83bd17c1db0297070adc07fc8cfc6a9a"
-  integrity sha512-jTgXh40RnvOrLQNgIkwEKnQ8rmHjHK4u+6UBEi+W+FPmvb+uo+chJXntKe7/3lW5mNysgSWD60KyesnhW8D6MQ==
-  dependencies:
-    cacache "^15.0.5"
-    find-cache-dir "^3.3.1"
-    jest-worker "^26.5.0"
-    p-limit "^3.0.2"
-    schema-utils "^3.0.0"
-    serialize-javascript "^5.0.1"
-    source-map "^0.6.1"
-    terser "^5.3.4"
-    webpack-sources "^1.4.3"
-
-terser-webpack-plugin@^1.4.3:
-  version "1.4.5"
-  resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz#a217aefaea330e734ffacb6120ec1fa312d6040b"
-  integrity sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==
-  dependencies:
-    cacache "^12.0.2"
-    find-cache-dir "^2.1.0"
-    is-wsl "^1.1.0"
-    schema-utils "^1.0.0"
-    serialize-javascript "^4.0.0"
-    source-map "^0.6.1"
-    terser "^4.1.2"
-    webpack-sources "^1.4.0"
-    worker-farm "^1.7.0"
-
-terser@^4.1.2, terser@^4.6.2, terser@^4.6.3:
-  version "4.8.1"
-  resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.1.tgz#a00e5634562de2239fd404c649051bf6fc21144f"
-  integrity sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw==
+terser-webpack-plugin@^5.2.5, terser-webpack-plugin@^5.3.7:
+  version "5.3.9"
+  resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.9.tgz#832536999c51b46d468067f9e37662a3b96adfe1"
+  integrity sha512-ZuXsqE07EcggTWQjXUj+Aot/OMcD0bMKGgF63f7UxYcu5/AJF53aIpK1YoP5xR9l6s/Hy2b+t1AM0bLNPRuhwA==
   dependencies:
-    commander "^2.20.0"
-    source-map "~0.6.1"
-    source-map-support "~0.5.12"
+    "@jridgewell/trace-mapping" "^0.3.17"
+    jest-worker "^27.4.5"
+    schema-utils "^3.1.1"
+    serialize-javascript "^6.0.1"
+    terser "^5.16.8"
 
-terser@^5.3.4:
-  version "5.7.0"
-  resolved "https://registry.yarnpkg.com/terser/-/terser-5.7.0.tgz#a761eeec206bc87b605ab13029876ead938ae693"
-  integrity sha512-HP5/9hp2UaZt5fYkuhNBR8YyRcT8juw8+uFbAme53iN9hblvKnLUTKkmwJG6ocWpIKf8UK4DoeWG4ty0J6S6/g==
+terser@^5.0.0, terser@^5.10.0, terser@^5.16.8:
+  version "5.18.1"
+  resolved "https://registry.yarnpkg.com/terser/-/terser-5.18.1.tgz#6d8642508ae9fb7b48768e48f16d675c89a78460"
+  integrity sha512-j1n0Ao919h/Ai5r43VAnfV/7azUYW43GPxK7qSATzrsERfW7+y2QW9Cp9ufnRF5CQUWbnLSo7UJokSWCqg4tsQ==
   dependencies:
+    "@jridgewell/source-map" "^0.3.3"
+    acorn "^8.8.2"
     commander "^2.20.0"
-    source-map "~0.7.2"
-    source-map-support "~0.5.19"
+    source-map-support "~0.5.20"
 
 test-exclude@^6.0.0:
   version "6.0.0"
@@ -18526,29 +16028,35 @@ test-exclude@^6.0.0:
     glob "^7.1.4"
     minimatch "^3.0.4"
 
-text-table@0.2.0, text-table@^0.2.0:
+text-table@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=
 
-throat@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b"
-  integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==
+thenify-all@^1.0.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726"
+  integrity sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==
+  dependencies:
+    thenify ">= 3.1.0 < 4"
+
+"thenify@>= 3.1.0 < 4":
+  version "3.3.1"
+  resolved "https://registry.yarnpkg.com/thenify/-/thenify-3.3.1.tgz#8932e686a4066038a016dd9e2ca46add9838a95f"
+  integrity sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==
+  dependencies:
+    any-promise "^1.0.0"
+
+throat@^6.0.1:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/throat/-/throat-6.0.2.tgz#51a3fbb5e11ae72e2cf74861ed5c8020f89f29fe"
+  integrity sha512-WKexMoJj3vEuK0yFEapj8y64V0A6xcuPuK9Gt1d0R+dzCSJc0lHqQytAbSB4cDAK0dWh4T0E2ETkoLE2WZ41OQ==
 
 throttle-debounce@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/throttle-debounce/-/throttle-debounce-3.0.1.tgz#32f94d84dfa894f786c9a1f290e7a645b6a19abb"
   integrity sha512-dTEWWNu6JmeVXY0ZYoPuH5cRIwc0MeGbJwah9KUNYSJwommQpCzTySTpEe8Gs1J23aeWEuAobe4Ag7EHVt/LOg==
 
-through2@^2.0.0:
-  version "2.0.5"
-  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
-  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
-  dependencies:
-    readable-stream "~2.3.6"
-    xtend "~4.0.1"
-
 through@2, through@^2.3.6, through@~2.3, through@~2.3.1:
   version "2.3.8"
   resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@@ -18559,18 +16067,6 @@ thunky@^1.0.2:
   resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.1.0.tgz#5abaf714a9405db0504732bbccd2cedd9ef9537d"
   integrity sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==
 
-timers-browserify@^2.0.4:
-  version "2.0.12"
-  resolved "https://registry.yarnpkg.com/timers-browserify/-/timers-browserify-2.0.12.tgz#44a45c11fbf407f34f97bccd1577c652361b00ee"
-  integrity sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==
-  dependencies:
-    setimmediate "^1.0.4"
-
-timsort@^0.3.0:
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/timsort/-/timsort-0.3.0.tgz#405411a8e7e6339fe64db9a234de11dc31e02bd4"
-  integrity sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q=
-
 tiny-emitter@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tiny-emitter/-/tiny-emitter-2.1.0.tgz#1d1a56edfc51c43e863cbb5382a72330e3555423"
@@ -18610,11 +16106,6 @@ tmpl@1.0.x:
   resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"
   integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=
 
-to-arraybuffer@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43"
-  integrity sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=
-
 to-fast-properties@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
@@ -18684,6 +16175,13 @@ tough-cookie@~2.5.0:
     psl "^1.1.28"
     punycode "^2.1.1"
 
+tr46@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09"
+  integrity sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==
+  dependencies:
+    punycode "^2.1.0"
+
 tr46@^2.0.2:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -18711,6 +16209,11 @@ ts-easing@^0.2.0:
   resolved "https://registry.yarnpkg.com/ts-easing/-/ts-easing-0.2.0.tgz#c8a8a35025105566588d87dbda05dd7fbfa5a4ec"
   integrity sha512-Z86EW+fFFh/IFB1fqQ3/+7Zpf9t2ebOAxNI/V6Wo7r5gqiqtxmgTlQ1qbqQcjLKYeSHPTsEmvlJUDg/EuL0uHQ==
 
+ts-interface-checker@^0.1.9:
+  version "0.1.13"
+  resolved "https://registry.yarnpkg.com/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz#784fd3d679722bc103b1b4b8030bcddb5db2a699"
+  integrity sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==
+
 ts-invariant@^0.7.0:
   version "0.7.3"
   resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.7.3.tgz#13aae22a4a165393aaf5cecdee45ef4128d358b8"
@@ -18723,6 +16226,25 @@ ts-log@^2.2.3:
   resolved "https://registry.yarnpkg.com/ts-log/-/ts-log-2.2.3.tgz#4da5640fe25a9fb52642cd32391c886721318efb"
   integrity sha512-XvB+OdKSJ708Dmf9ore4Uf/q62AYDTzFcAdxc8KNML1mmAWywRFVt/dn1KYJH8Agt5UJNujfM3znU5PxgAzA2w==
 
+ts-node@^10.7.0:
+  version "10.9.1"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b"
+  integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==
+  dependencies:
+    "@cspotcode/source-map-support" "^0.8.0"
+    "@tsconfig/node10" "^1.0.7"
+    "@tsconfig/node12" "^1.0.7"
+    "@tsconfig/node14" "^1.0.0"
+    "@tsconfig/node16" "^1.0.2"
+    acorn "^8.4.1"
+    acorn-walk "^8.1.1"
+    arg "^4.1.0"
+    create-require "^1.1.0"
+    diff "^4.0.1"
+    make-error "^1.1.1"
+    v8-compile-cache-lib "^3.0.1"
+    yn "3.1.1"
+
 ts-node@^9:
   version "9.1.1"
   resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.1.1.tgz#51a9a450a3e959401bda5f004a72d54b936d376d"
@@ -18735,19 +16257,14 @@ ts-node@^9:
     source-map-support "^0.5.17"
     yn "3.1.1"
 
-ts-pnp@1.2.0, ts-pnp@^1.1.6:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92"
-  integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw==
-
-tsconfig-paths@^3.9.0:
-  version "3.9.0"
-  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz#098547a6c4448807e8fcb8eae081064ee9a3c90b"
-  integrity sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw==
+tsconfig-paths@^3.14.1:
+  version "3.14.2"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088"
+  integrity sha512-o/9iXgCYc5L/JxCHPe3Hvh8Q/2xm5Z+p18PESBU6Ff33695QnCHBEjcytY2q19ua7Mbl/DavtBOLq+oG0RCL+g==
   dependencies:
     "@types/json5" "^0.0.29"
-    json5 "^1.0.1"
-    minimist "^1.2.0"
+    json5 "^1.0.2"
+    minimist "^1.2.6"
     strip-bom "^3.0.0"
 
 tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0:
@@ -18755,12 +16272,7 @@ tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
-tslib@^2, tslib@^2.0.3, tslib@^2.1.0, tslib@~2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.2.0.tgz#fb2c475977e35e241311ede2693cee1ec6698f5c"
-  integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==
-
-tslib@^2.0.0, tslib@^2.3.0, tslib@^2.4.0:
+tslib@^2, tslib@^2.0.0, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.4.0:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e"
   integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==
@@ -18775,6 +16287,11 @@ tslib@~2.1.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a"
   integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==
 
+tslib@~2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.2.0.tgz#fb2c475977e35e241311ede2693cee1ec6698f5c"
+  integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==
+
 tslib@~2.3.0:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
@@ -18785,18 +16302,13 @@ tslib@~2.5.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.3.tgz#24944ba2d990940e6e982c4bea147aba80209913"
   integrity sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==
 
-tsutils@^3.17.1:
+tsutils@^3.21.0:
   version "3.21.0"
   resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"
   integrity sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==
   dependencies:
     tslib "^1.8.1"
 
-tty-browserify@0.0.0:
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"
-  integrity sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=
-
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
@@ -18840,6 +16352,11 @@ type-detect@4.0.8, type-detect@^4.0.8:
   resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c"
   integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==
 
+type-fest@^0.16.0:
+  version "0.16.0"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.16.0.tgz#3240b891a78b0deae910dbeb86553e552a148860"
+  integrity sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==
+
 type-fest@^0.20.2:
   version "0.20.2"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
@@ -18850,21 +16367,6 @@ type-fest@^0.21.3:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37"
   integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==
 
-type-fest@^0.3.1:
-  version "0.3.1"
-  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.3.1.tgz#63d00d204e059474fe5e1b7c011112bbd1dc29e1"
-  integrity sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==
-
-type-fest@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b"
-  integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==
-
-type-fest@^0.8.1:
-  version "0.8.1"
-  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
-  integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
-
 type-fest@^1.2.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1"
@@ -18883,16 +16385,6 @@ type-is@~1.6.18:
     media-typer "0.3.0"
     mime-types "~2.1.24"
 
-type@^1.0.1:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/type/-/type-1.2.0.tgz#848dd7698dafa3e54a6c479e759c4bc3f18847a0"
-  integrity sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==
-
-type@^2.0.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/type/-/type-2.5.0.tgz#0a2e78c2e77907b252abe5f298c1b01c63f0db3d"
-  integrity sha512-180WMDQaIMm3+7hGXWf12GtdniDEy7nYcyFMKJn/eZz/6tSLXrUN9V0wKSbMjej0I1WHWbpREDEKHtqPQa9NNw==
-
 typedarray-to-buffer@^3.1.5:
   version "3.1.5"
   resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"
@@ -18900,16 +16392,6 @@ typedarray-to-buffer@^3.1.5:
   dependencies:
     is-typedarray "^1.0.0"
 
-typedarray@^0.0.6:
-  version "0.0.6"
-  resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
-  integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
-
-typescript@^4.1.3:
-  version "4.2.4"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.2.4.tgz#8610b59747de028fda898a8aef0e103f156d0961"
-  integrity sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg==
-
 typescript@^4.8.4:
   version "4.9.4"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78"
@@ -18920,16 +16402,6 @@ ua-parser-js@^0.7.30:
   resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532"
   integrity sha512-s8ax/CeZdK9R/56Sui0WM6y9OFREJarMRHqLB2EwkovemBxNQ+Bqu8GAsUnVcXKgphb++ghr/B2BZx4mahujPw==
 
-unbox-primitive@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"
-  integrity sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==
-  dependencies:
-    function-bind "^1.1.1"
-    has-bigints "^1.0.1"
-    has-symbols "^1.0.2"
-    which-boxed-primitive "^1.0.2"
-
 unbox-primitive@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e"
@@ -18945,28 +16417,28 @@ unc-path-regex@^0.1.2:
   resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa"
   integrity sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg==
 
-unicode-canonical-property-names-ecmascript@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"
-  integrity sha512-jDrNnXWHd4oHiTZnx/ZG7gtUTVp+gCcTTKr8L0HjlwphROEW3+Him+IpvC+xcJEFegapiMZyZe02CyuOnRmbnQ==
+unicode-canonical-property-names-ecmascript@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz#301acdc525631670d39f6146e0e77ff6bbdebddc"
+  integrity sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==
 
-unicode-match-property-ecmascript@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-1.0.4.tgz#8ed2a32569961bce9227d09cd3ffbb8fed5f020c"
-  integrity sha512-L4Qoh15vTfntsn4P1zqnHulG0LdXgjSO035fEpdtp6YxXhMT51Q6vgM5lYdG/5X3MjS+k/Y9Xw4SFCY9IkR0rg==
+unicode-match-property-ecmascript@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz#54fd16e0ecb167cf04cf1f756bdcc92eba7976c3"
+  integrity sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==
   dependencies:
-    unicode-canonical-property-names-ecmascript "^1.0.4"
-    unicode-property-aliases-ecmascript "^1.0.4"
+    unicode-canonical-property-names-ecmascript "^2.0.0"
+    unicode-property-aliases-ecmascript "^2.0.0"
 
-unicode-match-property-value-ecmascript@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.2.0.tgz#0d91f600eeeb3096aa962b1d6fc88876e64ea531"
-  integrity sha512-wjuQHGQVofmSJv1uVISKLE5zO2rNGzM/KCYZch/QQvez7C1hUhBIuZ701fYXExuufJFMPhv2SyL8CyoIfMLbIQ==
+unicode-match-property-value-ecmascript@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.1.0.tgz#cb5fffdcd16a05124f5a4b0bf7c3770208acbbe0"
+  integrity sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA==
 
-unicode-property-aliases-ecmascript@^1.0.4:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz#dd57a99f6207bedff4628abefb94c50db941c8f4"
-  integrity sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg==
+unicode-property-aliases-ecmascript@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz#43d41e3be698bd493ef911077c9b131f827e8ccd"
+  integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==
 
 unified@^9.0.0:
   version "9.2.1"
@@ -18990,36 +16462,12 @@ union-value@^1.0.0:
     is-extendable "^0.1.1"
     set-value "^2.0.1"
 
-uniq@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff"
-  integrity sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=
-
-uniqs@^2.0.0:
+unique-string@^2.0.0:
   version "2.0.0"
-  resolved "https://registry.yarnpkg.com/uniqs/-/uniqs-2.0.0.tgz#ffede4b36b25290696e6e165d4a59edb998e6b02"
-  integrity sha1-/+3ks2slKQaW5uFl1KWe25mOawI=
-
-unique-filename@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230"
-  integrity sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==
-  dependencies:
-    unique-slug "^2.0.0"
-
-unique-slug@^2.0.0:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/unique-slug/-/unique-slug-2.0.2.tgz#baabce91083fc64e945b0f3ad613e264f7cd4e6c"
-  integrity sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==
-  dependencies:
-    imurmurhash "^0.1.4"
-
-unique-string@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-1.0.0.tgz#9e1057cca851abb93398f8b33ae187b99caec11a"
-  integrity sha1-nhBXzKhRq7kzmPizOuGHuZyuwRo=
+  resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-2.0.0.tgz#39c6451f81afb2749de2b233e3f7c5e8843bd89d"
+  integrity sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==
   dependencies:
-    crypto-random-string "^1.0.0"
+    crypto-random-string "^2.0.0"
 
 unist-builder@^2.0.0:
   version "2.0.3"
@@ -19109,15 +16557,15 @@ unset-value@^1.0.0:
     has-value "^0.3.1"
     isobject "^3.0.0"
 
-upath@^1.1.1, upath@^1.1.2, upath@^1.2.0:
+upath@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"
   integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==
 
-update-browserslist-db@^1.0.9:
-  version "1.0.10"
-  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3"
-  integrity sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==
+update-browserslist-db@^1.0.11:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz#9a2a641ad2907ae7b3616506f4b977851db5b940"
+  integrity sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==
   dependencies:
     escalade "^3.1.1"
     picocolors "^1.0.0"
@@ -19148,15 +16596,6 @@ urix@^0.1.0:
   resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
   integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
 
-url-loader@4.1.1:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.1.tgz#28505e905cae158cf07c92ca622d7f237e70a4e2"
-  integrity sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==
-  dependencies:
-    loader-utils "^2.0.0"
-    mime-types "^2.1.27"
-    schema-utils "^3.0.0"
-
 url-parse-lax@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c"
@@ -19164,30 +16603,6 @@ url-parse-lax@^3.0.0:
   dependencies:
     prepend-http "^2.0.0"
 
-url-parse@^1.4.3:
-  version "1.5.10"
-  resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1"
-  integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==
-  dependencies:
-    querystringify "^2.1.1"
-    requires-port "^1.0.0"
-
-url-parse@^1.5.1:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.1.tgz#d5fa9890af8a5e1f274a2c98376510f6425f6e3b"
-  integrity sha512-HOfCOUJt7iSYzEx/UqgtwKRMC6EU91NFhsCHMv9oM03VJcVo2Qrp8T8kI9D7amFf1cu+/3CEhgb3rF9zL7k85Q==
-  dependencies:
-    querystringify "^2.1.1"
-    requires-port "^1.0.0"
-
-url@^0.11.0:
-  version "0.11.0"
-  resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
-  integrity sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=
-  dependencies:
-    punycode "1.3.2"
-    querystring "0.2.0"
-
 use-callback-ref@^1.2.5:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/use-callback-ref/-/use-callback-ref-1.3.0.tgz#772199899b9c9a50526fedc4993fc7fa1f7e32d5"
@@ -19225,14 +16640,6 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
 
-util.promisify@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.0.tgz#440f7165a459c9a16dc145eb8e72f35687097030"
-  integrity sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==
-  dependencies:
-    define-properties "^1.1.2"
-    object.getownpropertydescriptors "^2.0.3"
-
 util.promisify@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.1.tgz#6baf7774b80eeb0f7520d8b81d07982a59abbaee"
@@ -19243,20 +16650,6 @@ util.promisify@~1.0.0:
     has-symbols "^1.0.1"
     object.getownpropertydescriptors "^2.1.0"
 
-util@0.10.3:
-  version "0.10.3"
-  resolved "https://registry.yarnpkg.com/util/-/util-0.10.3.tgz#7afb1afe50805246489e3db7fe0ed379336ac0f9"
-  integrity sha1-evsa/lCAUkZInj23/g7TeTNqwPk=
-  dependencies:
-    inherits "2.0.1"
-
-util@^0.11.0:
-  version "0.11.1"
-  resolved "https://registry.yarnpkg.com/util/-/util-0.11.1.tgz#3236733720ec64bb27f6e26f421aaa2e1b588d61"
-  integrity sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==
-  dependencies:
-    inherits "2.0.3"
-
 utila@~0.4:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/utila/-/utila-0.4.0.tgz#8a16a05d445657a3aea5eecc5b12a4fa5379772c"
@@ -19267,25 +16660,25 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==
 
-uuid@^3.0.0, uuid@^3.3.2, uuid@^3.4.0:
+uuid@^3.0.0, uuid@^3.3.2:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
-uuid@^8.3.0, uuid@^8.3.2:
+uuid@^8.3.2:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
-v8-compile-cache@^2.0.3:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee"
-  integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==
+v8-compile-cache-lib@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
+  integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
 
-v8-to-istanbul@^7.0.0:
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-7.1.2.tgz#30898d1a7fa0c84d225a2c1434fb958f290883c1"
-  integrity sha512-TxNb7YEUwkLXCQYeudi6lgQ/SZrzNO4kMdlqVxaZPUIUjCv6iSSypUQX70kNBSERpQ8fk48+d61FXk+tgqcWow==
+v8-to-istanbul@^8.1.0:
+  version "8.1.1"
+  resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-8.1.1.tgz#77b752fd3975e31bbcef938f85e9bd1c7a8d60ed"
+  integrity sha512-FGtKtv3xIpR6BYhvgH8MI/y78oT7d8Au3ww4QIxymrCtZEh5b8gCw2siywE+puhEmuWKDtmfrvF5UlB298ut3w==
   dependencies:
     "@types/istanbul-lib-coverage" "^2.0.1"
     convert-source-map "^1.6.0"
@@ -19296,14 +16689,6 @@ valid-url@1.0.9, valid-url@^1.0.9:
   resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200"
   integrity sha1-HBRHm0DxOXp1eC8RXkCGRHQzogA=
 
-validate-npm-package-license@^3.0.1:
-  version "3.0.4"
-  resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"
-  integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==
-  dependencies:
-    spdx-correct "^3.0.0"
-    spdx-expression-parse "^3.0.0"
-
 value-equal@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-1.0.1.tgz#1e0b794c734c5c0cade179c437d356d931a34d6c"
@@ -19319,11 +16704,6 @@ vary@~1.1.2:
   resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
   integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==
 
-vendors@^1.0.0:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.4.tgz#e2b800a53e7a29b93506c3cf41100d16c4c4ad8e"
-  integrity sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w==
-
 verror@1.10.0:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400"
@@ -19361,11 +16741,6 @@ virtualizedtableforantd4@^1.2.1:
   resolved "https://registry.yarnpkg.com/virtualizedtableforantd4/-/virtualizedtableforantd4-1.2.1.tgz#331e8d2f203cdee6667cb5b9cbd7af823f99f65a"
   integrity sha512-Hl21jF3WZESanz/iKIjvbjeZ5gGX2t85h2cWQFJAagOQnN7t/pvC4kXhfYNseJtaiU6QHOm5RgX3ud+oXeST1Q==
 
-vm-browserify@^1.0.1:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0"
-  integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==
-
 w3c-hr-time@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd"
@@ -19396,30 +16771,20 @@ wait-on@5.3.0:
     minimist "^1.2.5"
     rxjs "^6.6.3"
 
-walker@^1.0.7, walker@~1.0.5:
+walker@^1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.7.tgz#2f7f9b8fd10d677262b18a884e28d19618e028fb"
   integrity sha1-L3+bj9ENZ3JisYqITijRlhjgKPs=
   dependencies:
     makeerror "1.0.x"
 
-watchpack-chokidar2@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz#38500072ee6ece66f3769936950ea1771be1c957"
-  integrity sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==
-  dependencies:
-    chokidar "^2.1.8"
-
-watchpack@^1.7.4:
-  version "1.7.5"
-  resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.7.5.tgz#1267e6c55e0b9b5be44c2023aed5437a2c26c453"
-  integrity sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==
+watchpack@^2.4.0:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.0.tgz#fa33032374962c78113f93c7f2fb4c54c9862a5d"
+  integrity sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==
   dependencies:
+    glob-to-regexp "^0.4.1"
     graceful-fs "^4.1.2"
-    neo-async "^2.5.0"
-  optionalDependencies:
-    chokidar "^3.4.1"
-    watchpack-chokidar2 "^2.0.1"
 
 wbuf@^1.1.0, wbuf@^1.7.3:
   version "1.7.3"
@@ -19443,6 +16808,11 @@ webidl-conversions@^3.0.0:
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
   integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
 
+webidl-conversions@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
+  integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==
+
 webidl-conversions@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"
@@ -19453,82 +16823,70 @@ webidl-conversions@^6.1.0:
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514"
   integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==
 
-webpack-dev-middleware@^3.7.2:
-  version "3.7.3"
-  resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-3.7.3.tgz#0639372b143262e2b84ab95d3b91a7597061c2c5"
-  integrity sha512-djelc/zGiz9nZj/U7PTBi2ViorGJXEWo/3ltkPbDyxCXhhEXkW0ce99falaok4TPj+AsxLiXJR0EBOb0zh9fKQ==
+webpack-dev-middleware@^5.3.1:
+  version "5.3.3"
+  resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz#efae67c2793908e7311f1d9b06f2a08dcc97e51f"
+  integrity sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==
   dependencies:
-    memory-fs "^0.4.1"
-    mime "^2.4.4"
-    mkdirp "^0.5.1"
+    colorette "^2.0.10"
+    memfs "^3.4.3"
+    mime-types "^2.1.31"
     range-parser "^1.2.1"
-    webpack-log "^2.0.0"
-
-webpack-dev-server@3.11.1:
-  version "3.11.1"
-  resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-3.11.1.tgz#c74028bf5ba8885aaf230e48a20e8936ab8511f0"
-  integrity sha512-u4R3mRzZkbxQVa+MBWi2uVpB5W59H3ekZAJsQlKUTdl7Elcah2EhygTPLmeFXybQkf9i2+L0kn7ik9SnXa6ihQ==
-  dependencies:
-    ansi-html "0.0.7"
-    bonjour "^3.5.0"
-    chokidar "^2.1.8"
+    schema-utils "^4.0.0"
+
+webpack-dev-server@^4.6.0:
+  version "4.15.1"
+  resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.15.1.tgz#8944b29c12760b3a45bdaa70799b17cb91b03df7"
+  integrity sha512-5hbAst3h3C3L8w6W4P96L5vaV0PxSmJhxZvWKYIdgxOQm8pNZ5dEOmmSLBVpP85ReeyRt6AS1QJNyo/oFFPeVA==
+  dependencies:
+    "@types/bonjour" "^3.5.9"
+    "@types/connect-history-api-fallback" "^1.3.5"
+    "@types/express" "^4.17.13"
+    "@types/serve-index" "^1.9.1"
+    "@types/serve-static" "^1.13.10"
+    "@types/sockjs" "^0.3.33"
+    "@types/ws" "^8.5.5"
+    ansi-html-community "^0.0.8"
+    bonjour-service "^1.0.11"
+    chokidar "^3.5.3"
+    colorette "^2.0.10"
     compression "^1.7.4"
-    connect-history-api-fallback "^1.6.0"
-    debug "^4.1.1"
-    del "^4.1.1"
-    express "^4.17.1"
-    html-entities "^1.3.1"
-    http-proxy-middleware "0.19.1"
-    import-local "^2.0.0"
-    internal-ip "^4.3.0"
-    ip "^1.1.5"
-    is-absolute-url "^3.0.3"
-    killable "^1.0.1"
-    loglevel "^1.6.8"
-    opn "^5.5.0"
-    p-retry "^3.0.1"
-    portfinder "^1.0.26"
-    schema-utils "^1.0.0"
-    selfsigned "^1.10.8"
-    semver "^6.3.0"
+    connect-history-api-fallback "^2.0.0"
+    default-gateway "^6.0.3"
+    express "^4.17.3"
+    graceful-fs "^4.2.6"
+    html-entities "^2.3.2"
+    http-proxy-middleware "^2.0.3"
+    ipaddr.js "^2.0.1"
+    launch-editor "^2.6.0"
+    open "^8.0.9"
+    p-retry "^4.5.0"
+    rimraf "^3.0.2"
+    schema-utils "^4.0.0"
+    selfsigned "^2.1.1"
     serve-index "^1.9.1"
-    sockjs "^0.3.21"
-    sockjs-client "^1.5.0"
+    sockjs "^0.3.24"
     spdy "^4.0.2"
-    strip-ansi "^3.0.1"
-    supports-color "^6.1.0"
-    url "^0.11.0"
-    webpack-dev-middleware "^3.7.2"
-    webpack-log "^2.0.0"
-    ws "^6.2.1"
-    yargs "^13.3.2"
-
-webpack-log@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/webpack-log/-/webpack-log-2.0.0.tgz#5b7928e0637593f119d32f6227c1e0ac31e1b47f"
-  integrity sha512-cX8G2vR/85UYG59FgkoMamwHUIkSSlV3bBMRsbxVXVUk2j6NleCKjQ/WE9eYg9WY4w25O9w8wKP4rzNZFmUcUg==
-  dependencies:
-    ansi-colors "^3.0.0"
-    uuid "^3.3.2"
+    webpack-dev-middleware "^5.3.1"
+    ws "^8.13.0"
 
-webpack-manifest-plugin@2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/webpack-manifest-plugin/-/webpack-manifest-plugin-2.2.0.tgz#19ca69b435b0baec7e29fbe90fb4015de2de4f16"
-  integrity sha512-9S6YyKKKh/Oz/eryM1RyLVDVmy3NSPV0JXMRhZ18fJsq+AwGxUY34X54VNwkzYcEmEkDwNxuEOboCZEebJXBAQ==
+webpack-manifest-plugin@^4.0.2:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/webpack-manifest-plugin/-/webpack-manifest-plugin-4.1.1.tgz#10f8dbf4714ff93a215d5a45bcc416d80506f94f"
+  integrity sha512-YXUAwxtfKIJIKkhg03MKuiFAD72PlrqCiwdwO4VEXdRO5V0ORCNwaOwAZawPZalCbmH9kBDmXnNeQOw+BIEiow==
   dependencies:
-    fs-extra "^7.0.0"
-    lodash ">=3.5 <5"
-    object.entries "^1.1.0"
-    tapable "^1.0.0"
+    tapable "^2.0.0"
+    webpack-sources "^2.2.0"
 
-webpack-merge@^4.2.2:
-  version "4.2.2"
-  resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-4.2.2.tgz#a27c52ea783d1398afd2087f547d7b9d2f43634d"
-  integrity sha512-TUE1UGoTX2Cd42j3krGYqObZbOD+xF7u28WB7tfUordytSjbWTIjK/8V0amkBfTYN4/pB/GIDlJZZ657BGG19g==
+webpack-merge@^5.8.0:
+  version "5.9.0"
+  resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-5.9.0.tgz#dc160a1c4cf512ceca515cc231669e9ddb133826"
+  integrity sha512-6NbRQw4+Sy50vYNTw7EyOn41OZItPiXB8GNv3INSoe3PSFaHJEz3SHTrYVaRm2LilNGnFUzh0FAwqPEmU/CwDg==
   dependencies:
-    lodash "^4.17.15"
+    clone-deep "^4.0.1"
+    wildcard "^2.0.0"
 
-webpack-sources@^1.1.0, webpack-sources@^1.3.0, webpack-sources@^1.4.0, webpack-sources@^1.4.1, webpack-sources@^1.4.3:
+webpack-sources@^1.4.3:
   version "1.4.3"
   resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.4.3.tgz#eedd8ec0b928fbf1cbfe994e22d2d890f330a933"
   integrity sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==
@@ -19536,34 +16894,48 @@ webpack-sources@^1.1.0, webpack-sources@^1.3.0, webpack-sources@^1.4.0, webpack-
     source-list-map "^2.0.0"
     source-map "~0.6.1"
 
-webpack@4.44.2:
-  version "4.44.2"
-  resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.44.2.tgz#6bfe2b0af055c8b2d1e90ed2cd9363f841266b72"
-  integrity sha512-6KJVGlCxYdISyurpQ0IPTklv+DULv05rs2hseIXer6D7KrUicRDLFb4IUM1S6LUAKypPM/nSiVSuv8jHu1m3/Q==
-  dependencies:
-    "@webassemblyjs/ast" "1.9.0"
-    "@webassemblyjs/helper-module-context" "1.9.0"
-    "@webassemblyjs/wasm-edit" "1.9.0"
-    "@webassemblyjs/wasm-parser" "1.9.0"
-    acorn "^6.4.1"
-    ajv "^6.10.2"
-    ajv-keywords "^3.4.1"
+webpack-sources@^2.2.0:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-2.3.1.tgz#570de0af163949fe272233c2cefe1b56f74511fd"
+  integrity sha512-y9EI9AO42JjEcrTJFOYmVywVZdKVUfOvDUPsJea5GIr1JOEGFVqwlY2K098fFoIjOkDzHn2AjRvM8dsBZu+gCA==
+  dependencies:
+    source-list-map "^2.0.1"
+    source-map "^0.6.1"
+
+webpack-sources@^3.2.3:
+  version "3.2.3"
+  resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde"
+  integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==
+
+webpack@^5.64.4:
+  version "5.88.0"
+  resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.88.0.tgz#a07aa2f8e7a64a8f1cec0c6c2e180e3cb34440c8"
+  integrity sha512-O3jDhG5e44qIBSi/P6KpcCcH7HD+nYIHVBhdWFxcLOcIGN8zGo5nqF3BjyNCxIh4p1vFdNnreZv2h2KkoAw3lw==
+  dependencies:
+    "@types/eslint-scope" "^3.7.3"
+    "@types/estree" "^1.0.0"
+    "@webassemblyjs/ast" "^1.11.5"
+    "@webassemblyjs/wasm-edit" "^1.11.5"
+    "@webassemblyjs/wasm-parser" "^1.11.5"
+    acorn "^8.7.1"
+    acorn-import-assertions "^1.9.0"
+    browserslist "^4.14.5"
     chrome-trace-event "^1.0.2"
-    enhanced-resolve "^4.3.0"
-    eslint-scope "^4.0.3"
-    json-parse-better-errors "^1.0.2"
-    loader-runner "^2.4.0"
-    loader-utils "^1.2.3"
-    memory-fs "^0.4.1"
-    micromatch "^3.1.10"
-    mkdirp "^0.5.3"
-    neo-async "^2.6.1"
-    node-libs-browser "^2.2.1"
-    schema-utils "^1.0.0"
-    tapable "^1.1.3"
-    terser-webpack-plugin "^1.4.3"
-    watchpack "^1.7.4"
-    webpack-sources "^1.4.1"
+    enhanced-resolve "^5.15.0"
+    es-module-lexer "^1.2.1"
+    eslint-scope "5.1.1"
+    events "^3.2.0"
+    glob-to-regexp "^0.4.1"
+    graceful-fs "^4.2.9"
+    json-parse-even-better-errors "^2.3.1"
+    loader-runner "^4.2.0"
+    mime-types "^2.1.27"
+    neo-async "^2.6.2"
+    schema-utils "^3.2.0"
+    tapable "^2.1.1"
+    terser-webpack-plugin "^5.3.7"
+    watchpack "^2.4.0"
+    webpack-sources "^3.2.3"
 
 websocket-driver@>=0.5.1, websocket-driver@^0.7.4:
   version "0.7.4"
@@ -19586,7 +16958,7 @@ whatwg-encoding@^1.0.5:
   dependencies:
     iconv-lite "0.4.24"
 
-whatwg-fetch@^3.4.1:
+whatwg-fetch@^3.4.1, whatwg-fetch@^3.6.2:
   version "3.6.2"
   resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz#dced24f37f2624ed0281725d51d0e2e3fe677f8c"
   integrity sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA==
@@ -19604,6 +16976,15 @@ whatwg-url@^5.0.0:
     tr46 "~0.0.3"
     webidl-conversions "^3.0.0"
 
+whatwg-url@^7.0.0:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06"
+  integrity sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==
+  dependencies:
+    lodash.sortby "^4.7.0"
+    tr46 "^1.0.1"
+    webidl-conversions "^4.0.2"
+
 whatwg-url@^8.0.0, whatwg-url@^8.5.0:
   version "8.5.0"
   resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.5.0.tgz#7752b8464fc0903fec89aa9846fc9efe07351fd3"
@@ -19629,191 +17010,198 @@ which-module@^2.0.0:
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
   integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=
 
-which@^1.2.9, which@^1.3.1:
+which@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
   integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
   dependencies:
     isexe "^2.0.0"
 
-which@^2.0.1, which@^2.0.2:
+which@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
   integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==
   dependencies:
     isexe "^2.0.0"
 
+wildcard@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.1.tgz#5ab10d02487198954836b6349f74fff961e10f67"
+  integrity sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==
+
 word-wrap@^1.2.3, word-wrap@~1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
-workbox-background-sync@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-background-sync/-/workbox-background-sync-5.1.4.tgz#5ae0bbd455f4e9c319e8d827c055bb86c894fd12"
-  integrity sha512-AH6x5pYq4vwQvfRDWH+vfOePfPIYQ00nCEB7dJRU1e0n9+9HMRyvI63FlDvtFT2AvXVRsXvUt7DNMEToyJLpSA==
+workbox-background-sync@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-background-sync/-/workbox-background-sync-6.6.1.tgz#08d603a33717ce663e718c30cc336f74909aff2f"
+  integrity sha512-trJd3ovpWCvzu4sW0E8rV3FUyIcC0W8G+AZ+VcqzzA890AsWZlUGOTSxIMmIHVusUw/FDq1HFWfy/kC/WTRqSg==
   dependencies:
-    workbox-core "^5.1.4"
+    idb "^7.0.1"
+    workbox-core "6.6.1"
 
-workbox-broadcast-update@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-broadcast-update/-/workbox-broadcast-update-5.1.4.tgz#0eeb89170ddca7f6914fa3523fb14462891f2cfc"
-  integrity sha512-HTyTWkqXvHRuqY73XrwvXPud/FN6x3ROzkfFPsRjtw/kGZuZkPzfeH531qdUGfhtwjmtO/ZzXcWErqVzJNdXaA==
+workbox-broadcast-update@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-broadcast-update/-/workbox-broadcast-update-6.6.1.tgz#0fad9454cf8e4ace0c293e5617c64c75d8a8c61e"
+  integrity sha512-fBhffRdaANdeQ1V8s692R9l/gzvjjRtydBOvR6WCSB0BNE2BacA29Z4r9/RHd9KaXCPl6JTdI9q0bR25YKP8TQ==
   dependencies:
-    workbox-core "^5.1.4"
+    workbox-core "6.6.1"
 
-workbox-build@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-build/-/workbox-build-5.1.4.tgz#23d17ed5c32060c363030c8823b39d0eabf4c8c7"
-  integrity sha512-xUcZn6SYU8usjOlfLb9Y2/f86Gdo+fy1fXgH8tJHjxgpo53VVsqRX0lUDw8/JuyzNmXuo8vXX14pXX2oIm9Bow==
+workbox-build@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-build/-/workbox-build-6.6.1.tgz#6010e9ce550910156761448f2dbea8cfcf759cb0"
+  integrity sha512-INPgDx6aRycAugUixbKgiEQBWD0MPZqU5r0jyr24CehvNuLPSXp/wGOpdRJmts656lNiXwqV7dC2nzyrzWEDnw==
   dependencies:
-    "@babel/core" "^7.8.4"
-    "@babel/preset-env" "^7.8.4"
-    "@babel/runtime" "^7.8.4"
-    "@hapi/joi" "^15.1.0"
-    "@rollup/plugin-node-resolve" "^7.1.1"
-    "@rollup/plugin-replace" "^2.3.1"
-    "@surma/rollup-plugin-off-main-thread" "^1.1.1"
+    "@apideck/better-ajv-errors" "^0.3.1"
+    "@babel/core" "^7.11.1"
+    "@babel/preset-env" "^7.11.0"
+    "@babel/runtime" "^7.11.2"
+    "@rollup/plugin-babel" "^5.2.0"
+    "@rollup/plugin-node-resolve" "^11.2.1"
+    "@rollup/plugin-replace" "^2.4.1"
+    "@surma/rollup-plugin-off-main-thread" "^2.2.3"
+    ajv "^8.6.0"
     common-tags "^1.8.0"
     fast-json-stable-stringify "^2.1.0"
-    fs-extra "^8.1.0"
+    fs-extra "^9.0.1"
     glob "^7.1.6"
-    lodash.template "^4.5.0"
+    lodash "^4.17.20"
     pretty-bytes "^5.3.0"
-    rollup "^1.31.1"
-    rollup-plugin-babel "^4.3.3"
-    rollup-plugin-terser "^5.3.1"
-    source-map "^0.7.3"
-    source-map-url "^0.4.0"
+    rollup "^2.43.1"
+    rollup-plugin-terser "^7.0.0"
+    source-map "^0.8.0-beta.0"
     stringify-object "^3.3.0"
-    strip-comments "^1.0.2"
-    tempy "^0.3.0"
+    strip-comments "^2.0.1"
+    tempy "^0.6.0"
     upath "^1.2.0"
-    workbox-background-sync "^5.1.4"
-    workbox-broadcast-update "^5.1.4"
-    workbox-cacheable-response "^5.1.4"
-    workbox-core "^5.1.4"
-    workbox-expiration "^5.1.4"
-    workbox-google-analytics "^5.1.4"
-    workbox-navigation-preload "^5.1.4"
-    workbox-precaching "^5.1.4"
-    workbox-range-requests "^5.1.4"
-    workbox-routing "^5.1.4"
-    workbox-strategies "^5.1.4"
-    workbox-streams "^5.1.4"
-    workbox-sw "^5.1.4"
-    workbox-window "^5.1.4"
-
-workbox-cacheable-response@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-cacheable-response/-/workbox-cacheable-response-5.1.4.tgz#9ff26e1366214bdd05cf5a43da9305b274078a54"
-  integrity sha512-0bfvMZs0Of1S5cdswfQK0BXt6ulU5kVD4lwer2CeI+03czHprXR3V4Y8lPTooamn7eHP8Iywi5QjyAMjw0qauA==
-  dependencies:
-    workbox-core "^5.1.4"
-
-workbox-core@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-core/-/workbox-core-5.1.4.tgz#8bbfb2362ecdff30e25d123c82c79ac65d9264f4"
-  integrity sha512-+4iRQan/1D8I81nR2L5vcbaaFskZC2CL17TLbvWVzQ4qiF/ytOGF6XeV54pVxAvKUtkLANhk8TyIUMtiMw2oDg==
-
-workbox-expiration@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-expiration/-/workbox-expiration-5.1.4.tgz#92b5df461e8126114943a3b15c55e4ecb920b163"
-  integrity sha512-oDO/5iC65h2Eq7jctAv858W2+CeRW5e0jZBMNRXpzp0ZPvuT6GblUiHnAsC5W5lANs1QS9atVOm4ifrBiYY7AQ==
-  dependencies:
-    workbox-core "^5.1.4"
-
-workbox-google-analytics@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-google-analytics/-/workbox-google-analytics-5.1.4.tgz#b3376806b1ac7d7df8418304d379707195fa8517"
-  integrity sha512-0IFhKoEVrreHpKgcOoddV+oIaVXBFKXUzJVBI+nb0bxmcwYuZMdteBTp8AEDJacENtc9xbR0wa9RDCnYsCDLjA==
-  dependencies:
-    workbox-background-sync "^5.1.4"
-    workbox-core "^5.1.4"
-    workbox-routing "^5.1.4"
-    workbox-strategies "^5.1.4"
-
-workbox-navigation-preload@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-navigation-preload/-/workbox-navigation-preload-5.1.4.tgz#30d1b720d26a05efc5fa11503e5cc1ed5a78902a"
-  integrity sha512-Wf03osvK0wTflAfKXba//QmWC5BIaIZARU03JIhAEO2wSB2BDROWI8Q/zmianf54kdV7e1eLaIEZhth4K4MyfQ==
-  dependencies:
-    workbox-core "^5.1.4"
-
-workbox-precaching@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-precaching/-/workbox-precaching-5.1.4.tgz#874f7ebdd750dd3e04249efae9a1b3f48285fe6b"
-  integrity sha512-gCIFrBXmVQLFwvAzuGLCmkUYGVhBb7D1k/IL7pUJUO5xacjLcFUaLnnsoVepBGAiKw34HU1y/YuqvTKim9qAZA==
-  dependencies:
-    workbox-core "^5.1.4"
-
-workbox-range-requests@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-range-requests/-/workbox-range-requests-5.1.4.tgz#7066a12c121df65bf76fdf2b0868016aa2bab859"
-  integrity sha512-1HSujLjgTeoxHrMR2muDW2dKdxqCGMc1KbeyGcmjZZAizJTFwu7CWLDmLv6O1ceWYrhfuLFJO+umYMddk2XMhw==
-  dependencies:
-    workbox-core "^5.1.4"
-
-workbox-routing@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-routing/-/workbox-routing-5.1.4.tgz#3e8cd86bd3b6573488d1a2ce7385e547b547e970"
-  integrity sha512-8ljknRfqE1vEQtnMtzfksL+UXO822jJlHTIR7+BtJuxQ17+WPZfsHqvk1ynR/v0EHik4x2+826Hkwpgh4GKDCw==
-  dependencies:
-    workbox-core "^5.1.4"
-
-workbox-strategies@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-strategies/-/workbox-strategies-5.1.4.tgz#96b1418ccdfde5354612914964074d466c52d08c"
-  integrity sha512-VVS57LpaJTdjW3RgZvPwX0NlhNmscR7OQ9bP+N/34cYMDzXLyA6kqWffP6QKXSkca1OFo/v6v7hW7zrrguo6EA==
-  dependencies:
-    workbox-core "^5.1.4"
-    workbox-routing "^5.1.4"
-
-workbox-streams@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-streams/-/workbox-streams-5.1.4.tgz#05754e5e3667bdc078df2c9315b3f41210d8cac0"
-  integrity sha512-xU8yuF1hI/XcVhJUAfbQLa1guQUhdLMPQJkdT0kn6HP5CwiPOGiXnSFq80rAG4b1kJUChQQIGPrq439FQUNVrw==
-  dependencies:
-    workbox-core "^5.1.4"
-    workbox-routing "^5.1.4"
-
-workbox-sw@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-sw/-/workbox-sw-5.1.4.tgz#2bb34c9f7381f90d84cef644816d45150011d3db"
-  integrity sha512-9xKnKw95aXwSNc8kk8gki4HU0g0W6KXu+xks7wFuC7h0sembFnTrKtckqZxbSod41TDaGh+gWUA5IRXrL0ECRA==
-
-workbox-webpack-plugin@5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-webpack-plugin/-/workbox-webpack-plugin-5.1.4.tgz#7bfe8c16e40fe9ed8937080ac7ae9c8bde01e79c"
-  integrity sha512-PZafF4HpugZndqISi3rZ4ZK4A4DxO8rAqt2FwRptgsDx7NF8TVKP86/huHquUsRjMGQllsNdn4FNl8CD/UvKmQ==
-  dependencies:
-    "@babel/runtime" "^7.5.5"
-    fast-json-stable-stringify "^2.0.0"
-    source-map-url "^0.4.0"
-    upath "^1.1.2"
-    webpack-sources "^1.3.0"
-    workbox-build "^5.1.4"
-
-workbox-window@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/workbox-window/-/workbox-window-5.1.4.tgz#2740f7dea7f93b99326179a62f1cc0ca2c93c863"
-  integrity sha512-vXQtgTeMCUq/4pBWMfQX8Ee7N2wVC4Q7XYFqLnfbXJ2hqew/cU1uMTD2KqGEgEpE4/30luxIxgE+LkIa8glBYw==
+    workbox-background-sync "6.6.1"
+    workbox-broadcast-update "6.6.1"
+    workbox-cacheable-response "6.6.1"
+    workbox-core "6.6.1"
+    workbox-expiration "6.6.1"
+    workbox-google-analytics "6.6.1"
+    workbox-navigation-preload "6.6.1"
+    workbox-precaching "6.6.1"
+    workbox-range-requests "6.6.1"
+    workbox-recipes "6.6.1"
+    workbox-routing "6.6.1"
+    workbox-strategies "6.6.1"
+    workbox-streams "6.6.1"
+    workbox-sw "6.6.1"
+    workbox-window "6.6.1"
+
+workbox-cacheable-response@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-cacheable-response/-/workbox-cacheable-response-6.6.1.tgz#284c2b86be3f4fd191970ace8c8e99797bcf58e9"
+  integrity sha512-85LY4veT2CnTCDxaVG7ft3NKaFbH6i4urZXgLiU4AiwvKqS2ChL6/eILiGRYXfZ6gAwDnh5RkuDbr/GMS4KSag==
+  dependencies:
+    workbox-core "6.6.1"
+
+workbox-core@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-core/-/workbox-core-6.6.1.tgz#7184776d4134c5ed2f086878c882728fc9084265"
+  integrity sha512-ZrGBXjjaJLqzVothoE12qTbVnOAjFrHDXpZe7coCb6q65qI/59rDLwuFMO4PcZ7jcbxY+0+NhUVztzR/CbjEFw==
+
+workbox-expiration@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-expiration/-/workbox-expiration-6.6.1.tgz#a841fa36676104426dbfb9da1ef6a630b4f93739"
+  integrity sha512-qFiNeeINndiOxaCrd2DeL1Xh1RFug3JonzjxUHc5WkvkD2u5abY3gZL1xSUNt3vZKsFFGGORItSjVTVnWAZO4A==
+  dependencies:
+    idb "^7.0.1"
+    workbox-core "6.6.1"
+
+workbox-google-analytics@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-google-analytics/-/workbox-google-analytics-6.6.1.tgz#a07a6655ab33d89d1b0b0a935ffa5dea88618c5d"
+  integrity sha512-1TjSvbFSLmkpqLcBsF7FuGqqeDsf+uAXO/pjiINQKg3b1GN0nBngnxLcXDYo1n/XxK4N7RaRrpRlkwjY/3ocuA==
+  dependencies:
+    workbox-background-sync "6.6.1"
+    workbox-core "6.6.1"
+    workbox-routing "6.6.1"
+    workbox-strategies "6.6.1"
+
+workbox-navigation-preload@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-navigation-preload/-/workbox-navigation-preload-6.6.1.tgz#61a34fe125558dd88cf09237f11bd966504ea059"
+  integrity sha512-DQCZowCecO+wRoIxJI2V6bXWK6/53ff+hEXLGlQL4Rp9ZaPDLrgV/32nxwWIP7QpWDkVEtllTAK5h6cnhxNxDA==
+  dependencies:
+    workbox-core "6.6.1"
+
+workbox-precaching@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-precaching/-/workbox-precaching-6.6.1.tgz#dedeeba10a2d163d990bf99f1c2066ac0d1a19e2"
+  integrity sha512-K4znSJ7IKxCnCYEdhNkMr7X1kNh8cz+mFgx9v5jFdz1MfI84pq8C2zG+oAoeE5kFrUf7YkT5x4uLWBNg0DVZ5A==
+  dependencies:
+    workbox-core "6.6.1"
+    workbox-routing "6.6.1"
+    workbox-strategies "6.6.1"
+
+workbox-range-requests@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-range-requests/-/workbox-range-requests-6.6.1.tgz#ddaf7e73af11d362fbb2f136a9063a4c7f507a39"
+  integrity sha512-4BDzk28govqzg2ZpX0IFkthdRmCKgAKreontYRC5YsAPB2jDtPNxqx3WtTXgHw1NZalXpcH/E4LqUa9+2xbv1g==
+  dependencies:
+    workbox-core "6.6.1"
+
+workbox-recipes@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-recipes/-/workbox-recipes-6.6.1.tgz#ea70d2b2b0b0bce8de0a9d94f274d4a688e69fae"
+  integrity sha512-/oy8vCSzromXokDA+X+VgpeZJvtuf8SkQ8KL0xmRivMgJZrjwM3c2tpKTJn6PZA6TsbxGs3Sc7KwMoZVamcV2g==
+  dependencies:
+    workbox-cacheable-response "6.6.1"
+    workbox-core "6.6.1"
+    workbox-expiration "6.6.1"
+    workbox-precaching "6.6.1"
+    workbox-routing "6.6.1"
+    workbox-strategies "6.6.1"
+
+workbox-routing@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-routing/-/workbox-routing-6.6.1.tgz#cba9a1c7e0d1ea11e24b6f8c518840efdc94f581"
+  integrity sha512-j4ohlQvfpVdoR8vDYxTY9rA9VvxTHogkIDwGdJ+rb2VRZQ5vt1CWwUUZBeD/WGFAni12jD1HlMXvJ8JS7aBWTg==
+  dependencies:
+    workbox-core "6.6.1"
+
+workbox-strategies@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-strategies/-/workbox-strategies-6.6.1.tgz#38d0f0fbdddba97bd92e0c6418d0b1a2ccd5b8bf"
+  integrity sha512-WQLXkRnsk4L81fVPkkgon1rZNxnpdO5LsO+ws7tYBC6QQQFJVI6v98klrJEjFtZwzw/mB/HT5yVp7CcX0O+mrw==
+  dependencies:
+    workbox-core "6.6.1"
+
+workbox-streams@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-streams/-/workbox-streams-6.6.1.tgz#b2f7ba7b315c27a6e3a96a476593f99c5d227d26"
+  integrity sha512-maKG65FUq9e4BLotSKWSTzeF0sgctQdYyTMq529piEN24Dlu9b6WhrAfRpHdCncRS89Zi2QVpW5V33NX8PgH3Q==
+  dependencies:
+    workbox-core "6.6.1"
+    workbox-routing "6.6.1"
+
+workbox-sw@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-sw/-/workbox-sw-6.6.1.tgz#d4c4ca3125088e8b9fd7a748ed537fa0247bd72c"
+  integrity sha512-R7whwjvU2abHH/lR6kQTTXLHDFU2izht9kJOvBRYK65FbwutT4VvnUAJIgHvfWZ/fokrOPhfoWYoPCMpSgUKHQ==
+
+workbox-webpack-plugin@^6.4.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-webpack-plugin/-/workbox-webpack-plugin-6.6.1.tgz#4f81cc1ad4e5d2cd7477a86ba83c84ee2d187531"
+  integrity sha512-zpZ+ExFj9NmiI66cFEApyjk7hGsfJ1YMOaLXGXBoZf0v7Iu6hL0ZBe+83mnDq3YYWAfA3fnyFejritjOHkFcrA==
   dependencies:
-    workbox-core "^5.1.4"
-
-worker-farm@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"
-  integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==
-  dependencies:
-    errno "~0.1.7"
+    fast-json-stable-stringify "^2.1.0"
+    pretty-bytes "^5.4.1"
+    upath "^1.2.0"
+    webpack-sources "^1.4.3"
+    workbox-build "6.6.1"
 
-worker-rpc@^0.1.0:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5"
-  integrity sha512-P1WjMrUB3qgJNI9jfmpZ/htmBEjFh//6l/5y8SD9hg1Ef5zTTVVoRjTrTEzPrNBQvmhMxkoTsjOXN10GWU7aCg==
+workbox-window@6.6.1:
+  version "6.6.1"
+  resolved "https://registry.yarnpkg.com/workbox-window/-/workbox-window-6.6.1.tgz#f22a394cbac36240d0dadcbdebc35f711bb7b89e"
+  integrity sha512-wil4nwOY58nTdCvif/KEZjQ2NP8uk3gGeRNy2jPBbzypU4BT4D9L8xiwbmDBpZlSgJd2xsT9FvSNU0gsxV51JQ==
   dependencies:
-    microevent.ts "~0.1.1"
+    "@types/trusted-types" "^2.0.2"
+    workbox-core "6.6.1"
 
 wrap-ansi@^3.0.1:
   version "3.0.1"
@@ -19823,15 +17211,6 @@ wrap-ansi@^3.0.1:
     string-width "^2.1.1"
     strip-ansi "^4.0.0"
 
-wrap-ansi@^5.1.0:
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09"
-  integrity sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==
-  dependencies:
-    ansi-styles "^3.2.0"
-    string-width "^3.0.0"
-    strip-ansi "^5.0.0"
-
 wrap-ansi@^6.2.0:
   version "6.2.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53"
@@ -19877,17 +17256,15 @@ ws@^5.2.0:
   dependencies:
     async-limiter "~1.0.0"
 
-ws@^6.2.1:
-  version "6.2.1"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb"
-  integrity sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==
-  dependencies:
-    async-limiter "~1.0.0"
+ws@^7.4.6:
+  version "7.5.9"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
+  integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==
 
-ws@^7.4.5:
-  version "7.4.6"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c"
-  integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==
+ws@^8.13.0:
+  version "8.13.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.13.0.tgz#9a9fb92f93cf41512a0735c8f4dd09b8a1211cd0"
+  integrity sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==
 
 xml-name-validator@^3.0.0:
   version "3.0.0"
@@ -19899,7 +17276,7 @@ xmlchars@^2.2.0:
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
   integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
 
-xtend@^4.0.0, xtend@~4.0.1:
+xtend@^4.0.0:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
@@ -19944,11 +17321,16 @@ yaml-ast-parser@^0.0.43:
   resolved "https://registry.yarnpkg.com/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb"
   integrity sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==
 
-yaml@^1.10.0, yaml@^1.7.2:
+yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2:
   version "1.10.2"
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"
   integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==
 
+yaml@^2.1.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.1.tgz#02fe0975d23cd441242aa7204e09fc28ac2ac33b"
+  integrity sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==
+
 yamljs@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/yamljs/-/yamljs-0.3.0.tgz#dc060bf267447b39f7304e9b2bfbe8b5a7ddb03b"
@@ -19957,14 +17339,6 @@ yamljs@^0.3.0:
     argparse "^1.0.7"
     glob "^7.0.5"
 
-yargs-parser@^13.1.2:
-  version "13.1.2"
-  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38"
-  integrity sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==
-  dependencies:
-    camelcase "^5.0.0"
-    decamelize "^1.2.0"
-
 yargs-parser@^18.1.2:
   version "18.1.3"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
@@ -19978,23 +17352,7 @@ yargs-parser@^20.2.2:
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.7.tgz#61df85c113edfb5a7a4e36eb8aa60ef423cbc90a"
   integrity sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==
 
-yargs@^13.3.2:
-  version "13.3.2"
-  resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd"
-  integrity sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==
-  dependencies:
-    cliui "^5.0.0"
-    find-up "^3.0.0"
-    get-caller-file "^2.0.1"
-    require-directory "^2.1.1"
-    require-main-filename "^2.0.0"
-    set-blocking "^2.0.0"
-    string-width "^3.0.0"
-    which-module "^2.0.0"
-    y18n "^4.0.0"
-    yargs-parser "^13.1.2"
-
-yargs@^15.3.1, yargs@^15.4.1:
+yargs@^15.3.1:
   version "15.4.1"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.4.1.tgz#0d87a16de01aee9d8bec2bfbf74f67851730f4f8"
   integrity sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==

From 27127eb9148d0fbff89869c274f253e4d5d93eda Mon Sep 17 00:00:00 2001
From: Chris Collins <chriscollins3456@gmail.com>
Date: Mon, 27 Nov 2023 15:50:33 -0500
Subject: [PATCH 254/259] fix(java) Fixes NPE ES service (#9311)

---
 .../metadata/search/elasticsearch/ElasticSearchService.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
index 9b43642d7621c..68a5483fa469c 100644
--- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
+++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
@@ -144,7 +144,7 @@ public AutoCompleteResult autoComplete(@Nonnull String entityName, @Nonnull Stri
   @Override
   public Map<String, Long> aggregateByValue(@Nullable List<String> entityNames, @Nonnull String field,
       @Nullable Filter requestParams, int limit) {
-    log.debug("Aggregating by value: {}, field: {}, requestParams: {}, limit: {}", entityNames.toString(), field,
+    log.debug("Aggregating by value: {}, field: {}, requestParams: {}, limit: {}", entityNames != null ? entityNames.toString() : null, field,
         requestParams, limit);
     return esSearchDAO.aggregateByValue(entityNames, field, requestParams, limit);
   }

From 03be68ca789a80cc7ce2164b384050aa528e9efd Mon Sep 17 00:00:00 2001
From: skrydal <piotr.skrydalewicz@gmail.com>
Date: Tue, 28 Nov 2023 05:59:28 +0100
Subject: [PATCH 255/259] feat(config): Configurable bootstrap of ownership
 types (#9308)

---
 .../src/main/resources/application.yml        |  2 +
 .../factories/BootstrapManagerFactory.java    |  5 ++-
 .../boot/steps/IngestOwnershipTypesStep.java  | 44 ++++++-------------
 3 files changed, 19 insertions(+), 32 deletions(-)

diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 571cb66c84aa8..a52b705cb8da6 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -292,6 +292,8 @@ bootstrap:
     file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
     # eg for local file
     # file: "file:///datahub/datahub-gms/resources/custom-policies.json"
+  ownershipTypes:
+    file: ${BOOTSTRAP_OWNERSHIP_TYPES_FILE:classpath:boot/ownership_types.json}
   servlets:
     waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize
 
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
index 3a761bd12647e..c4e6c941303c8 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
@@ -94,6 +94,9 @@ public class BootstrapManagerFactory {
   @Value("${bootstrap.policies.file}")
   private Resource _policiesResource;
 
+  @Value("${bootstrap.ownershipTypes.file}")
+  private Resource _ownershipTypesResource;
+
   @Bean(name = "bootstrapManager")
   @Scope("singleton")
   @Nonnull
@@ -116,7 +119,7 @@ protected BootstrapManager createInstance() {
     final IngestDefaultGlobalSettingsStep ingestSettingsStep = new IngestDefaultGlobalSettingsStep(_entityService);
     final WaitForSystemUpdateStep waitForSystemUpdateStep = new WaitForSystemUpdateStep(_dataHubUpgradeKafkaListener,
         _configurationProvider);
-    final IngestOwnershipTypesStep ingestOwnershipTypesStep = new IngestOwnershipTypesStep(_entityService);
+    final IngestOwnershipTypesStep ingestOwnershipTypesStep = new IngestOwnershipTypesStep(_entityService, _ownershipTypesResource);
 
     final List<BootstrapStep> finalSteps = new ArrayList<>(ImmutableList.of(
         waitForSystemUpdateStep,
diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java
index 55d612618ff9f..6d64ceea32339 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java
@@ -7,7 +7,7 @@
 import com.linkedin.common.urn.Urn;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
-import com.linkedin.metadata.boot.UpgradeStep;
+import com.linkedin.metadata.boot.BootstrapStep;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.transactions.AspectsBatchImpl;
 import com.linkedin.metadata.models.AspectSpec;
@@ -16,9 +16,10 @@
 import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.ownership.OwnershipTypeInfo;
-import javax.annotation.Nonnull;
+
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
 
 import java.util.List;
 
@@ -33,17 +34,12 @@
  * If not, it ingests the ownership type into DataHub.
  */
 @Slf4j
-public class IngestOwnershipTypesStep extends UpgradeStep {
-
-  private static final String UPGRADE_ID = "ingest-default-metadata-ownership-types";
-  private static final String VERSION = "1";
-  private static final int SLEEP_SECONDS = 60;
+@RequiredArgsConstructor
+public class IngestOwnershipTypesStep implements BootstrapStep {
 
   private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
-
-  public IngestOwnershipTypesStep(EntityService entityService) {
-    super(entityService, VERSION, UPGRADE_ID);
-  }
+  private final EntityService _entityService;
+  private final Resource _ownershipTypesResource;
 
   @Override
   public String name() {
@@ -51,24 +47,17 @@ public String name() {
   }
 
   @Override
-  public void upgrade() throws Exception {
-    log.info("Ingesting default ownership types...");
-
-    // Sleep to ensure deployment process finishes.
-    Thread.sleep(SLEEP_SECONDS * 1000);
+  public void execute() throws Exception {
+    log.info("Ingesting default ownership types from {}...", _ownershipTypesResource);
 
     // 1. Read from the file into JSON.
-    final JsonNode ownershipTypesObj = JSON_MAPPER.readTree(new ClassPathResource("./boot/ownership_types.json")
-        .getFile());
+    final JsonNode ownershipTypesObj = JSON_MAPPER.readTree(_ownershipTypesResource.getFile());
 
     if (!ownershipTypesObj.isArray()) {
       throw new RuntimeException(String.format("Found malformed ownership file, expected an Array but found %s",
           ownershipTypesObj.getNodeType()));
     }
 
-    final AspectSpec ownershipTypeInfoAspectSpec = _entityService.getEntityRegistry()
-        .getEntitySpec(OWNERSHIP_TYPE_ENTITY_NAME)
-        .getAspectSpec(OWNERSHIP_TYPE_INFO_ASPECT_NAME);
     final AuditStamp auditStamp =
         new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis());
 
@@ -79,14 +68,13 @@ public void upgrade() throws Exception {
       final OwnershipTypeInfo info = RecordUtils.toRecordTemplate(OwnershipTypeInfo.class, roleObj.get("info")
           .toString());
       log.info(String.format("Ingesting default ownership type with urn %s", urn));
-      ingestOwnershipType(urn, info, auditStamp, ownershipTypeInfoAspectSpec);
+      ingestOwnershipType(urn, info, auditStamp);
       numIngested++;
     }
     log.info("Ingested {} new ownership types", numIngested);
   }
 
-  private void ingestOwnershipType(final Urn ownershipTypeUrn, final OwnershipTypeInfo info, final AuditStamp auditStamp,
-      final AspectSpec ownershipTypeInfoAspectSpec) {
+  private void ingestOwnershipType(final Urn ownershipTypeUrn, final OwnershipTypeInfo info, final AuditStamp auditStamp) {
 
     // 3. Write key & aspect MCPs.
     final MetadataChangeProposal keyAspectProposal = new MetadataChangeProposal();
@@ -112,10 +100,4 @@ private void ingestOwnershipType(final Urn ownershipTypeUrn, final OwnershipType
             .mcps(List.of(keyAspectProposal, proposal), _entityService.getEntityRegistry()).build(), auditStamp,
             false);
   }
-
-  @Nonnull
-  @Override
-  public ExecutionMode getExecutionMode() {
-    return ExecutionMode.ASYNC;
-  }
 }

From ac7fa5624ffab4b369e66977e21cc430b468ce2d Mon Sep 17 00:00:00 2001
From: kushagra-apptware <81357546+kushagra-apptware@users.noreply.github.com>
Date: Tue, 28 Nov 2023 10:45:14 +0530
Subject: [PATCH 256/259] fix(sec): update the "json-schema" version from
 package.json to solve json-schema vulnerability (#9289)

---
 datahub-web-react/package.json | 3 ++-
 datahub-web-react/yarn.lock    | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index f55588e46c9c7..62186125b4ad2 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -148,6 +148,7 @@
     },
     "resolutions": {
         "@ant-design/colors": "6.0.0",
-        "refractor": "3.3.1"
+        "refractor": "3.3.1",
+        "json-schema": "0.4.0"
     }
 }
diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
index fbc800c93c460..b9a6c62c88de3 100644
--- a/datahub-web-react/yarn.lock
+++ b/datahub-web-react/yarn.lock
@@ -10793,10 +10793,10 @@ json-schema-traverse@^1.0.0:
   resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
   integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
 
-json-schema@0.2.3:
-  version "0.2.3"
-  resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
-  integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=
+json-schema@0.2.3, json-schema@0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5"
+  integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==
 
 json-schema@^0.4.0:
   version "0.4.0"

From 63257298ad4702c0f65de5d93be27c84cde32988 Mon Sep 17 00:00:00 2001
From: Raj Tekal <varadaraj_tekal@optum.com>
Date: Wed, 3 Jan 2024 13:22:17 -0500
Subject: [PATCH 257/259] Changed Join to ERModelRelation - build compiles

---
 .../datahub/graphql/GmsGraphQLEngine.java     |  46 +++---
 .../datahub/graphql/GmsGraphQLEngineArgs.java |   4 +-
 .../graphql/featureflags/FeatureFlags.java    |   2 +-
 .../graphql/resolvers/EntityTypeMapper.java   |   2 +-
 .../resolvers/config/AppConfigResolver.java   |   6 +-
 .../common/mappers/UrnToEntityMapper.java     |  10 +-
 .../CreateERModelRelationResolver.java        |  89 +++++++++++
 .../ERModelRelationType.java}                 |  56 +++----
 .../UpdateERModelRelationResolver.java}       |  26 ++--
 .../mappers/ERModelRelationMapper.java        | 143 ++++++++++++++++++
 .../ERModelRelationUpdateInputMapper.java     | 127 ++++++++++++++++
 .../types/join/CreateJoinResolver.java        |  89 -----------
 .../types/join/mappers/JoinMapper.java        | 142 -----------------
 .../join/mappers/JoinUpdateInputMapper.java   | 127 ----------------
 .../src/main/resources/app.graphql            |   4 +-
 .../src/main/resources/entity.graphql         | 118 +++++++--------
 datahub-web-react/src/App.tsx                 |   4 +-
 .../ERModelRelationEntity.tsx}                |  48 +++---
 .../preview/ERModelRelationAction.less}       |   0
 .../preview/ERModelRelationPreviewCard.tsx}   |  16 +-
 .../CreateERModelRelationModal.less}          |  12 +-
 .../CreateERModelRelationModal.tsx}           | 128 ++++++++--------
 .../ERModelRelationPreview.less}              |   4 +-
 .../ERModelRelationPreview.tsx}               |  88 +++++------
 .../ERModelRelationUtils.tsx}                 |  20 +--
 .../EditableCell.tsx                          |  12 +-
 .../{Join => ERModelRelation}/EditableRow.tsx |   2 +-
 .../containers/profile/EntityProfile.tsx      |   4 +-
 .../tabs/Dataset/Queries/QueryBuilderForm.tsx |   2 +-
 .../Dataset/Relationship/RelationshipsTab.tsx |  72 ++++-----
 .../ERModelRelationTab.less}                  |   4 +-
 .../ERModelRelation/ERModelRelationTab.tsx    |  21 +++
 .../app/entity/shared/tabs/Join/JoinTab.tsx   |  21 ---
 datahub-web-react/src/appConfigContext.tsx    |   2 +-
 datahub-web-react/src/graphql/app.graphql     |   2 +-
 .../{join.graphql => ermodelrelation.graphql} |  24 +--
 .../src/graphql/fragments.graphql             |  16 +-
 datahub-web-react/src/graphql/lineage.graphql |   8 +-
 datahub-web-react/src/graphql/search.graphql  |   8 +-
 .../{joinIcon.svg => ermodelrelationIcon.svg} |   0
 docker/docker-compose.dev.yml                 |   2 +-
 docs/deploy/environment-vars.md               |   4 +-
 .../java/com/linkedin/metadata/Constants.java |  10 +-
 .../common/urn/ERModelRelationUrn.java        |  66 ++++++++
 .../com/linkedin/common/urn/JoinUrn.java      |  66 --------
 .../linkedin/common/ERModelRelationUrn.pdl    |  24 +++
 .../pegasus/com/linkedin/common/JoinUrn.pdl   |  24 ---
 .../ERModelRelationFieldMapping.pdl           |  15 ++
 .../ERModelRelationProperties.pdl}            |  22 +--
 .../EditableERModelRelationProperties.pdl}    |  12 +-
 .../{join => ermodelrelation}/FieldMap.pdl    |   6 +-
 .../com/linkedin/join/JoinFieldMapping.pdl    |  15 --
 .../metadata/key/ERModelRelationKey.pdl       |  14 ++
 .../com/linkedin/metadata/key/JoinKey.pdl     |  14 --
 .../src/main/resources/entity-registry.yml    |  10 +-
 .../src/main/resources/application.yml        |   2 +-
 .../ERModelRelationServiceFactory.java}       |  12 +-
 .../factory/graphql/GraphQLEngineFactory.java |   8 +-
 ...rvice.java => ERModelRelationService.java} |  28 ++--
 .../war/src/main/resources/boot/policies.json |   2 +-
 .../authorization/PoliciesConfig.java         |  22 +--
 61 files changed, 944 insertions(+), 943 deletions(-)
 create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java
 rename datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/{join/JoinType.java => ermodelrelation/ERModelRelationType.java} (78%)
 rename datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/{join/UpdateJoinResolver.java => ermodelrelation/UpdateERModelRelationResolver.java} (63%)
 create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationMapper.java
 create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationUpdateInputMapper.java
 delete mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/CreateJoinResolver.java
 delete mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java
 delete mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java
 rename datahub-web-react/src/app/entity/{joins/JoinEntity.tsx => ermodelrelations/ERModelRelationEntity.tsx} (70%)
 rename datahub-web-react/src/app/entity/{joins/preview/JoinAction.less => ermodelrelations/preview/ERModelRelationAction.less} (100%)
 rename datahub-web-react/src/app/entity/{joins/preview/JoinPreviewCard.tsx => ermodelrelations/preview/ERModelRelationPreviewCard.tsx} (74%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join/CreateJoinModal.less => ERModelRelation/CreateERModelRelationModal.less} (97%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join/CreateJoinModal.tsx => ERModelRelation/CreateERModelRelationModal.tsx} (70%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join/JoinPreview.less => ERModelRelation/ERModelRelationPreview.less} (98%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join/JoinPreview.tsx => ERModelRelation/ERModelRelationPreview.tsx} (63%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join/JoinUtils.tsx => ERModelRelation/ERModelRelationUtils.tsx} (60%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join => ERModelRelation}/EditableCell.tsx (87%)
 rename datahub-web-react/src/app/entity/shared/components/styled/{Join => ERModelRelation}/EditableRow.tsx (88%)
 rename datahub-web-react/src/app/entity/shared/tabs/{Join/JoinTab.less => ERModelRelation/ERModelRelationTab.less} (94%)
 create mode 100644 datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.tsx
 delete mode 100644 datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.tsx
 rename datahub-web-react/src/graphql/{join.graphql => ermodelrelation.graphql} (53%)
 rename datahub-web-react/src/images/{joinIcon.svg => ermodelrelationIcon.svg} (100%)
 create mode 100644 li-utils/src/main/javaPegasus/com/linkedin/common/urn/ERModelRelationUrn.java
 delete mode 100644 li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java
 create mode 100644 li-utils/src/main/pegasus/com/linkedin/common/ERModelRelationUrn.pdl
 delete mode 100644 li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationFieldMapping.pdl
 rename metadata-models/src/main/pegasus/com/linkedin/{join/JoinProperties.pdl => ermodelrelation/ERModelRelationProperties.pdl} (69%)
 rename metadata-models/src/main/pegasus/com/linkedin/{join/EditableJoinProperties.pdl => ermodelrelation/EditableERModelRelationProperties.pdl} (54%)
 rename metadata-models/src/main/pegasus/com/linkedin/{join => ermodelrelation}/FieldMap.pdl (54%)
 delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl
 create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/ERModelRelationKey.pdl
 delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl
 rename metadata-service/factories/src/main/java/com/linkedin/gms/factory/{join/JoinServiceFactory.java => ermodelrelation/ERModelRelationServiceFactory.java} (72%)
 rename metadata-service/services/src/main/java/com/linkedin/metadata/service/{JoinService.java => ERModelRelationService.java} (64%)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index 59f6939298cd0..2953538078998 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -60,8 +60,8 @@
 import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation;
 import com.linkedin.datahub.graphql.generated.IngestionSource;
 import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
-import com.linkedin.datahub.graphql.generated.Join;
-import com.linkedin.datahub.graphql.generated.JoinProperties;
+import com.linkedin.datahub.graphql.generated.ERModelRelation;
+import com.linkedin.datahub.graphql.generated.ERModelRelationProperties;
 import com.linkedin.datahub.graphql.generated.LineageRelationship;
 import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
 import com.linkedin.datahub.graphql.generated.ListDomainsResult;
@@ -289,8 +289,8 @@
 import com.linkedin.datahub.graphql.types.rolemetadata.RoleType;
 import com.linkedin.datahub.graphql.types.glossary.GlossaryNodeType;
 import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType;
-import com.linkedin.datahub.graphql.types.join.CreateJoinResolver;
-import com.linkedin.datahub.graphql.types.join.UpdateJoinResolver;
+import com.linkedin.datahub.graphql.types.ermodelrelation.CreateERModelRelationResolver;
+import com.linkedin.datahub.graphql.types.ermodelrelation.UpdateERModelRelationResolver;
 import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureTableType;
 import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureType;
 import com.linkedin.datahub.graphql.types.mlmodel.MLModelGroupType;
@@ -305,7 +305,7 @@
 import com.linkedin.datahub.graphql.types.tag.TagType;
 import com.linkedin.datahub.graphql.types.test.TestType;
 import com.linkedin.datahub.graphql.types.view.DataHubViewType;
-import com.linkedin.datahub.graphql.types.join.JoinType;
+import com.linkedin.datahub.graphql.types.ermodelrelation.ERModelRelationType;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.metadata.config.DataHubConfiguration;
@@ -328,7 +328,7 @@
 import com.linkedin.metadata.service.QueryService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
-import com.linkedin.metadata.service.JoinService;
+import com.linkedin.metadata.service.ERModelRelationService;
 import com.linkedin.metadata.timeline.TimelineService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.version.GitVersion;
@@ -440,12 +440,12 @@ public class GmsGraphQLEngine {
     private final DataHubPolicyType dataHubPolicyType;
     private final DataHubRoleType dataHubRoleType;
     private final SchemaFieldType schemaFieldType;
-    private final JoinType joinType;
+    private final ERModelRelationType ermodelrelationType;
     private final DataHubViewType dataHubViewType;
     private final QueryType queryType;
     private final DataProductType dataProductType;
     private final OwnershipType ownershipType;
-    private final JoinService joinService;
+    private final ERModelRelationService ermodelrelationService;
 
     /**
      * A list of GraphQL Plugins that extend the core engine
@@ -511,7 +511,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
         this.settingsService = args.settingsService;
         this.lineageService = args.lineageService;
         this.queryService = args.queryService;
-        this.joinService = args.joinService;
+        this.ermodelrelationService = args.ermodelrelationService;
         this.dataProductService = args.dataProductService;
 
         this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration);
@@ -553,7 +553,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
         this.dataHubPolicyType = new DataHubPolicyType(entityClient);
         this.dataHubRoleType = new DataHubRoleType(entityClient);
         this.schemaFieldType = new SchemaFieldType();
-        this.joinType = new JoinType(entityClient, featureFlags);
+        this.ermodelrelationType = new ERModelRelationType(entityClient, featureFlags);
         this.dataHubViewType = new DataHubViewType(entityClient);
         this.queryType = new QueryType(entityClient);
         this.dataProductType = new DataProductType(entityClient);
@@ -589,7 +589,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
             dataHubPolicyType,
             dataHubRoleType,
             schemaFieldType,
-            joinType,
+            ermodelrelationType,
             dataHubViewType,
             queryType,
             dataProductType,
@@ -672,7 +672,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
         configureTestResultResolvers(builder);
         configureRoleResolvers(builder);
         configureSchemaFieldResolvers(builder);
-        configureJoinResolvers(builder);
+        configureERModelRelationResolvers(builder);
         configureEntityPathResolvers(builder);
         configureViewResolvers(builder);
         configureQueryEntityResolvers(builder);
@@ -832,7 +832,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("glossaryTerm", getResolver(glossaryTermType))
             .dataFetcher("glossaryNode", getResolver(glossaryNodeType))
             .dataFetcher("domain", getResolver((domainType)))
-            .dataFetcher("join", getResolver(joinType))
+            .dataFetcher("ermodelrelation", getResolver(ermodelrelationType))
             .dataFetcher("dataPlatform", getResolver(dataPlatformType))
             .dataFetcher("dataPlatformInstance", getResolver(dataPlatformInstanceType))
             .dataFetcher("mlFeatureTable", getResolver(mlFeatureTableType))
@@ -936,8 +936,8 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("updateDataFlow", new MutableTypeResolver<>(dataFlowType))
             .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType))
             .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType))
-            .dataFetcher("updateJoin", new UpdateJoinResolver(this.entityClient))
-            .dataFetcher("createJoin", new CreateJoinResolver(this.entityClient, this.joinService))
+            .dataFetcher("updateERModelRelation", new UpdateERModelRelationResolver(this.entityClient))
+            .dataFetcher("createERModelRelation", new CreateERModelRelationResolver(this.entityClient, this.ermodelrelationService))
             .dataFetcher("addTag", new AddTagResolver(entityService))
             .dataFetcher("addTags", new AddTagsResolver(entityService))
             .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService))
@@ -1510,25 +1510,25 @@ private void configureTypeExtensions(final RuntimeWiring.Builder builder) {
     }
 
     /**
-     * Configures resolvers responsible for resolving the {@link Join} type.
+     * Configures resolvers responsible for resolving the {@link ERModelRelation} type.
      */
-    private void configureJoinResolvers(final RuntimeWiring.Builder builder) {
+    private void configureERModelRelationResolvers(final RuntimeWiring.Builder builder) {
         builder
-            .type("Join", typeWiring -> typeWiring
+            .type("ERModelRelation", typeWiring -> typeWiring
                 .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
                 .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)))
-            .type("JoinProperties", typeWiring -> typeWiring
+            .type("ERModelRelationProperties", typeWiring -> typeWiring
                     .dataFetcher("datasetA",
                             new LoadableTypeResolver<>(datasetType,
                                     (env) -> {
-                                        final JoinProperties joinProperties = env.getSource();
-                                        return joinProperties.getDatasetA() != null ? joinProperties.getDatasetA().getUrn() : null;
+                                        final ERModelRelationProperties ermodelrelationProperties = env.getSource();
+                                        return ermodelrelationProperties.getDatasetA() != null ? ermodelrelationProperties.getDatasetA().getUrn() : null;
                                     }))
                     .dataFetcher("datasetB",
                             new LoadableTypeResolver<>(datasetType,
                                     (env) -> {
-                                        final JoinProperties joinProperties = env.getSource();
-                                        return joinProperties.getDatasetB() != null ? joinProperties.getDatasetB().getUrn() : null;
+                                        final ERModelRelationProperties ermodelrelationProperties = env.getSource();
+                                        return ermodelrelationProperties.getDatasetB() != null ? ermodelrelationProperties.getDatasetB().getUrn() : null;
                                     }))
                     )
             .type("Owner", typeWiring -> typeWiring
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java
index 115a2f7a369e0..c5e34151ae2d8 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java
@@ -28,7 +28,7 @@
 import com.linkedin.metadata.service.LineageService;
 import com.linkedin.metadata.service.OwnershipTypeService;
 import com.linkedin.metadata.service.QueryService;
-import com.linkedin.metadata.service.JoinService;
+import com.linkedin.metadata.service.ERModelRelationService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
 import com.linkedin.metadata.timeline.TimelineService;
@@ -74,6 +74,6 @@ public class GmsGraphQLEngineArgs {
     QueryService queryService;
     FeatureFlags featureFlags;
     DataProductService dataProductService;
-    JoinService joinService;
+    ERModelRelationService ermodelrelationService;
     //any fork specific args should go below this line
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
index c4b3d01e7d2a7..9707616124f55 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java
@@ -15,7 +15,7 @@ public class FeatureFlags {
   private boolean showBrowseV2 = false;
   private PreProcessHooks preProcessHooks;
   private boolean showAcrylInfo = false;
-  private boolean joinFeatureEnabled = false;
+  private boolean ermodelrelationFeatureEnabled = false;
   private boolean showAccessManagement = false;
   private boolean nestedDomainsEnabled = false;
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java
index 465bea608803c..e0cc9e73b1741 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java
@@ -39,7 +39,7 @@ public class EntityTypeMapper {
           .put(EntityType.NOTEBOOK, "notebook")
           .put(EntityType.DATA_PLATFORM_INSTANCE, "dataPlatformInstance")
           .put(EntityType.TEST, "test")
-          .put(EntityType.JOIN, Constants.JOIN_ENTITY_NAME)
+          .put(EntityType.ERMODELRELATION, Constants.ERMODELRELATION_ENTITY_NAME)
           .put(EntityType.DATAHUB_VIEW, Constants.DATAHUB_VIEW_ENTITY_NAME)
           .put(EntityType.DATA_PRODUCT, Constants.DATA_PRODUCT_ENTITY_NAME)
           .build();
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
index 3b2b0d8f50c15..ea9e9142e94f9 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java
@@ -171,7 +171,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
       .setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled())
       .setShowBrowseV2(_featureFlags.isShowBrowseV2())
       .setShowAcrylInfo(_featureFlags.isShowAcrylInfo())
-      .setJoinFeatureEnabled(_featureFlags.isJoinFeatureEnabled())
+      .setErmodelrelationFeatureEnabled(_featureFlags.isErmodelrelationFeatureEnabled())
       .setShowAccessManagement(_featureFlags.isShowAccessManagement())
       .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled())
       .build();
@@ -227,8 +227,8 @@ private EntityType mapResourceTypeToEntityType(final String resourceType) {
       return EntityType.CORP_GROUP;
     } else if (com.linkedin.metadata.authorization.PoliciesConfig.CORP_USER_PRIVILEGES.getResourceType().equals(resourceType)) {
       return EntityType.CORP_USER;
-    } else if (com.linkedin.metadata.authorization.PoliciesConfig.JOIN_PRIVILEGES.getResourceType().equals(resourceType)) {
-      return EntityType.JOIN;
+    } else if (com.linkedin.metadata.authorization.PoliciesConfig.ERMODELRELATION_PRIVILEGES.getResourceType().equals(resourceType)) {
+      return EntityType.ERMODELRELATION;
     } else {
       return null;
     }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java
index 9cb6bbf9c3bde..4c22a9e090838 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java
@@ -21,7 +21,7 @@
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.GlossaryNode;
 import com.linkedin.datahub.graphql.generated.GlossaryTerm;
-import com.linkedin.datahub.graphql.generated.Join;
+import com.linkedin.datahub.graphql.generated.ERModelRelation;
 import com.linkedin.datahub.graphql.generated.MLFeature;
 import com.linkedin.datahub.graphql.generated.MLFeatureTable;
 import com.linkedin.datahub.graphql.generated.MLModel;
@@ -154,10 +154,10 @@ public Entity apply(Urn input) {
       ((Domain) partialEntity).setUrn(input.toString());
       ((Domain) partialEntity).setType(EntityType.DOMAIN);
     }
-    if (input.getEntityType().equals("join")) {
-      partialEntity = new Join();
-      ((Join) partialEntity).setUrn(input.toString());
-      ((Join) partialEntity).setType(EntityType.JOIN);
+    if (input.getEntityType().equals("ermodelrelation")) {
+      partialEntity = new ERModelRelation();
+      ((ERModelRelation) partialEntity).setUrn(input.toString());
+      ((ERModelRelation) partialEntity).setType(EntityType.ERMODELRELATION);
     }
     if (input.getEntityType().equals("assertion")) {
       partialEntity = new Assertion();
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java
new file mode 100644
index 0000000000000..021bdd0603110
--- /dev/null
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java
@@ -0,0 +1,89 @@
+package com.linkedin.datahub.graphql.types.ermodelrelation;
+
+import com.datahub.authentication.Authentication;
+import com.linkedin.common.urn.CorpuserUrn;
+import com.linkedin.common.urn.ERModelRelationUrn;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.exception.AuthorizationException;
+import com.linkedin.datahub.graphql.generated.ERModelRelationPropertiesInput;
+import com.linkedin.datahub.graphql.generated.ERModelRelationUpdateInput;
+import com.linkedin.datahub.graphql.types.ermodelrelation.mappers.ERModelRelationMapper;
+import com.linkedin.datahub.graphql.types.ermodelrelation.mappers.ERModelRelationUpdateInputMapper;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.metadata.service.ERModelRelationService;
+import com.linkedin.mxe.MetadataChangeProposal;
+import com.linkedin.r2.RemoteInvocationException;
+import graphql.schema.DataFetcher;
+import graphql.schema.DataFetchingEnvironment;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.concurrent.CompletableFuture;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import com.linkedin.datahub.graphql.generated.ERModelRelation;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
+
+
+@Slf4j
+@RequiredArgsConstructor
+public class CreateERModelRelationResolver implements DataFetcher<CompletableFuture<ERModelRelation>> {
+
+    private final EntityClient _entityClient;
+    private final ERModelRelationService _ermodelrelationService;
+
+    @Override
+    public CompletableFuture<ERModelRelation> get(DataFetchingEnvironment environment) throws Exception {
+        final ERModelRelationUpdateInput input = bindArgument(environment.getArgument("input"), ERModelRelationUpdateInput.class);
+
+        final ERModelRelationPropertiesInput ermodelrelationPropertiesInput = input.getProperties();
+        String ermodelrelationName = ermodelrelationPropertiesInput.getName();
+        String datasetA = ermodelrelationPropertiesInput.getDataSetA();
+        String datasetB = ermodelrelationPropertiesInput.getDatasetB();
+
+        String lowDataset =  datasetA;
+        String highDataset = datasetB;
+        if (datasetA.compareTo(datasetB)  > 0) {
+            lowDataset = datasetB;
+            highDataset = datasetA;
+        }
+        // The following sequence mimics datahub.emitter.mce_builder.datahub_guid
+
+        String ermodelrelationKey =
+            "{\"DatasetA\":\"" + lowDataset + "\",\"DatasetB\":\"" + highDataset + "\",\"ERModelRelationName\":\"" + ermodelrelationName
+                + "\"}";
+
+        byte[] mybytes = ermodelrelationKey.getBytes(StandardCharsets.UTF_8);
+
+        String ermodelrelationKeyEncoded = new String(mybytes, StandardCharsets.UTF_8);
+        String ermodelrelationGuid = DigestUtils.md5Hex(ermodelrelationKeyEncoded);
+        log.info("ermodelrelationkey {}, ermodelrelationGuid {}", ermodelrelationKeyEncoded, ermodelrelationGuid);
+
+        ERModelRelationUrn inputUrn = new ERModelRelationUrn(ermodelrelationGuid);
+        QueryContext context = environment.getContext();
+        final Authentication authentication = context.getAuthentication();
+        final CorpuserUrn actor = CorpuserUrn.createFromString(context.getActorUrn());
+        if (!ERModelRelationType.canCreateERModelRelation(context, Urn.createFromString(input.getProperties().getDataSetA()),
+                Urn.createFromString(input.getProperties().getDatasetB()))) {
+            throw new AuthorizationException("Unauthorized to create ermodelrelation. Please contact your DataHub administrator.");
+        }
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                log.debug("Create ERModelRelation input: {}", input);
+                final Collection<MetadataChangeProposal> proposals = ERModelRelationUpdateInputMapper.map(input, actor);
+                proposals.forEach(proposal -> proposal.setEntityUrn(inputUrn));
+                try {
+                    _entityClient.batchIngestProposals(proposals, context.getAuthentication(), false);
+                } catch (RemoteInvocationException e) {
+                    throw new RuntimeException("Failed to create ermodelrelation entity", e);
+                }
+                return ERModelRelationMapper.map(_ermodelrelationService.getERModelRelationResponse(Urn.createFromString(inputUrn.toString()), authentication));
+            } catch (Exception e) {
+                log.error("Failed to create ERModelRelation to resource with input {}, {}", input, e.getMessage());
+                throw new RuntimeException(String.format("Failed to create ermodelrelation to resource with input %s", input), e);
+            }
+        });
+    }
+}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/ERModelRelationType.java
similarity index 78%
rename from datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java
rename to datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/ERModelRelationType.java
index b61857b966c86..a9d933ab81c68 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/ERModelRelationType.java
@@ -1,10 +1,10 @@
-package com.linkedin.datahub.graphql.types.join;
+package com.linkedin.datahub.graphql.types.ermodelrelation;
 
 import com.datahub.authorization.ConjunctivePrivilegeGroup;
 import com.datahub.authorization.DisjunctivePrivilegeGroup;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.linkedin.common.urn.JoinUrn;
+import com.linkedin.common.urn.ERModelRelationUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.StringArray;
@@ -17,13 +17,13 @@
 import com.linkedin.datahub.graphql.generated.Entity;
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
-import com.linkedin.datahub.graphql.generated.Join;
-import com.linkedin.datahub.graphql.generated.JoinUpdateInput;
+import com.linkedin.datahub.graphql.generated.ERModelRelation;
+import com.linkedin.datahub.graphql.generated.ERModelRelationUpdateInput;
 import com.linkedin.datahub.graphql.generated.SearchResults;
 import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.BrowsableEntityType;
 import com.linkedin.datahub.graphql.types.SearchableEntityType;
-import com.linkedin.datahub.graphql.types.join.mappers.JoinMapper;
+import com.linkedin.datahub.graphql.types.ermodelrelation.mappers.ERModelRelationMapper;
 import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper;
 import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper;
 import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper;
@@ -51,14 +51,14 @@
 import static com.linkedin.metadata.Constants.*;
 
 
-public class JoinType implements com.linkedin.datahub.graphql.types.EntityType<Join, String>,
-                                 BrowsableEntityType<Join, String>, SearchableEntityType<Join, String> {
+public class ERModelRelationType implements com.linkedin.datahub.graphql.types.EntityType<ERModelRelation, String>,
+                                 BrowsableEntityType<ERModelRelation, String>, SearchableEntityType<ERModelRelation, String> {
 
 
   static final Set<String> ASPECTS_TO_RESOLVE = ImmutableSet.of(
-      JOIN_KEY_ASPECT_NAME,
-      JOIN_PROPERTIES_ASPECT_NAME,
-      EDITABLE_JOIN_PROPERTIES_ASPECT_NAME,
+      ERMODELRELATION_KEY_ASPECT_NAME,
+      ERMODELRELATION_PROPERTIES_ASPECT_NAME,
+      EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME,
       INSTITUTIONAL_MEMORY_ASPECT_NAME,
       OWNERSHIP_ASPECT_NAME,
       STATUS_ASPECT_NAME,
@@ -67,24 +67,24 @@ public class JoinType implements com.linkedin.datahub.graphql.types.EntityType<J
   );
 
   private static final Set<String> FACET_FIELDS = ImmutableSet.of("name");
-  private static final String ENTITY_NAME = "join";
+  private static final String ENTITY_NAME = "ermodelrelation";
 
   private final EntityClient _entityClient;
   private final FeatureFlags _featureFlags;
 
-  public JoinType(final EntityClient entityClient, final FeatureFlags featureFlags)  {
+  public ERModelRelationType(final EntityClient entityClient, final FeatureFlags featureFlags)  {
     _entityClient = entityClient;
-    _featureFlags = featureFlags; // TODO: check if Join Feture is Enabled and throw error when called
+    _featureFlags = featureFlags; // TODO: check if ERModelRelation Feture is Enabled and throw error when called
   }
 
   @Override
-  public Class<Join> objectClass() {
-    return Join.class;
+  public Class<ERModelRelation> objectClass() {
+    return ERModelRelation.class;
   }
 
   @Override
   public EntityType type() {
-    return EntityType.JOIN;
+    return EntityType.ERMODELRELATION;
   }
 
   @Override
@@ -93,32 +93,32 @@ public Function<Entity, String> getKeyProvider() {
   }
 
   @Override
-  public List<DataFetcherResult<Join>> batchLoad(@Nonnull final List<String> urns, @Nonnull final QueryContext context)
+  public List<DataFetcherResult<ERModelRelation>> batchLoad(@Nonnull final List<String> urns, @Nonnull final QueryContext context)
       throws Exception {
-    final List<Urn> joinUrns = urns.stream()
+    final List<Urn> ermodelrelationUrns = urns.stream()
         .map(UrnUtils::getUrn)
         .collect(Collectors.toList());
 
     try {
       final Map<Urn, EntityResponse> entities = _entityClient.batchGetV2(
-          JOIN_ENTITY_NAME,
-          new HashSet<>(joinUrns),
+          ERMODELRELATION_ENTITY_NAME,
+          new HashSet<>(ermodelrelationUrns),
           ASPECTS_TO_RESOLVE,
           context.getAuthentication());
 
       final List<EntityResponse> gmsResults = new ArrayList<>();
-      for (Urn urn : joinUrns) {
+      for (Urn urn : ermodelrelationUrns) {
         gmsResults.add(entities.getOrDefault(urn, null));
       }
       return gmsResults.stream()
           .map(gmsResult ->
-              gmsResult == null ? null : DataFetcherResult.<Join>newResult()
-                  .data(JoinMapper.map(gmsResult))
+              gmsResult == null ? null : DataFetcherResult.<ERModelRelation>newResult()
+                  .data(ERModelRelationMapper.map(gmsResult))
                   .build()
           )
           .collect(Collectors.toList());
     } catch (Exception e) {
-      throw new RuntimeException("Failed to load join entity", e);
+      throw new RuntimeException("Failed to load ermodelrelation entity", e);
     }
   }
 
@@ -129,7 +129,7 @@ public BrowseResults browse(@Nonnull List<String> path, @Nullable List<FacetFilt
     final Map<String, String> facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS);
     final String pathStr = path.size() > 0 ? BROWSE_PATH_DELIMITER + String.join(BROWSE_PATH_DELIMITER, path) : "";
     final BrowseResult result = _entityClient.browse(
-        "join",
+        "ermodelrelation",
         pathStr,
         facetFilters,
         start,
@@ -162,7 +162,7 @@ public AutoCompleteResults autoComplete(@Nonnull String query, @Nullable String
     return AutoCompleteResultsMapper.map(result);
   }
 
-  public static boolean canUpdateJoin(@Nonnull QueryContext context, JoinUrn resourceUrn, JoinUpdateInput updateInput) {
+  public static boolean canUpdateERModelRelation(@Nonnull QueryContext context, ERModelRelationUrn resourceUrn, ERModelRelationUpdateInput updateInput) {
     final ConjunctivePrivilegeGroup editPrivilegesGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(
             PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType()
     ));
@@ -184,12 +184,12 @@ public static boolean canUpdateJoin(@Nonnull QueryContext context, JoinUrn resou
             resourceUrn.toString(),
             orPrivilegeGroups);
   }
-  public static boolean canCreateJoin(@Nonnull QueryContext context, Urn datasetAUrn, Urn datasetBUrn) {
+  public static boolean canCreateERModelRelation(@Nonnull QueryContext context, Urn datasetAUrn, Urn datasetBUrn) {
     final ConjunctivePrivilegeGroup editPrivilegesGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(
             PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType()
     ));
     final ConjunctivePrivilegeGroup createPrivilegesGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(
-            PoliciesConfig.CREATE_JOIN_PRIVILEGE.getType()
+            PoliciesConfig.CREATE_ERMODELRELATION_PRIVILEGE.getType()
     ));
     // If you either have all entity privileges, or have the specific privileges required, you are authorized.
     DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of(
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/UpdateJoinResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/UpdateERModelRelationResolver.java
similarity index 63%
rename from datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/UpdateJoinResolver.java
rename to datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/UpdateERModelRelationResolver.java
index 1bc6f54914985..30234fbd307be 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/UpdateJoinResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/UpdateERModelRelationResolver.java
@@ -1,11 +1,11 @@
-package com.linkedin.datahub.graphql.types.join;
+package com.linkedin.datahub.graphql.types.ermodelrelation;
 
 import com.linkedin.common.urn.CorpuserUrn;
-import com.linkedin.common.urn.JoinUrn;
+import com.linkedin.common.urn.ERModelRelationUrn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
-import com.linkedin.datahub.graphql.generated.JoinUpdateInput;
-import com.linkedin.datahub.graphql.types.join.mappers.JoinUpdateInputMapper;
+import com.linkedin.datahub.graphql.generated.ERModelRelationUpdateInput;
+import com.linkedin.datahub.graphql.types.ermodelrelation.mappers.ERModelRelationUpdateInputMapper;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.r2.RemoteInvocationException;
@@ -21,34 +21,34 @@
 
 @Slf4j
 @RequiredArgsConstructor
-public class UpdateJoinResolver implements DataFetcher<CompletableFuture<Boolean>> {
+public class UpdateERModelRelationResolver implements DataFetcher<CompletableFuture<Boolean>> {
 
     private final EntityClient _entityClient;
     @Override
     public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throws Exception {
-        final JoinUpdateInput input = bindArgument(environment.getArgument("input"), JoinUpdateInput.class);
+        final ERModelRelationUpdateInput input = bindArgument(environment.getArgument("input"), ERModelRelationUpdateInput.class);
         final String urn = bindArgument(environment.getArgument("urn"), String.class);
-        JoinUrn inputUrn = JoinUrn.createFromString(urn);
+        ERModelRelationUrn inputUrn = ERModelRelationUrn.createFromString(urn);
         QueryContext context = environment.getContext();
         final CorpuserUrn actor = CorpuserUrn.createFromString(context.getActorUrn());
-        if (!JoinType.canUpdateJoin(context, inputUrn, input)) {
+        if (!ERModelRelationType.canUpdateERModelRelation(context, inputUrn, input)) {
             throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
         }
         return CompletableFuture.supplyAsync(() -> {
             try {
-                log.debug("Create Join input: {}", input);
-                final Collection<MetadataChangeProposal> proposals = JoinUpdateInputMapper.map(input, actor);
+                log.debug("Create ERModelRelation input: {}", input);
+                final Collection<MetadataChangeProposal> proposals = ERModelRelationUpdateInputMapper.map(input, actor);
                 proposals.forEach(proposal -> proposal.setEntityUrn(inputUrn));
 
                 try {
                     _entityClient.batchIngestProposals(proposals, context.getAuthentication(), false);
                 } catch (RemoteInvocationException e) {
-                    throw new RuntimeException(String.format("Failed to update join entity"), e);
+                    throw new RuntimeException(String.format("Failed to update ermodelrelation entity"), e);
                 }
                 return true;
             } catch (Exception e) {
-                log.error("Failed to update Join to resource with input {}, {}", input, e.getMessage());
-                throw new RuntimeException(String.format("Failed to update join to resource with input %s", input), e);
+                log.error("Failed to update ERModelRelation to resource with input {}, {}", input, e.getMessage());
+                throw new RuntimeException(String.format("Failed to update ermodelrelation to resource with input %s", input), e);
             }
         });
     }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationMapper.java
new file mode 100644
index 0000000000000..f862852f3748f
--- /dev/null
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationMapper.java
@@ -0,0 +1,143 @@
+package com.linkedin.datahub.graphql.types.ermodelrelation.mappers;
+
+import com.linkedin.common.GlobalTags;
+import com.linkedin.common.GlossaryTerms;
+import com.linkedin.common.InstitutionalMemory;
+import com.linkedin.common.Ownership;
+import com.linkedin.common.Status;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.data.DataMap;
+import com.linkedin.data.template.RecordTemplate;
+import com.linkedin.datahub.graphql.generated.Dataset;
+import com.linkedin.datahub.graphql.generated.EntityType;
+import com.linkedin.datahub.graphql.generated.ERModelRelation;
+import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
+import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
+import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
+import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.ermodelrelation.EditableERModelRelationProperties;
+import com.linkedin.ermodelrelation.ERModelRelationProperties;
+import com.linkedin.metadata.key.ERModelRelationKey;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+
+import static com.linkedin.metadata.Constants.*;
+
+/**
+ * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
+ *
+ * To be replaced by auto-generated mappers implementations
+ */
+public class ERModelRelationMapper implements ModelMapper<EntityResponse, ERModelRelation> {
+
+  public static final ERModelRelationMapper INSTANCE = new ERModelRelationMapper();
+
+  public static ERModelRelation map(@Nonnull final EntityResponse entityResponse) {
+    return INSTANCE.apply(entityResponse);
+  }
+
+  public ERModelRelation apply(final EntityResponse entityResponse) {
+    final ERModelRelation result = new ERModelRelation();
+    final Urn entityUrn = entityResponse.getUrn();
+
+    result.setUrn(entityUrn.toString());
+    result.setType(EntityType.ERMODELRELATION);
+
+    final EnvelopedAspectMap aspectMap = entityResponse.getAspects();
+    MappingHelper<ERModelRelation> mappingHelper = new MappingHelper<>(aspectMap, result);
+    mappingHelper.mapToResult(ERMODELRELATION_KEY_ASPECT_NAME, this::mapERModelRelationKey);
+    mappingHelper.mapToResult(ERMODELRELATION_PROPERTIES_ASPECT_NAME, this::mapProperties);
+    if (aspectMap != null && aspectMap.containsKey(EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME)) {
+      mappingHelper.mapToResult(EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME, this::mapEditableProperties);
+    }
+    if (aspectMap != null && aspectMap.containsKey(INSTITUTIONAL_MEMORY_ASPECT_NAME)) {
+      mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (ermodelrelation, dataMap) ->
+              ermodelrelation.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap), entityUrn)));
+    }
+    if (aspectMap != null && aspectMap.containsKey(OWNERSHIP_ASPECT_NAME)) {
+      mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (ermodelrelation, dataMap) ->
+              ermodelrelation.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn)));
+    }
+    if (aspectMap != null && aspectMap.containsKey(STATUS_ASPECT_NAME)) {
+      mappingHelper.mapToResult(STATUS_ASPECT_NAME, (ermodelrelation, dataMap) ->
+              ermodelrelation.setStatus(StatusMapper.map(new Status(dataMap))));
+    }
+    if (aspectMap != null && aspectMap.containsKey(GLOBAL_TAGS_ASPECT_NAME)) {
+      mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (ermodelrelation, dataMap) -> this.mapGlobalTags(ermodelrelation, dataMap, entityUrn));
+    }
+    if (aspectMap != null && aspectMap.containsKey(GLOSSARY_TERMS_ASPECT_NAME)) {
+      mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (ermodelrelation, dataMap) ->
+              ermodelrelation.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn)));
+    }
+    return mappingHelper.getResult();
+  }
+
+  private void mapEditableProperties(@Nonnull ERModelRelation ermodelrelation, @Nonnull DataMap dataMap) {
+    final EditableERModelRelationProperties editableERModelRelationProperties = new EditableERModelRelationProperties(dataMap);
+    ermodelrelation.setEditableProperties(
+        com.linkedin.datahub.graphql.generated.ERModelRelationEditableProperties.builder()
+            .setDescription(editableERModelRelationProperties.getDescription())
+            .setName(editableERModelRelationProperties.getName())
+            .build()
+    );
+  }
+
+
+  private void mapERModelRelationKey(@Nonnull ERModelRelation ermodelrelation, @Nonnull DataMap datamap) {
+    ERModelRelationKey ermodelrelationKey = new ERModelRelationKey(datamap);
+    ermodelrelation.setErmodelrelationId(ermodelrelationKey.getErmodelrelationId());
+  }
+
+  private void mapProperties(@Nonnull ERModelRelation ermodelrelation, @Nonnull DataMap dataMap) {
+    final ERModelRelationProperties ermodelrelationProperties = new ERModelRelationProperties(dataMap);
+    ermodelrelation.setProperties(com.linkedin.datahub.graphql.generated.ERModelRelationProperties.builder()
+        .setName(ermodelrelationProperties.getName())
+        .setDatasetA(createPartialDataset(ermodelrelationProperties.getDatasetA()))
+        .setDatasetB(createPartialDataset(ermodelrelationProperties.getDatasetB()))
+        .setErmodelrelationFieldMapping(mapERModelRelationFieldMappings(ermodelrelationProperties))
+        .setCreatedTime(ermodelrelationProperties.hasCreated() && ermodelrelationProperties.getCreated().getTime() > 0
+                ? ermodelrelationProperties.getCreated().getTime() : 0)
+        .build());
+    if (ermodelrelationProperties.hasCreated() && ermodelrelationProperties.getCreated().hasActor()) {
+      ermodelrelation.getProperties().setCreatedActor(UrnToEntityMapper.map(ermodelrelationProperties.getCreated().getActor()));
+    }
+  }
+  private Dataset createPartialDataset(@Nonnull Urn datasetUrn) {
+
+    Dataset partialDataset = new Dataset();
+
+    partialDataset.setUrn(datasetUrn.toString());
+
+    return partialDataset;
+
+  }
+  private com.linkedin.datahub.graphql.generated.ERModelRelationFieldMapping mapERModelRelationFieldMappings(
+      ERModelRelationProperties ermodelrelationProperties) {
+    return com.linkedin.datahub.graphql.generated.ERModelRelationFieldMapping.builder()
+        .setFieldMappings(ermodelrelationProperties.getErmodelrelationFieldMapping()
+            .getFieldMappings()
+            .stream()
+            .map(this::mapFieldMap)
+            .collect(Collectors.toList()))
+        .build();
+  }
+
+  private com.linkedin.datahub.graphql.generated.FieldMap mapFieldMap(com.linkedin.ermodelrelation.FieldMap fieldMap) {
+    return com.linkedin.datahub.graphql.generated.FieldMap.builder()
+        .setAfield(fieldMap.getAfield())
+        .setBfield(fieldMap.getBfield())
+        .build();
+  }
+
+  private void mapGlobalTags(@Nonnull ERModelRelation ermodelrelation, @Nonnull DataMap dataMap, @Nonnull final Urn entityUrn) {
+    com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn);
+    ermodelrelation.setTags(globalTags);
+  }
+
+}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationUpdateInputMapper.java
new file mode 100644
index 0000000000000..589f645a27ea0
--- /dev/null
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/mappers/ERModelRelationUpdateInputMapper.java
@@ -0,0 +1,127 @@
+package com.linkedin.datahub.graphql.types.ermodelrelation.mappers;
+
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.DatasetUrn;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.SetMode;
+import com.linkedin.datahub.graphql.generated.ERModelRelationFieldMappingInput;
+import com.linkedin.datahub.graphql.generated.ERModelRelationUpdateInput;
+import com.linkedin.datahub.graphql.generated.ERModelRelationPropertiesInput;
+import com.linkedin.datahub.graphql.generated.ERModelRelationEditablePropertiesUpdate;
+import com.linkedin.datahub.graphql.types.common.mappers.util.UpdateMappingHelper;
+import com.linkedin.datahub.graphql.types.mappers.InputModelMapper;
+import com.linkedin.ermodelrelation.ERModelRelationFieldMapping;
+import com.linkedin.ermodelrelation.ERModelRelationProperties;
+import com.linkedin.ermodelrelation.EditableERModelRelationProperties;
+import com.linkedin.ermodelrelation.FieldMapArray;
+import com.linkedin.ermodelrelation.FieldMap;
+import com.linkedin.mxe.MetadataChangeProposal;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import javax.annotation.Nonnull;
+
+import static com.linkedin.metadata.Constants.*;
+
+
+public class ERModelRelationUpdateInputMapper
+    implements InputModelMapper<ERModelRelationUpdateInput, Collection<MetadataChangeProposal>, Urn> {
+  public static final ERModelRelationUpdateInputMapper INSTANCE = new ERModelRelationUpdateInputMapper();
+
+  public static Collection<MetadataChangeProposal> map(@Nonnull final ERModelRelationUpdateInput ermodelrelationUpdateInput,
+      @Nonnull final Urn actor) {
+    return INSTANCE.apply(ermodelrelationUpdateInput, actor);
+  }
+
+  @Override
+  public Collection<MetadataChangeProposal> apply(ERModelRelationUpdateInput input, Urn actor) {
+    final Collection<MetadataChangeProposal> proposals = new ArrayList<>(8);
+    final UpdateMappingHelper updateMappingHelper = new UpdateMappingHelper(ERMODELRELATION_ENTITY_NAME);
+    final long currentTime = System.currentTimeMillis();
+    final AuditStamp auditstamp = new AuditStamp();
+    auditstamp.setActor(actor, SetMode.IGNORE_NULL);
+    auditstamp.setTime(currentTime);
+    if (input.getProperties() != null) {
+      com.linkedin.ermodelrelation.ERModelRelationProperties ermodelrelationProperties = createERModelRelationProperties(input.getProperties(), auditstamp);
+      proposals.add(updateMappingHelper.aspectToProposal(ermodelrelationProperties, ERMODELRELATION_PROPERTIES_ASPECT_NAME));
+    }
+    if (input.getEditableProperties() != null) {
+      final EditableERModelRelationProperties editableERModelRelationProperties = ermodelrelationEditablePropsSettings(input.getEditableProperties());
+      proposals.add(updateMappingHelper.aspectToProposal(editableERModelRelationProperties, EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME));
+    }
+    return proposals;
+  }
+  private ERModelRelationProperties createERModelRelationProperties(ERModelRelationPropertiesInput inputProperties, AuditStamp auditstamp) {
+    com.linkedin.ermodelrelation.ERModelRelationProperties ermodelrelationProperties = new com.linkedin.ermodelrelation.ERModelRelationProperties();
+    if (inputProperties.getName() != null) {
+      ermodelrelationProperties.setName(inputProperties.getName());
+    }
+    try {
+      if (inputProperties.getDataSetA() != null) {
+        ermodelrelationProperties.setDatasetA(DatasetUrn.createFromString(inputProperties.getDataSetA()));
+      }
+      if (inputProperties.getDatasetB() != null) {
+        ermodelrelationProperties.setDatasetB(DatasetUrn.createFromString(inputProperties.getDatasetB()));
+      }
+    } catch (URISyntaxException e) {
+      e.printStackTrace();
+    }
+
+    if (inputProperties.getErmodelrelationFieldmapping() != null) {
+      ERModelRelationFieldMappingInput ermodelrelationFieldMapping = inputProperties.getErmodelrelationFieldmapping();
+      if ((ermodelrelationFieldMapping.getFieldMappings() != null && ermodelrelationFieldMapping.getFieldMappings().size() > 0)) {
+        ERModelRelationFieldMapping ermodelrelationFieldMappingUnit = ermodelrelationFieldMappingSettings(ermodelrelationFieldMapping);
+        ermodelrelationProperties.setErmodelrelationFieldMapping(ermodelrelationFieldMappingUnit);
+      }
+      if (inputProperties.getCreated() != null && inputProperties.getCreated()) {
+        ermodelrelationProperties.setCreated(auditstamp);
+      } else {
+        if (inputProperties.getCreatedBy() != null && inputProperties.getCreatedAt() != 0) {
+          final AuditStamp auditstampEdit = new AuditStamp();
+          try {
+            auditstampEdit.setActor(Urn.createFromString(inputProperties.getCreatedBy()));
+          } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+          }
+          auditstampEdit.setTime(inputProperties.getCreatedAt());
+          ermodelrelationProperties.setCreated(auditstampEdit);
+        }
+      }
+      ermodelrelationProperties.setLastModified(auditstamp);
+    }
+    return ermodelrelationProperties;
+  }
+
+  private static ERModelRelationFieldMapping ermodelrelationFieldMappingSettings(ERModelRelationFieldMappingInput ermodelrelationFieldMapping) {
+    ERModelRelationFieldMapping ermodelrelationFieldMappingUnit = new ERModelRelationFieldMapping();
+
+    if (ermodelrelationFieldMapping.getFieldMappings() != null && ermodelrelationFieldMapping.getFieldMappings().size() > 0) {
+      com.linkedin.ermodelrelation.FieldMapArray fieldMapArray = new FieldMapArray();
+      ermodelrelationFieldMapping.getFieldMappings().forEach(fieldMappingInput -> {
+        FieldMap fieldMap = new FieldMap();
+        if (fieldMappingInput.getAfield() != null) {
+          fieldMap.setAfield(fieldMappingInput.getAfield());
+        }
+        if (fieldMappingInput.getBfield() != null) {
+          fieldMap.setBfield(fieldMappingInput.getBfield());
+        }
+        fieldMapArray.add(fieldMap);
+      });
+      ermodelrelationFieldMappingUnit.setFieldMappings(fieldMapArray);
+    }
+    return ermodelrelationFieldMappingUnit;
+  }
+  private static EditableERModelRelationProperties ermodelrelationEditablePropsSettings(ERModelRelationEditablePropertiesUpdate editPropsInput) {
+    final EditableERModelRelationProperties editableERModelRelationProperties = new EditableERModelRelationProperties();
+    if (editPropsInput.getName() != null
+            && editPropsInput.getName().trim().length() > 0) {
+      editableERModelRelationProperties.setName(editPropsInput.getName());
+    }
+    if (editPropsInput.getDescription() != null
+            && editPropsInput.getDescription().trim().length() > 0) {
+      editableERModelRelationProperties.setDescription(editPropsInput.getDescription());
+    }
+    return editableERModelRelationProperties;
+  }
+}
+
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/CreateJoinResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/CreateJoinResolver.java
deleted file mode 100644
index 1995034044353..0000000000000
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/CreateJoinResolver.java
+++ /dev/null
@@ -1,89 +0,0 @@
-package com.linkedin.datahub.graphql.types.join;
-
-import com.datahub.authentication.Authentication;
-import com.linkedin.common.urn.CorpuserUrn;
-import com.linkedin.common.urn.JoinUrn;
-import com.linkedin.common.urn.Urn;
-import com.linkedin.datahub.graphql.QueryContext;
-import com.linkedin.datahub.graphql.exception.AuthorizationException;
-import com.linkedin.datahub.graphql.generated.JoinPropertiesInput;
-import com.linkedin.datahub.graphql.generated.JoinUpdateInput;
-import com.linkedin.datahub.graphql.types.join.mappers.JoinMapper;
-import com.linkedin.datahub.graphql.types.join.mappers.JoinUpdateInputMapper;
-import com.linkedin.entity.client.EntityClient;
-import com.linkedin.metadata.service.JoinService;
-import com.linkedin.mxe.MetadataChangeProposal;
-import com.linkedin.r2.RemoteInvocationException;
-import graphql.schema.DataFetcher;
-import graphql.schema.DataFetchingEnvironment;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.concurrent.CompletableFuture;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import com.linkedin.datahub.graphql.generated.Join;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
-
-
-@Slf4j
-@RequiredArgsConstructor
-public class CreateJoinResolver implements DataFetcher<CompletableFuture<Join>> {
-
-    private final EntityClient _entityClient;
-    private final JoinService _joinService;
-
-    @Override
-    public CompletableFuture<Join> get(DataFetchingEnvironment environment) throws Exception {
-        final JoinUpdateInput input = bindArgument(environment.getArgument("input"), JoinUpdateInput.class);
-
-        final JoinPropertiesInput joinPropertiesInput = input.getProperties();
-        String joinName = joinPropertiesInput.getName();
-        String datasetA = joinPropertiesInput.getDataSetA();
-        String datasetB = joinPropertiesInput.getDatasetB();
-
-        String lowDataset =  datasetA;
-        String highDataset = datasetB;
-        if (datasetA.compareTo(datasetB)  > 0) {
-            lowDataset = datasetB;
-            highDataset = datasetA;
-        }
-        // The following sequence mimics datahub.emitter.mce_builder.datahub_guid
-
-        String joinKey =
-            "{\"DatasetA\":\"" + lowDataset + "\",\"DatasetB\":\"" + highDataset + "\",\"JoinName\":\"" + joinName
-                + "\"}";
-
-        byte[] mybytes = joinKey.getBytes(StandardCharsets.UTF_8);
-
-        String joinKeyEncoded = new String(mybytes, StandardCharsets.UTF_8);
-        String joinGuid = DigestUtils.md5Hex(joinKeyEncoded);
-        log.info("joinkey {}, joinGuid {}", joinKeyEncoded, joinGuid);
-
-        JoinUrn inputUrn = new JoinUrn(joinGuid);
-        QueryContext context = environment.getContext();
-        final Authentication authentication = context.getAuthentication();
-        final CorpuserUrn actor = CorpuserUrn.createFromString(context.getActorUrn());
-        if (!JoinType.canCreateJoin(context, Urn.createFromString(input.getProperties().getDataSetA()),
-                Urn.createFromString(input.getProperties().getDatasetB()))) {
-            throw new AuthorizationException("Unauthorized to create join. Please contact your DataHub administrator.");
-        }
-        return CompletableFuture.supplyAsync(() -> {
-            try {
-                log.debug("Create Join input: {}", input);
-                final Collection<MetadataChangeProposal> proposals = JoinUpdateInputMapper.map(input, actor);
-                proposals.forEach(proposal -> proposal.setEntityUrn(inputUrn));
-                try {
-                    _entityClient.batchIngestProposals(proposals, context.getAuthentication(), false);
-                } catch (RemoteInvocationException e) {
-                    throw new RuntimeException("Failed to create join entity", e);
-                }
-                return JoinMapper.map(_joinService.getJoinResponse(Urn.createFromString(inputUrn.toString()), authentication));
-            } catch (Exception e) {
-                log.error("Failed to create Join to resource with input {}, {}", input, e.getMessage());
-                throw new RuntimeException(String.format("Failed to create join to resource with input %s", input), e);
-            }
-        });
-    }
-}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java
deleted file mode 100644
index 81e9bb0d99c44..0000000000000
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java
+++ /dev/null
@@ -1,142 +0,0 @@
-package com.linkedin.datahub.graphql.types.join.mappers;
-
-import com.linkedin.common.GlobalTags;
-import com.linkedin.common.GlossaryTerms;
-import com.linkedin.common.InstitutionalMemory;
-import com.linkedin.common.Ownership;
-import com.linkedin.common.Status;
-import com.linkedin.common.urn.Urn;
-import com.linkedin.data.DataMap;
-import com.linkedin.data.template.RecordTemplate;
-import com.linkedin.datahub.graphql.generated.Dataset;
-import com.linkedin.datahub.graphql.generated.EntityType;
-import com.linkedin.datahub.graphql.generated.Join;
-import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
-import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
-import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
-import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
-import com.linkedin.entity.EntityResponse;
-import com.linkedin.entity.EnvelopedAspectMap;
-import com.linkedin.join.EditableJoinProperties;
-import com.linkedin.join.JoinProperties;
-import com.linkedin.metadata.key.JoinKey;
-import java.util.stream.Collectors;
-import javax.annotation.Nonnull;
-
-import static com.linkedin.metadata.Constants.*;
-
-/**
- * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
- *
- * To be replaced by auto-generated mappers implementations
- */
-public class JoinMapper implements ModelMapper<EntityResponse, Join> {
-
-  public static final JoinMapper INSTANCE = new JoinMapper();
-
-  public static Join map(@Nonnull final EntityResponse entityResponse) {
-    return INSTANCE.apply(entityResponse);
-  }
-
-  public Join apply(final EntityResponse entityResponse) {
-    final Join result = new Join();
-    final Urn entityUrn = entityResponse.getUrn();
-
-    result.setUrn(entityUrn.toString());
-    result.setType(EntityType.JOIN);
-
-    final EnvelopedAspectMap aspectMap = entityResponse.getAspects();
-    MappingHelper<Join> mappingHelper = new MappingHelper<>(aspectMap, result);
-    mappingHelper.mapToResult(JOIN_KEY_ASPECT_NAME, this::mapJoinKey);
-    mappingHelper.mapToResult(JOIN_PROPERTIES_ASPECT_NAME, this::mapProperties);
-    if (aspectMap != null && aspectMap.containsKey(EDITABLE_JOIN_PROPERTIES_ASPECT_NAME)) {
-      mappingHelper.mapToResult(EDITABLE_JOIN_PROPERTIES_ASPECT_NAME, this::mapEditableProperties);
-    }
-    if (aspectMap != null && aspectMap.containsKey(INSTITUTIONAL_MEMORY_ASPECT_NAME)) {
-      mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (join, dataMap) ->
-              join.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap), entityUrn)));
-    }
-    if (aspectMap != null && aspectMap.containsKey(OWNERSHIP_ASPECT_NAME)) {
-      mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (join, dataMap) ->
-              join.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn)));
-    }
-    if (aspectMap != null && aspectMap.containsKey(STATUS_ASPECT_NAME)) {
-      mappingHelper.mapToResult(STATUS_ASPECT_NAME, (join, dataMap) ->
-              join.setStatus(StatusMapper.map(new Status(dataMap))));
-    }
-    if (aspectMap != null && aspectMap.containsKey(GLOBAL_TAGS_ASPECT_NAME)) {
-      mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (join, dataMap) -> this.mapGlobalTags(join, dataMap, entityUrn));
-    }
-    if (aspectMap != null && aspectMap.containsKey(GLOSSARY_TERMS_ASPECT_NAME)) {
-      mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (join, dataMap) ->
-              join.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn)));
-    }
-    return mappingHelper.getResult();
-  }
-
-  private void mapEditableProperties(@Nonnull Join join, @Nonnull DataMap dataMap) {
-    final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(dataMap);
-    join.setEditableProperties(
-        com.linkedin.datahub.graphql.generated.JoinEditableProperties.builder()
-            .setDescription(editableJoinProperties.getDescription())
-            .setName(editableJoinProperties.getName())
-            .build()
-    );
-  }
-
-
-  private void mapJoinKey(@Nonnull Join join, @Nonnull DataMap datamap) {
-    JoinKey joinKey = new JoinKey(datamap);
-    join.setJoinId(joinKey.getJoinId());
-  }
-
-  private void mapProperties(@Nonnull Join join, @Nonnull DataMap dataMap) {
-    final JoinProperties joinProperties = new JoinProperties(dataMap);
-    join.setProperties(com.linkedin.datahub.graphql.generated.JoinProperties.builder()
-        .setName(joinProperties.getName())
-        .setDatasetA(createPartialDataset(joinProperties.getDatasetA()))
-        .setDatasetB(createPartialDataset(joinProperties.getDatasetB()))
-        .setJoinFieldMapping(mapJoinFieldMappings(joinProperties))
-        .setCreatedTime(joinProperties.hasCreated() && joinProperties.getCreated().getTime() > 0
-                ? joinProperties.getCreated().getTime() : 0)
-        .build());
-    if (joinProperties.hasCreated() && joinProperties.getCreated().hasActor()) {
-      join.getProperties().setCreatedActor(UrnToEntityMapper.map(joinProperties.getCreated().getActor()));
-    }
-  }
-  private Dataset createPartialDataset(@Nonnull Urn datasetUrn) {
-
-    Dataset partialDataset = new Dataset();
-
-    partialDataset.setUrn(datasetUrn.toString());
-
-    return partialDataset;
-
-  }
-  private com.linkedin.datahub.graphql.generated.JoinFieldMapping mapJoinFieldMappings(JoinProperties joinProperties) {
-    return com.linkedin.datahub.graphql.generated.JoinFieldMapping.builder()
-        .setFieldMappings(joinProperties.getJoinFieldMapping()
-            .getFieldMappings()
-            .stream()
-            .map(this::mapFieldMap)
-            .collect(Collectors.toList()))
-        .build();
-  }
-
-  private com.linkedin.datahub.graphql.generated.FieldMap mapFieldMap(com.linkedin.join.FieldMap fieldMap) {
-    return com.linkedin.datahub.graphql.generated.FieldMap.builder()
-        .setAfield(fieldMap.getAfield())
-        .setBfield(fieldMap.getBfield())
-        .build();
-  }
-
-  private void mapGlobalTags(@Nonnull Join join, @Nonnull DataMap dataMap, @Nonnull final Urn entityUrn) {
-    com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn);
-    join.setTags(globalTags);
-  }
-
-}
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java
deleted file mode 100644
index 09f26bfd219dd..0000000000000
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java
+++ /dev/null
@@ -1,127 +0,0 @@
-package com.linkedin.datahub.graphql.types.join.mappers;
-
-import com.linkedin.common.AuditStamp;
-import com.linkedin.common.urn.DatasetUrn;
-import com.linkedin.common.urn.Urn;
-import com.linkedin.data.template.SetMode;
-import com.linkedin.datahub.graphql.generated.JoinFieldMappingInput;
-import com.linkedin.datahub.graphql.generated.JoinUpdateInput;
-import com.linkedin.datahub.graphql.generated.JoinPropertiesInput;
-import com.linkedin.datahub.graphql.generated.JoinEditablePropertiesUpdate;
-import com.linkedin.datahub.graphql.types.common.mappers.util.UpdateMappingHelper;
-import com.linkedin.datahub.graphql.types.mappers.InputModelMapper;
-import com.linkedin.join.JoinFieldMapping;
-import com.linkedin.join.JoinProperties;
-import com.linkedin.join.EditableJoinProperties;
-import com.linkedin.join.FieldMapArray;
-import com.linkedin.join.FieldMap;
-import com.linkedin.mxe.MetadataChangeProposal;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import javax.annotation.Nonnull;
-
-import static com.linkedin.metadata.Constants.*;
-
-
-public class JoinUpdateInputMapper
-    implements InputModelMapper<JoinUpdateInput, Collection<MetadataChangeProposal>, Urn> {
-  public static final JoinUpdateInputMapper INSTANCE = new JoinUpdateInputMapper();
-
-  public static Collection<MetadataChangeProposal> map(@Nonnull final JoinUpdateInput joinUpdateInput,
-      @Nonnull final Urn actor) {
-    return INSTANCE.apply(joinUpdateInput, actor);
-  }
-
-  @Override
-  public Collection<MetadataChangeProposal> apply(JoinUpdateInput input, Urn actor) {
-    final Collection<MetadataChangeProposal> proposals = new ArrayList<>(8);
-    final UpdateMappingHelper updateMappingHelper = new UpdateMappingHelper(JOIN_ENTITY_NAME);
-    final long currentTime = System.currentTimeMillis();
-    final AuditStamp auditstamp = new AuditStamp();
-    auditstamp.setActor(actor, SetMode.IGNORE_NULL);
-    auditstamp.setTime(currentTime);
-    if (input.getProperties() != null) {
-      com.linkedin.join.JoinProperties joinProperties = createJoinProperties(input.getProperties(), auditstamp);
-      proposals.add(updateMappingHelper.aspectToProposal(joinProperties, JOIN_PROPERTIES_ASPECT_NAME));
-    }
-    if (input.getEditableProperties() != null) {
-      final EditableJoinProperties editableJoinProperties = joinEditablePropsSettings(input.getEditableProperties());
-      proposals.add(updateMappingHelper.aspectToProposal(editableJoinProperties, EDITABLE_JOIN_PROPERTIES_ASPECT_NAME));
-    }
-    return proposals;
-  }
-  private JoinProperties createJoinProperties(JoinPropertiesInput inputProperties, AuditStamp auditstamp) {
-    com.linkedin.join.JoinProperties joinProperties = new com.linkedin.join.JoinProperties();
-    if (inputProperties.getName() != null) {
-      joinProperties.setName(inputProperties.getName());
-    }
-    try {
-      if (inputProperties.getDataSetA() != null) {
-        joinProperties.setDatasetA(DatasetUrn.createFromString(inputProperties.getDataSetA()));
-      }
-      if (inputProperties.getDatasetB() != null) {
-        joinProperties.setDatasetB(DatasetUrn.createFromString(inputProperties.getDatasetB()));
-      }
-    } catch (URISyntaxException e) {
-      e.printStackTrace();
-    }
-
-    if (inputProperties.getJoinFieldmapping() != null) {
-      JoinFieldMappingInput joinFieldMapping = inputProperties.getJoinFieldmapping();
-      if ((joinFieldMapping.getFieldMappings() != null && joinFieldMapping.getFieldMappings().size() > 0)) {
-        JoinFieldMapping joinFieldMappingUnit = joinFieldMappingSettings(joinFieldMapping);
-        joinProperties.setJoinFieldMapping(joinFieldMappingUnit);
-      }
-      if (inputProperties.getCreated() != null && inputProperties.getCreated()) {
-        joinProperties.setCreated(auditstamp);
-      } else {
-        if (inputProperties.getCreatedBy() != null && inputProperties.getCreatedAt() != 0) {
-          final AuditStamp auditstampEdit = new AuditStamp();
-          try {
-            auditstampEdit.setActor(Urn.createFromString(inputProperties.getCreatedBy()));
-          } catch (URISyntaxException e) {
-            throw new RuntimeException(e);
-          }
-          auditstampEdit.setTime(inputProperties.getCreatedAt());
-          joinProperties.setCreated(auditstampEdit);
-        }
-      }
-      joinProperties.setLastModified(auditstamp);
-    }
-    return joinProperties;
-  }
-
-  private static JoinFieldMapping joinFieldMappingSettings(JoinFieldMappingInput joinFieldMapping) {
-    JoinFieldMapping joinFieldMappingUnit = new JoinFieldMapping();
-
-    if (joinFieldMapping.getFieldMappings() != null && joinFieldMapping.getFieldMappings().size() > 0) {
-      com.linkedin.join.FieldMapArray fieldMapArray = new FieldMapArray();
-      joinFieldMapping.getFieldMappings().forEach(fieldMappingInput -> {
-        FieldMap fieldMap = new FieldMap();
-        if (fieldMappingInput.getAfield() != null) {
-          fieldMap.setAfield(fieldMappingInput.getAfield());
-        }
-        if (fieldMappingInput.getBfield() != null) {
-          fieldMap.setBfield(fieldMappingInput.getBfield());
-        }
-        fieldMapArray.add(fieldMap);
-      });
-      joinFieldMappingUnit.setFieldMappings(fieldMapArray);
-    }
-    return joinFieldMappingUnit;
-  }
-  private static EditableJoinProperties joinEditablePropsSettings(JoinEditablePropertiesUpdate editPropsInput) {
-    final EditableJoinProperties editableJoinProperties = new EditableJoinProperties();
-    if (editPropsInput.getName() != null
-            && editPropsInput.getName().trim().length() > 0) {
-      editableJoinProperties.setName(editPropsInput.getName());
-    }
-    if (editPropsInput.getDescription() != null
-            && editPropsInput.getDescription().trim().length() > 0) {
-      editableJoinProperties.setDescription(editPropsInput.getDescription());
-    }
-    return editableJoinProperties;
-  }
-}
-
diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql
index 605597be5fec2..adb687ab7c7cb 100644
--- a/datahub-graphql-core/src/main/resources/app.graphql
+++ b/datahub-graphql-core/src/main/resources/app.graphql
@@ -442,9 +442,9 @@ type FeatureFlagsConfig {
   """
   showAcrylInfo: Boolean!
   """
-  Whether Join Tables Feature should be shown.
+  Whether ERModelRelation Tables Feature should be shown.
   """
-  joinFeatureEnabled: Boolean!
+  ermodelrelationFeatureEnabled: Boolean!
 
   """
   Whether we should show AccessManagement tab in the datahub UI.
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index 3a0acc37e4a26..5326254b75e41 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -74,9 +74,9 @@ type Query {
     role(urn: String!): Role
 
     """
-    Fetch a Join by primary key (urn)
+    Fetch a ERModelRelation by primary key (urn)
     """
-    join(urn: String!): Join
+    ermodelrelation(urn: String!): ERModelRelation
 
     """
     Fetch a Glossary Term by primary key (urn)
@@ -239,9 +239,9 @@ type Query {
 }
 
 """
-A Join is a high-level abstraction that dictates what datasets fields are joined.
+A ERModelRelation is a high-level abstraction that dictates what datasets fields are ermodelrelationed.
 """
-type Join implements EntityWithRelationships & Entity {
+type ERModelRelation implements EntityWithRelationships & Entity {
     """
     The primary key of the role
     """
@@ -253,19 +253,19 @@ type Join implements EntityWithRelationships & Entity {
     type: EntityType!
 
     """
-    Unique id for the join
+    Unique id for the ermodelrelation
     """
-    joinId: String!
+    ermodelrelationId: String!
 
     """
     An additional set of read only properties
     """
-    properties: JoinProperties
+    properties: ERModelRelationProperties
 
     """
     An additional set of of read write properties
     """
-    editableProperties: JoinEditableProperties
+    editableProperties: ERModelRelationEditableProperties
 
     """
     References to internal resources related to the dataset
@@ -309,27 +309,27 @@ type Join implements EntityWithRelationships & Entity {
 }
 
 """
-Additional properties about a Join
+Additional properties about a ERModelRelation
 """
-type JoinEditableProperties {
+type ERModelRelationEditableProperties {
 
     """
-    Documentation of the Join
+    Documentation of the ERModelRelation
     """
     description: String
     """
-    Display name of the Join
+    Display name of the ERModelRelation
     """
     name: String
 }
 
 """
-Additional properties about a Join
+Additional properties about a ERModelRelation
 """
-type JoinProperties {
+type ERModelRelationProperties {
 
     """
-    The name of the Join used in display
+    The name of the ERModelRelation used in display
     """
     name: String!
     """
@@ -343,25 +343,25 @@ type JoinProperties {
     datasetB: Dataset!
 
     """
-    The joinFieldMappings
+    The ermodelrelationFieldMappings
     """
-    joinFieldMapping: JoinFieldMapping!
+    ermodelrelationFieldMapping: ERModelRelationFieldMapping!
 
     """
-    Created timestamp millis associated with the Join
+    Created timestamp millis associated with the ERModelRelation
     """
     createdTime: Long
 
     """
-    Created actor urn associated with the Join
+    Created actor urn associated with the ERModelRelation
     """
     createdActor: Entity
 }
 
 """
-Join Field Mapping
+ERModelRelation Field Mapping
 """
-type JoinFieldMapping {
+type ERModelRelationFieldMapping {
     """
     The Array of FieldMap
     """
@@ -369,7 +369,7 @@ type JoinFieldMapping {
 }
 
 """
-Join FieldMap
+ERModelRelation FieldMap
 """
 type FieldMap {
     """
@@ -617,26 +617,26 @@ type Mutation {
     unsetDomain(entityUrn: String!): Boolean
 
     """
-    Create a Join
+    Create a ERModelRelation
     """
-    createJoin(
-        "Input required to create a new Join"
-        input: JoinUpdateInput!): Join
+    createERModelRelation(
+        "Input required to create a new ERModelRelation"
+        input: ERModelRelationUpdateInput!): ERModelRelation
 
     """
-    Update a Join
+    Update a ERModelRelation
     """
-    updateJoin(
-        "The urn of the Join to update"
+    updateERModelRelation(
+        "The urn of the ERModelRelation to update"
         urn: String!,
         "Input required to updat an existing DataHub View"
-        input: JoinUpdateInput!): Boolean
+        input: ERModelRelationUpdateInput!): Boolean
 
     """
-    Delete a Join
+    Delete a ERModelRelation
     """
-    deleteJoin(
-        "The urn of the Join to delete"
+    deleteERModelRelation(
+        "The urn of the ERModelRelation to delete"
         urn: String!): Boolean
 
 
@@ -931,9 +931,9 @@ enum EntityType {
     DATA_PLATFORM
 
     """
-    The Join Entity
+    The ERModelRelation Entity
     """
-    JOIN
+    ERMODELRELATION
 
     """
     The Dashboard Entity
@@ -4498,14 +4498,14 @@ input DatasetEditablePropertiesUpdate {
 """
 Update to writable Dataset fields
 """
-input JoinEditablePropertiesUpdate {
+input ERModelRelationEditablePropertiesUpdate {
     """
-    Display name of the Join
+    Display name of the ERModelRelation
     """
     name: String
 
     """
-    Writable description for Join
+    Writable description for ERModelRelation
     """
     description: String!
 }
@@ -4629,72 +4629,72 @@ input CreateTagInput {
 }
 
 """
-Input required to create/update a new Join
+Input required to create/update a new ERModelRelation
 """
-input JoinUpdateInput {
+input ERModelRelationUpdateInput {
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
-    properties: JoinPropertiesInput
+    properties: ERModelRelationPropertiesInput
     """
     Update to editable properties
     """
-    editableProperties: JoinEditablePropertiesUpdate
+    editableProperties: ERModelRelationEditablePropertiesUpdate
 }
 
 """
-Details about the Join
+Details about the ERModelRelation
 """
-input JoinPropertiesInput {
+input ERModelRelationPropertiesInput {
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
     name: String!
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
     dataSetA: String!
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
     datasetB: String!
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
-    joinFieldmapping: JoinFieldMappingInput!
+    ermodelrelationFieldmapping: ERModelRelationFieldMappingInput!
     """
-    optional flag about the Join is getting create
+    optional flag about the ERModelRelation is getting create
     """
     created: Boolean
     """
-    optional field to prevent created time while the Join is getting update
+    optional field to prevent created time while the ERModelRelation is getting update
     """
     createdAt: Long
     """
-    optional field to prevent create actor while the Join is getting update
+    optional field to prevent create actor while the ERModelRelation is getting update
     """
     createdBy: String
 }
 
 """
-Details about the Join
+Details about the ERModelRelation
 """
-input JoinFieldMappingInput {
+input ERModelRelationFieldMappingInput {
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
     fieldMappings: [FieldMappingInput]
 }
 """
-Details about the Join
+Details about the ERModelRelation
 """
 input FieldMappingInput {
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
     afield: String
     """
-    Details about the Join
+    Details about the ERModelRelation
     """
     bfield: String
 }
diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx
index 4d5a8a790261b..02b9e60d42627 100644
--- a/datahub-web-react/src/App.tsx
+++ b/datahub-web-react/src/App.tsx
@@ -34,7 +34,7 @@ import { ContainerEntity } from './app/entity/container/ContainerEntity';
 import GlossaryNodeEntity from './app/entity/glossaryNode/GlossaryNodeEntity';
 import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity';
 import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity';
-import { JoinEntity } from './app/entity/joins/JoinEntity';
+import { ERModelRelationEntity } from './app/entity/ermodelrelations/ERModelRelationEntity';
 import { DataPlatformInstanceEntity } from './app/entity/dataPlatformInstance/DataPlatformInstanceEntity';
 import { RoleEntity } from './app/entity/Access/RoleEntity';
 import possibleTypesResult from './possibleTypes.generated';
@@ -124,7 +124,7 @@ const App: React.VFC = () => {
         register.register(new RoleEntity());
         register.register(new DataPlatformEntity());
         register.register(new DataProductEntity());
-        register.register(new JoinEntity());
+        register.register(new ERModelRelationEntity());
         register.register(new DataPlatformInstanceEntity());
         return register;
     }, []);
diff --git a/datahub-web-react/src/app/entity/joins/JoinEntity.tsx b/datahub-web-react/src/app/entity/ermodelrelations/ERModelRelationEntity.tsx
similarity index 70%
rename from datahub-web-react/src/app/entity/joins/JoinEntity.tsx
rename to datahub-web-react/src/app/entity/ermodelrelations/ERModelRelationEntity.tsx
index 0d36b9446d413..1c0a0bc31ac59 100644
--- a/datahub-web-react/src/app/entity/joins/JoinEntity.tsx
+++ b/datahub-web-react/src/app/entity/ermodelrelations/ERModelRelationEntity.tsx
@@ -1,27 +1,27 @@
 import * as React from 'react';
 import { DatabaseOutlined, DatabaseFilled } from '@ant-design/icons';
-import { EntityType, Join, OwnershipType, SearchResult } from '../../../types.generated';
+import { EntityType, ErModelRelation, OwnershipType, SearchResult } from '../../../types.generated';
 import { Entity, IconStyleType, PreviewType } from '../Entity';
 import { getDataForEntityType } from '../shared/containers/profile/utils';
 import { GenericEntityProperties } from '../shared/types';
-import { JoinPreviewCard } from './preview/JoinPreviewCard';
-import joinIcon from '../../../images/joinIcon.svg';
-import { JoinTab } from '../shared/tabs/Join/JoinTab';
-import { useGetJoinQuery, useUpdateJoinMutation } from '../../../graphql/join.generated';
+import { ERModelRelationPreviewCard } from './preview/ERModelRelationPreviewCard';
+import ermodelrelationIcon from '../../../images/ermodelrelationIcon.svg';
+import { ERModelRelationTab } from '../shared/tabs/ERModelRelation/ERModelRelationTab';
+import { useGetErModelRelationQuery, useUpdateErModelRelationMutation } from '../../../graphql/ermodelrelation.generated';
 import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
 import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection';
 import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection';
 import { EntityProfile } from '../shared/containers/profile/EntityProfile';
-import './preview/JoinAction.less';
+import './preview/ERModelRelationAction.less';
 import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection';
 
 /**
- * Definition of the DataHub Join entity.
+ * Definition of the DataHub ERModelRelation entity.
  */
 
-export class JoinEntity implements Entity<Join> {
-    type: EntityType = EntityType.Join;
+export class ERModelRelationEntity implements Entity<ErModelRelation> {
+    type: EntityType = EntityType.Ermodelrelation;
 
     icon = (fontSize: number, styleType: IconStyleType) => {
         if (styleType === IconStyleType.TAB_VIEW) {
@@ -38,7 +38,7 @@ export class JoinEntity implements Entity<Join> {
             );
         }
 
-        return <img src={joinIcon} style={{ height: '16px', width: '16px' }} alt="" />;
+        return <img src={ermodelrelationIcon} style={{ height: '16px', width: '16px' }} alt="" />;
     };
 
     isSearchEnabled = () => true;
@@ -49,23 +49,23 @@ export class JoinEntity implements Entity<Join> {
 
     getAutoCompleteFieldName = () => 'name';
 
-    getPathName = () => 'join';
+    getPathName = () => 'ermodelrelation';
 
     getCollectionName = () => '';
 
-    getEntityName = () => 'Join';
+    getEntityName = () => 'ERModelRelation';
 
     renderProfile = (urn: string) => (
         <EntityProfile
             urn={urn}
-            entityType={EntityType.Join}
-            useEntityQuery={useGetJoinQuery}
-            useUpdateQuery={useUpdateJoinMutation}
+            entityType={EntityType.Ermodelrelation}
+            useEntityQuery={useGetErModelRelationQuery}
+            useUpdateQuery={useUpdateErModelRelationMutation}
             getOverrideProperties={this.getOverridePropertiesFromEntity}
             tabs={[
                 {
-                    name: 'Join',
-                    component: JoinTab,
+                    name: 'ERModelRelation',
+                    component: ERModelRelationTab,
                 },
                 {
                     name: 'Documentation',
@@ -98,17 +98,17 @@ export class JoinEntity implements Entity<Join> {
         />
     );
 
-    getOverridePropertiesFromEntity = (_join?: Join | null): GenericEntityProperties => {
+    getOverridePropertiesFromEntity = (_ermodelrelation?: ErModelRelation | null): GenericEntityProperties => {
         return {};
     };
 
-    renderPreview = (_: PreviewType, data: Join) => {
+    renderPreview = (_: PreviewType, data: ErModelRelation) => {
         return (
             <>
-                <JoinPreviewCard
+                <ERModelRelationPreviewCard
                     urn={data.urn}
                     name={
-                        <span className="joinName">{data.properties?.name || data.editableProperties?.name || ''}</span>
+                        <span className="ermodelrelationName">{data.properties?.name || data.editableProperties?.name || ''}</span>
                     }
                     description={data?.editableProperties?.description || ''}
                     owners={data.ownership?.owners}
@@ -120,14 +120,14 @@ export class JoinEntity implements Entity<Join> {
     };
 
     renderSearch = (result: SearchResult) => {
-        return this.renderPreview(PreviewType.SEARCH, result.entity as Join);
+        return this.renderPreview(PreviewType.SEARCH, result.entity as ErModelRelation);
     };
 
-    displayName = (data: Join) => {
+    displayName = (data: ErModelRelation) => {
         return data.properties?.name || data.editableProperties?.name || data.urn;
     };
 
-    getGenericEntityProperties = (data: Join) => {
+    getGenericEntityProperties = (data: ErModelRelation) => {
         return getDataForEntityType({
             data,
             entityType: this.type,
diff --git a/datahub-web-react/src/app/entity/joins/preview/JoinAction.less b/datahub-web-react/src/app/entity/ermodelrelations/preview/ERModelRelationAction.less
similarity index 100%
rename from datahub-web-react/src/app/entity/joins/preview/JoinAction.less
rename to datahub-web-react/src/app/entity/ermodelrelations/preview/ERModelRelationAction.less
diff --git a/datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx b/datahub-web-react/src/app/entity/ermodelrelations/preview/ERModelRelationPreviewCard.tsx
similarity index 74%
rename from datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx
rename to datahub-web-react/src/app/entity/ermodelrelations/preview/ERModelRelationPreviewCard.tsx
index eac2b9f2d333a..8201265969bfb 100644
--- a/datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx
+++ b/datahub-web-react/src/app/entity/ermodelrelations/preview/ERModelRelationPreviewCard.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { Card, Collapse } from 'antd';
-import joinIcon from '../../../../images/joinIcon.svg';
+import ermodelrelationIcon from '../../../../images/ermodelrelationIcon.svg';
 import { EntityType, Owner, GlobalTags, GlossaryTerms } from '../../../../types.generated';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
@@ -8,7 +8,7 @@ import { IconStyleType } from '../../Entity';
 
 const { Panel } = Collapse;
 
-export const JoinPreviewCard = ({
+export const ERModelRelationPreviewCard = ({
     urn,
     name,
     owners,
@@ -24,20 +24,20 @@ export const JoinPreviewCard = ({
     owners?: Array<Owner> | null;
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
-    const getJoinHeader = (): JSX.Element => {
+    const getERModelRelationHeader = (): JSX.Element => {
         return (
             <div style={{ width: '100%', display: 'inline-block' }}>
                 <DefaultPreviewCard
-                    url={entityRegistry.getEntityUrl(EntityType.Join, urn)}
+                    url={entityRegistry.getEntityUrl(EntityType.Ermodelrelation, urn)}
                     name={name || ''}
                     urn={urn}
                     description={description || ''}
-                    logoComponent={<img src={joinIcon} alt="Join" style={{ fontSize: '20px' }} />}
+                    logoComponent={<img src={ermodelrelationIcon} alt="ERModelRelation" style={{ fontSize: '20px' }} />}
                     tags={globalTags || undefined}
                     glossaryTerms={glossaryTerms || undefined}
                     owners={owners}
-                    type="Join"
-                    typeIcon={entityRegistry.getIcon(EntityType.Join, 14, IconStyleType.ACCENT)}
+                    type="ERModelRelation"
+                    typeIcon={entityRegistry.getIcon(EntityType.Ermodelrelation, 14, IconStyleType.ACCENT)}
                     titleSizePx={18}
                 />
             </div>
@@ -47,7 +47,7 @@ export const JoinPreviewCard = ({
     return (
         <>
             <Card className="cardStyle" bordered key={`${urn}_1`}>
-                <Panel header={getJoinHeader()} key={`${urn}_2`} />
+                <Panel header={getERModelRelationHeader()} key={`${urn}_2`} />
             </Card>
         </>
     );
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/CreateERModelRelationModal.less
similarity index 97%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/CreateERModelRelationModal.less
index 0cebcaca91483..b50d3debaf1ef 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/CreateERModelRelationModal.less
@@ -1,7 +1,7 @@
 @import "../../../../../../../node_modules/antd/dist/antd.less";
 
-.CreateJoinModal {
-    .join-name {
+.CreateERModelRelationModal {
+    .ermodelrelation-name {
         padding: 8px 16px;
         width: 948.5px !important;
         height: 40px !important;
@@ -47,7 +47,7 @@
         border-bottom: 0px !important;
     }
 
-    .join-title {
+    .ermodelrelation-title {
         width: 300px !important;
         height: 22px;
         font-family: 'Arial';
@@ -174,7 +174,7 @@
         padding-left: 8px;
         display: flex;
     }
-    .join-select-selector {
+    .ermodelrelation-select-selector {
         align-items: center;
         width: 300px !important;
         height: 38px !important;
@@ -189,7 +189,7 @@
         font-style: normal;
         color: #000000D9;
     }
-    .join-details-ta {
+    .ermodelrelation-details-ta {
         height: 95px;
         width: 720px;
         font-style: normal;
@@ -198,7 +198,7 @@
         line-height: 22px;
         color: rgba(0, 0, 0, 0.85);
     }
-    .JoinTable {
+    .ERModelRelationTable {
         .icon-image {
             box-sizing: border-box;
             width: 16px;
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/CreateERModelRelationModal.tsx
similarity index 70%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/CreateERModelRelationModal.tsx
index 93dc274a8e8de..cda2a6ed9316f 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/CreateERModelRelationModal.tsx
@@ -3,13 +3,13 @@ import { Button, Form, Input, message, Modal, Table } from 'antd';
 import TextArea from 'antd/lib/input/TextArea';
 import { PlusOutlined } from '@ant-design/icons';
 import arrow from '../../../../../../images/Arrow.svg';
-import './CreateJoinModal.less';
-import { EntityType, Join, OwnerEntityType } from '../../../../../../types.generated';
-import { useCreateJoinMutation, useUpdateJoinMutation } from '../../../../../../graphql/join.generated';
+import './CreateERModelRelationModal.less';
+import { EntityType, ErModelRelation, OwnerEntityType } from '../../../../../../types.generated';
+import { useCreateErModelRelationMutation, useUpdateErModelRelationMutation } from '../../../../../../graphql/ermodelrelation.generated';
 import { useUserContext } from '../../../../../context/useUserContext';
 import { EditableRow } from './EditableRow';
 import { EditableCell } from './EditableCell';
-import { checkDuplicateJoin, getDatasetName, JoinDataType, validateJoin } from './JoinUtils';
+import { checkDuplicateERModelRelation, getDatasetName, ERModelRelationDataType, validateERModelRelation } from './ERModelRelationUtils';
 import { useGetSearchResultsQuery } from '../../../../../../graphql/search.generated';
 import { useAddOwnerMutation } from '../../../../../../graphql/mutations.generated';
 
@@ -21,7 +21,7 @@ type Props = {
     visible: boolean;
     setModalVisible?: any;
     onCancel: () => void;
-    editJoin?: Join;
+    editERModelRelation?: ErModelRelation;
     isEditing?: boolean;
     refetch: () => Promise<any>;
 };
@@ -29,7 +29,7 @@ type Props = {
 type EditableTableProps = Parameters<typeof Table>[0];
 type ColumnTypes = Exclude<EditableTableProps['columns'], undefined>;
 
-export const CreateJoinModal = ({
+export const CreateERModelRelationModal = ({
     table1,
     table1Schema,
     table2,
@@ -37,7 +37,7 @@ export const CreateJoinModal = ({
     visible,
     setModalVisible,
     onCancel,
-    editJoin,
+    editERModelRelation,
     isEditing,
     refetch,
 }: Props) => {
@@ -45,17 +45,17 @@ export const CreateJoinModal = ({
     const { user } = useUserContext();
     const ownerEntityType =
         user && user.type === EntityType.CorpGroup ? OwnerEntityType.CorpGroup : OwnerEntityType.CorpUser;
-    const table1Dataset = editJoin?.properties?.datasetA || table1?.dataset;
-    const table1DatasetSchema = editJoin?.properties?.datasetA || table1Schema;
-    const table2Dataset = editJoin?.properties?.datasetB || table2?.dataset;
-    const table2DatasetSchema = editJoin?.properties?.datasetB || table2Schema?.dataset;
+    const table1Dataset = editERModelRelation?.properties?.datasetA || table1?.dataset;
+    const table1DatasetSchema = editERModelRelation?.properties?.datasetA || table1Schema;
+    const table2Dataset = editERModelRelation?.properties?.datasetB || table2?.dataset;
+    const table2DatasetSchema = editERModelRelation?.properties?.datasetB || table2Schema?.dataset;
 
-    const [details, setDetails] = useState<string>(editJoin?.editableProperties?.description || '');
-    const [joinName, setJoinName] = useState<string>(
-        editJoin?.editableProperties?.name || editJoin?.properties?.name || editJoin?.joinId || '',
+    const [details, setDetails] = useState<string>(editERModelRelation?.editableProperties?.description || '');
+    const [ermodelrelationName, setERModelRelationName] = useState<string>(
+        editERModelRelation?.editableProperties?.name || editERModelRelation?.properties?.name || editERModelRelation?.ermodelrelationId || '',
     );
-    const [tableData, setTableData] = useState<JoinDataType[]>(
-        editJoin?.properties?.joinFieldMapping?.fieldMappings?.map((item, index) => {
+    const [tableData, setTableData] = useState<ERModelRelationDataType[]>(
+        editERModelRelation?.properties?.ermodelrelationFieldMapping?.fieldMappings?.map((item, index) => {
             return {
                 key: index,
                 field1Name: item.afield,
@@ -66,11 +66,11 @@ export const CreateJoinModal = ({
             { key: '1', field1Name: '', field2Name: '' },
         ],
     );
-    const [count, setCount] = useState(editJoin?.properties?.joinFieldMapping?.fieldMappings?.length || 2);
-    const [createMutation] = useCreateJoinMutation();
-    const [updateMutation] = useUpdateJoinMutation();
+    const [count, setCount] = useState(editERModelRelation?.properties?.ermodelrelationFieldMapping?.fieldMappings?.length || 2);
+    const [createMutation] = useCreateErModelRelationMutation();
+    const [updateMutation] = useUpdateErModelRelationMutation();
     const [addOwnerMutation] = useAddOwnerMutation();
-    const { refetch: getSearchResultsJoins } = useGetSearchResultsQuery({
+    const { refetch: getSearchResultsERModelRelations } = useGetSearchResultsQuery({
         skip: true,
     });
 
@@ -82,12 +82,12 @@ export const CreateJoinModal = ({
         Modal.confirm({
             title: `Exit`,
             className: 'cancel-modal',
-            content: `Are you sure you want to exit?  The changes made to the join will not be applied.`,
+            content: `Are you sure you want to exit?  The changes made to the ermodelrelation will not be applied.`,
             onOk() {
-                setJoinName(editJoin?.properties?.name || '');
-                setDetails(editJoin?.editableProperties?.description || '');
+                setERModelRelationName(editERModelRelation?.properties?.name || '');
+                setDetails(editERModelRelation?.editableProperties?.description || '');
                 setTableData(
-                    editJoin?.properties?.joinFieldMapping?.fieldMappings?.map((item, index) => {
+                    editERModelRelation?.properties?.ermodelrelationFieldMapping?.fieldMappings?.map((item, index) => {
                         return {
                             key: index,
                             field1Name: item.afield,
@@ -98,7 +98,7 @@ export const CreateJoinModal = ({
                         { key: '1', field1Name: '', field2Name: '' },
                     ],
                 );
-                setCount(editJoin?.properties?.joinFieldMapping?.fieldMappings?.length || 2);
+                setCount(editERModelRelation?.properties?.ermodelrelationFieldMapping?.fieldMappings?.length || 2);
                 onCancel?.();
             },
             onCancel() {},
@@ -107,15 +107,15 @@ export const CreateJoinModal = ({
             closable: true,
         });
     };
-    const createJoin = () => {
+    const createERModelRelation = () => {
         createMutation({
             variables: {
                 input: {
                     properties: {
                         dataSetA: table1Dataset?.urn || '',
                         datasetB: table2Dataset?.urn || '',
-                        name: joinName,
-                        joinFieldmapping: {
+                        name: ermodelrelationName,
+                        ermodelrelationFieldmapping: {
                             fieldMappings: tableData.map((r) => {
                                 return {
                                     afield: r.field1Name,
@@ -126,7 +126,7 @@ export const CreateJoinModal = ({
                         created: true,
                     },
                     editableProperties: {
-                        name: joinName,
+                        name: ermodelrelationName,
                         description: details,
                     },
                 },
@@ -140,7 +140,7 @@ export const CreateJoinModal = ({
                 setTimeout(() => {
                     refetch();
                     message.success({
-                        content: `Join created!`,
+                        content: `ERModelRelation created!`,
                         duration: 2,
                     });
                 }, 2000);
@@ -148,7 +148,7 @@ export const CreateJoinModal = ({
                     variables: {
                         input: {
                             ownerUrn: user?.urn || '',
-                            resourceUrn: data?.createJoin?.urn || '',
+                            resourceUrn: data?.createERModelRelation?.urn || '',
                             ownershipTypeUrn: 'urn:li:ownershipType:__system__technical_owner',
                             ownerEntityType: ownerEntityType || EntityType,
                         },
@@ -157,22 +157,22 @@ export const CreateJoinModal = ({
             })
             .catch((e) => {
                 message.destroy();
-                message.error({ content: `Failed to create join: ${e.message || ''}`, duration: 3 });
+                message.error({ content: `Failed to create ermodelrelation: ${e.message || ''}`, duration: 3 });
             });
     };
-    const originalJoinName = editJoin?.properties?.name;
-    const updateJoin = () => {
+    const originalERModelRelationName = editERModelRelation?.properties?.name;
+    const updateERModelRelation = () => {
         updateMutation({
             variables: {
-                urn: editJoin?.urn || '',
+                urn: editERModelRelation?.urn || '',
                 input: {
                     properties: {
                         dataSetA: table1Dataset?.urn || '',
                         datasetB: table2Dataset?.urn || '',
-                        name: originalJoinName || '',
-                        createdBy: editJoin?.properties?.createdActor?.urn || user?.urn,
-                        createdAt: editJoin?.properties?.createdTime || 0,
-                        joinFieldmapping: {
+                        name: originalERModelRelationName || '',
+                        createdBy: editERModelRelation?.properties?.createdActor?.urn || user?.urn,
+                        createdAt: editERModelRelation?.properties?.createdTime || 0,
+                        ermodelrelationFieldmapping: {
                             fieldMappings: tableData.map((r) => {
                                 return {
                                     afield: r.field1Name,
@@ -182,7 +182,7 @@ export const CreateJoinModal = ({
                         },
                     },
                     editableProperties: {
-                        name: joinName,
+                        name: ermodelrelationName,
                         description: details,
                     },
                 },
@@ -196,28 +196,28 @@ export const CreateJoinModal = ({
                 setTimeout(() => {
                     refetch();
                     message.success({
-                        content: `Join updated!`,
+                        content: `ERModelRelation updated!`,
                         duration: 2,
                     });
                 }, 2000);
             })
             .catch((e) => {
                 message.destroy();
-                message.error({ content: `Failed to update join: ${e.message || ''}`, duration: 3 });
+                message.error({ content: `Failed to update ermodelrelation: ${e.message || ''}`, duration: 3 });
             });
     };
     const onSubmit = async () => {
-        const errors = validateJoin(joinName, tableData, isEditing, getSearchResultsJoins);
+        const errors = validateERModelRelation(ermodelrelationName, tableData, isEditing, getSearchResultsERModelRelations);
         if ((await errors).length > 0) {
             const err = (await errors).join(`, `);
             message.error(err);
             return;
         }
         if (isEditing) {
-            updateJoin();
+            updateERModelRelation();
         } else {
-            createJoin();
-            setJoinName('');
+            createERModelRelation();
+            setERModelRelationName('');
             setDetails('');
             setTableData([
                 { key: '0', field1Name: '', field2Name: '' },
@@ -234,7 +234,7 @@ export const CreateJoinModal = ({
     const table2NameTech = table2Dataset?.name || table2Dataset?.urn.split(',').at(1) || '';
 
     const handleAdd = () => {
-        const newData: JoinDataType = {
+        const newData: ERModelRelationDataType = {
             key: count,
             field1Name: '',
             field2Name: '',
@@ -287,7 +287,7 @@ export const CreateJoinModal = ({
                 ) : null,
         },
     ];
-    const handleSave = (row: JoinDataType) => {
+    const handleSave = (row: ERModelRelationDataType) => {
         const newData = [...tableData];
         const index = newData.findIndex((item) => row.key === item.key);
         const item = newData[index];
@@ -310,7 +310,7 @@ export const CreateJoinModal = ({
         }
         return {
             ...col,
-            onCell: (record: JoinDataType) => ({
+            onCell: (record: ERModelRelationDataType) => ({
                 record,
                 editable: col.editable,
                 dataIndex: col.dataIndex,
@@ -324,7 +324,7 @@ export const CreateJoinModal = ({
         <Modal
             title={
                 <div className="footer-parent-div">
-                    <p className="join-title">Join parameters</p>
+                    <p className="ermodelrelation-title">ERModelRelation parameters</p>
                     <div>
                         <Button onClick={onCancelSelect} className="cancel-btn" size="large">
                             Cancel
@@ -339,7 +339,7 @@ export const CreateJoinModal = ({
             }
             visible={visible}
             closable={false}
-            className="CreateJoinModal"
+            className="CreateERModelRelationModal"
             okButtonProps={{ hidden: true }}
             cancelButtonProps={{ hidden: true }}
             onCancel={onCancelSelect}
@@ -352,31 +352,31 @@ export const CreateJoinModal = ({
                 <p className="all-content-heading">Table 2</p>
                 <p className="all-information">{table2NameBusiness}</p>
                 <div className="techNameDisplay">{table2NameTech !== table2NameBusiness && table2NameTech}</div>
-                <p className="all-content-heading">Join name</p>
+                <p className="all-content-heading">ERModelRelation name</p>
                 <Form
                     form={form}
                     layout="vertical"
                     fields={[
-                        { name: 'joinNameForm', value: joinName },
-                        { name: 'joinDetails', value: details },
+                        { name: 'ermodelrelationNameForm', value: ermodelrelationName },
+                        { name: 'ermodelrelationDetails', value: details },
                     ]}
                     onFinish={onSubmit}
                 >
                     <Form.Item
                         style={{ margin: 0 }}
-                        name="joinNameForm"
+                        name="ermodelrelationNameForm"
                         rules={[
                             {
                                 required: true,
-                                message: `Join name is required.`,
+                                message: `ERModelRelation name is required.`,
                             },
                             {
                                 validator: (_, value) =>
-                                    checkDuplicateJoin(getSearchResultsJoins, value?.trim()).then((result) => {
+                                    checkDuplicateERModelRelation(getSearchResultsERModelRelations, value?.trim()).then((result) => {
                                         return result === true && !isEditing
                                             ? Promise.reject(
                                                   new Error(
-                                                      'This join name already exists. A unique name for each join is required.',
+                                                      'This ermodelrelation name already exists. A unique name for each ermodelrelation is required.',
                                                   ),
                                               )
                                             : Promise.resolve();
@@ -384,25 +384,25 @@ export const CreateJoinModal = ({
                             },
                         ]}
                     >
-                        <Input size="large" className="join-name" onChange={(e) => setJoinName(e.target.value)} />
+                        <Input size="large" className="ermodelrelation-name" onChange={(e) => setERModelRelationName(e.target.value)} />
                     </Form.Item>
                     <p className="all-content-heading">Fields</p>
                     <Table
                         bordered
                         components={components}
                         dataSource={tableData}
-                        className="JoinTable"
+                        className="ERModelRelationTable"
                         columns={columns as ColumnTypes}
                         pagination={false}
                     />
                     <Button type="link" className="add-btn-link" onClick={handleAdd}>
                         <PlusOutlined /> Add Row
                     </Button>
-                    <p className="all-content-heading">Join details</p>
-                    <Form.Item style={{ margin: 0 }} name="joinDetails">
+                    <p className="all-content-heading">ERModelRelation details</p>
+                    <Form.Item style={{ margin: 0 }} name="ermodelrelationDetails">
                         <TextArea
-                            className="join-details-ta"
-                            placeholder="Please enter join details here"
+                            className="ermodelrelation-details-ta"
+                            placeholder="Please enter ermodelrelation details here"
                             onChange={(e) => setDetails(e.target.value)}
                         />
                     </Form.Item>
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.less b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationPreview.less
similarity index 98%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.less
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationPreview.less
index f7a127a6974da..8583a2e1b43dd 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.less
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationPreview.less
@@ -1,6 +1,6 @@
 @import "../../../../../../../node_modules/antd/dist/antd.less";
 
-.JoinPreview {
+.ERModelRelationPreview {
     .preview-main-div{
         display: flex;
         justify-content: space-between;
@@ -113,7 +113,7 @@
     .extra-margin-rev {
         margin-left: 4px;
     }
-    .JoinTable {
+    .ERModelRelationTable {
         width: 100%;
         .ant-table-bordered{
             margin-bottom: 12px;
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationPreview.tsx
similarity index 63%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.tsx
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationPreview.tsx
index 2ffe99dad2b0e..9629d0e5bb13f 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationPreview.tsx
@@ -4,65 +4,65 @@ import { RightOutlined } from '@ant-design/icons';
 import { useEntityRegistry } from '../../../../../useEntityRegistry';
 import arrow from '../../../../../../images/Arrow.svg';
 import editIcon from '../../../../../../images/editIconBlack.svg';
-import './JoinPreview.less';
-import { EntityType, Join } from '../../../../../../types.generated';
-import { CreateJoinModal } from './CreateJoinModal';
-import { getDatasetName } from './JoinUtils';
+import './ERModelRelationPreview.less';
+import { EntityType, ErModelRelation } from '../../../../../../types.generated';
+import { CreateERModelRelationModal } from './CreateERModelRelationModal';
+import { getDatasetName } from './ERModelRelationUtils';
 
-type JoinRecord = {
+type ERModelRelationRecord = {
     afield: string;
     bfield: string;
 };
 type Props = {
-    joinData: Join;
+    ermodelrelationData: ErModelRelation;
     baseEntityUrn?: any;
     prePageType?: string;
     refetch: () => Promise<any>;
 };
 type EditableTableProps = Parameters<typeof Table>[0];
 type ColumnTypes = Exclude<EditableTableProps['columns'], undefined>;
-export const JoinPreview = ({ joinData, baseEntityUrn, prePageType, refetch }: Props) => {
+export const ERModelRelationPreview = ({ ermodelrelationData, baseEntityUrn, prePageType, refetch }: Props) => {
     const entityRegistry = useEntityRegistry();
     const handleViewEntity = (entityType, urn) => {
         const entityUrl = entityRegistry.getEntityUrl(entityType, urn);
         window.open(entityUrl, '_blank');
     };
     const [modalVisible, setModalVisible] = useState(false);
-    const shuffleFlag = !(prePageType === 'Dataset' && baseEntityUrn === joinData?.properties?.datasetA?.urn);
+    const shuffleFlag = !(prePageType === 'Dataset' && baseEntityUrn === ermodelrelationData?.properties?.datasetA?.urn);
     const table1EditableName = shuffleFlag
-        ? getDatasetName(joinData?.properties?.datasetB)
-        : getDatasetName(joinData?.properties?.datasetA);
+        ? getDatasetName(ermodelrelationData?.properties?.datasetB)
+        : getDatasetName(ermodelrelationData?.properties?.datasetA);
     const table2EditableName = shuffleFlag
-        ? getDatasetName(joinData?.properties?.datasetA)
-        : getDatasetName(joinData?.properties?.datasetB);
+        ? getDatasetName(ermodelrelationData?.properties?.datasetA)
+        : getDatasetName(ermodelrelationData?.properties?.datasetB);
     const table1Name =
-        shuffleFlag && prePageType !== 'Join'
-            ? joinData?.properties?.datasetB?.name
-            : joinData?.properties?.datasetA?.name;
+        shuffleFlag && prePageType !== 'ERModelRelation'
+            ? ermodelrelationData?.properties?.datasetB?.name
+            : ermodelrelationData?.properties?.datasetA?.name;
     const table2Name =
-        shuffleFlag && prePageType !== 'Join'
-            ? joinData?.properties?.datasetA?.name
-            : joinData?.properties?.datasetB?.name;
+        shuffleFlag && prePageType !== 'ERModelRelation'
+            ? ermodelrelationData?.properties?.datasetA?.name
+            : ermodelrelationData?.properties?.datasetB?.name;
     const table1Urn =
-        shuffleFlag && prePageType !== 'Join'
-            ? joinData?.properties?.datasetB?.urn
-            : joinData?.properties?.datasetA?.urn;
+        shuffleFlag && prePageType !== 'ERModelRelation'
+            ? ermodelrelationData?.properties?.datasetB?.urn
+            : ermodelrelationData?.properties?.datasetA?.urn;
     const table2Urn =
-        shuffleFlag && prePageType !== 'Join'
-            ? joinData?.properties?.datasetA?.urn
-            : joinData?.properties?.datasetB?.urn;
-    const joinHeader = joinData?.editableProperties?.name || joinData?.properties?.name || '';
-    function getFieldMap(): JoinRecord[] {
-        const newData = [] as JoinRecord[];
-        if (shuffleFlag && prePageType !== 'Join') {
-            joinData?.properties?.joinFieldMapping?.fieldMappings?.map((item) => {
+        shuffleFlag && prePageType !== 'ERModelRelation'
+            ? ermodelrelationData?.properties?.datasetA?.urn
+            : ermodelrelationData?.properties?.datasetB?.urn;
+    const ermodelrelationHeader = ermodelrelationData?.editableProperties?.name || ermodelrelationData?.properties?.name || '';
+    function getFieldMap(): ERModelRelationRecord[] {
+        const newData = [] as ERModelRelationRecord[];
+        if (shuffleFlag && prePageType !== 'ERModelRelation') {
+            ermodelrelationData?.properties?.ermodelrelationFieldMapping?.fieldMappings?.map((item) => {
                 return newData.push({
                     afield: item.bfield,
                     bfield: item.afield,
                 });
             });
         } else {
-            joinData?.properties?.joinFieldMapping?.fieldMappings?.map((item) => {
+            ermodelrelationData?.properties?.ermodelrelationFieldMapping?.fieldMappings?.map((item) => {
                 return newData.push({
                     afield: item.afield,
                     bfield: item.bfield,
@@ -78,7 +78,7 @@ export const JoinPreview = ({ joinData, baseEntityUrn, prePageType, refetch }: P
                     <div className="firstRow">
                         <div className="titleNameDisplay"> {table1EditableName || table1Name}</div>
                         <div>
-                            {prePageType === 'Join' && (
+                            {prePageType === 'ERModelRelation' && (
                                 <Button
                                     type="link"
                                     className="div-view-dataset"
@@ -128,28 +128,28 @@ export const JoinPreview = ({ joinData, baseEntityUrn, prePageType, refetch }: P
     ];
 
     return (
-        <div className="JoinPreview">
-            {joinData?.properties?.joinFieldMapping !== undefined && (
-                <CreateJoinModal
+        <div className="ERModelRelationPreview">
+            {ermodelrelationData?.properties?.ermodelrelationFieldMapping !== undefined && (
+                <CreateERModelRelationModal
                     visible={modalVisible}
                     setModalVisible={setModalVisible}
                     onCancel={() => {
                         setModalVisible(false);
                     }}
-                    editJoin={joinData}
+                    editERModelRelation={ermodelrelationData}
                     isEditing
                     refetch={refetch}
                 />
             )}
             <div className="preview-main-div">
                 <div>
-                    {(prePageType === 'Dataset' || joinHeader !== joinData?.properties?.name) && (
+                    {(prePageType === 'Dataset' || ermodelrelationHeader !== ermodelrelationData?.properties?.name) && (
                         <Row>
-                            <p className="all-table-heading">{joinHeader}</p>
+                            <p className="all-table-heading">{ermodelrelationHeader}</p>
                             {prePageType === 'Dataset' && (
-                                <Button type="link" onClick={() => handleViewEntity(EntityType.Join, joinData?.urn)}>
+                                <Button type="link" onClick={() => handleViewEntity(EntityType.Ermodelrelation, ermodelrelationData?.urn)}>
                                     <div className="div-view">
-                                        View join <RightOutlined />{' '}
+                                        View ermodelrelation <RightOutlined />{' '}
                                     </div>
                                 </Button>
                             )}
@@ -165,8 +165,8 @@ export const JoinPreview = ({ joinData, baseEntityUrn, prePageType, refetch }: P
                         }}
                     >
                         <div className="div-edit-img">
-                            <img src={editIcon} alt="" /> <div className="div-edit">Edit Join</div>
-                            {prePageType === 'Join' && <div className="extra-margin-rev" />}
+                            <img src={editIcon} alt="" /> <div className="div-edit">Edit ERModelRelation</div>
+                            {prePageType === 'ERModelRelation' && <div className="extra-margin-rev" />}
                         </div>
                     </Button>
                 </div>
@@ -175,15 +175,15 @@ export const JoinPreview = ({ joinData, baseEntityUrn, prePageType, refetch }: P
                 <Table
                     bordered
                     dataSource={getFieldMap()}
-                    className="JoinTable"
+                    className="ERModelRelationTable"
                     columns={columns as ColumnTypes}
                     pagination={false}
                 />
             </Row>
             {prePageType === 'Dataset' && (
                 <div>
-                    <p className="all-content-heading">Join details</p>
-                    <p className="all-content-info">{joinData?.editableProperties?.description}</p>
+                    <p className="all-content-heading">ERModelRelation details</p>
+                    <p className="all-content-info">{ermodelrelationData?.editableProperties?.description}</p>
                 </div>
             )}
         </div>
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/JoinUtils.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationUtils.tsx
similarity index 60%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/JoinUtils.tsx
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationUtils.tsx
index f37d5b3c570f1..39aa140e29796 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/JoinUtils.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/ERModelRelationUtils.tsx
@@ -3,16 +3,16 @@ import { FormInstance } from 'antd';
 import { EntityType } from '../../../../../../types.generated';
 
 export const EditableContext = React.createContext<FormInstance<any> | null>(null);
-export interface JoinDataType {
+export interface ERModelRelationDataType {
     key: React.Key;
     field1Name: string;
     field2Name: string;
 }
 
-export const checkDuplicateJoin = async (getSearchResultsJoins, name): Promise<boolean> => {
-    const { data: searchResults } = await getSearchResultsJoins({
+export const checkDuplicateERModelRelation = async (getSearchResultsERModelRelations, name): Promise<boolean> => {
+    const { data: searchResults } = await getSearchResultsERModelRelations({
         input: {
-            type: EntityType.Join,
+            type: EntityType.Ermodelrelation,
             query: '',
             orFilters: [
                 {
@@ -31,24 +31,24 @@ export const checkDuplicateJoin = async (getSearchResultsJoins, name): Promise<b
     const recordExists = searchResults && searchResults.search && searchResults.search.total > 0;
     return recordExists || false;
 };
-const validateTableData = (fieldMappingData: JoinDataType) => {
+const validateTableData = (fieldMappingData: ERModelRelationDataType) => {
     if (fieldMappingData.field1Name !== '' && fieldMappingData.field2Name !== '') {
         return true;
     }
     return false;
 };
-export const validateJoin = async (nameField: string, tableSchema: JoinDataType[], editFlag, getSearchResultsJoins) => {
+export const validateERModelRelation = async (nameField: string, tableSchema: ERModelRelationDataType[], editFlag, getSearchResultsERModelRelations) => {
     const errors: string[] = [];
-    const bDuplicateName = await checkDuplicateJoin(getSearchResultsJoins, nameField?.trim()).then((result) => result);
+    const bDuplicateName = await checkDuplicateERModelRelation(getSearchResultsERModelRelations, nameField?.trim()).then((result) => result);
     if (nameField === '') {
-        errors.push('Join name is required');
+        errors.push('ERModelRelation name is required');
     }
     if (bDuplicateName && !editFlag) {
-        errors.push('This join name already exists. A unique name for each join is required');
+        errors.push('This ermodelrelation name already exists. A unique name for each ermodelrelation is required');
     }
     const faultyRows = tableSchema.filter((item) => validateTableData(item) !== true);
     if (faultyRows.length > 0) {
-        errors.push('Please fill out or remove all empty Join fields');
+        errors.push('Please fill out or remove all empty ERModelRelation fields');
     }
     return errors;
 };
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/EditableCell.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/EditableCell.tsx
similarity index 87%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/EditableCell.tsx
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/EditableCell.tsx
index 0b22482cbbd7a..a190d9090cc8c 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/EditableCell.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/EditableCell.tsx
@@ -1,18 +1,18 @@
 import React, { useContext } from 'react';
 import { Form, Select } from 'antd';
-import { EditableContext } from './JoinUtils';
+import { EditableContext } from './ERModelRelationUtils';
 import { Dataset } from '../../../../../../types.generated';
 
 interface EditableCellProps {
     editable: boolean;
     children: React.ReactNode;
-    dataIndex: keyof JoinRecord;
-    record: JoinRecord;
+    dataIndex: keyof ERModelRelationRecord;
+    record: ERModelRelationRecord;
     tableRecord?: Dataset;
     value?: any;
-    handleSave: (record: JoinRecord) => void;
+    handleSave: (record: ERModelRelationRecord) => void;
 }
-interface JoinRecord {
+interface ERModelRelationRecord {
     key: string;
     field1Name: string;
     field2Name: string;
@@ -52,7 +52,7 @@ export const EditableCell = ({
             >
                 <Select
                     size="large"
-                    className="join-select-selector"
+                    className="ermodelrelation-select-selector"
                     options={tableRecord?.schemaMetadata?.fields.map((result) => ({
                         value: result.fieldPath,
                         label: result.fieldPath,
diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/EditableRow.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/EditableRow.tsx
similarity index 88%
rename from datahub-web-react/src/app/entity/shared/components/styled/Join/EditableRow.tsx
rename to datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/EditableRow.tsx
index fb409619b0304..120dff44cfc0d 100644
--- a/datahub-web-react/src/app/entity/shared/components/styled/Join/EditableRow.tsx
+++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelation/EditableRow.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { Form } from 'antd';
-import { EditableContext } from './JoinUtils';
+import { EditableContext } from './ERModelRelationUtils';
 
 interface EditableRowProps {
     index: number;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
index af2cf19066a67..7275f01d83417 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
@@ -167,7 +167,7 @@ export const EntityProfile = <T, U>({
     subHeader,
 }: Props<T, U>): JSX.Element => {
     const { config } = useAppConfig();
-    const { joinFeatureEnabled } = config.featureFlags;
+    const { ermodelrelationFeatureEnabled } = config.featureFlags;
     const isLineageMode = useIsLineageMode();
     const isHideSiblingMode = useIsSeparateSiblingsMode();
     const entityRegistry = useEntityRegistry();
@@ -176,7 +176,7 @@ export const EntityProfile = <T, U>({
     const isCompact = React.useContext(CompactContext);
     const tabsWithDefaults = tabs.map((tab) => ({ ...tab, display: { ...defaultTabDisplayConfig, ...tab.display } }));
 
-    if (joinFeatureEnabled) {
+    if (ermodelrelationFeatureEnabled) {
         const relationIndex = tabsWithDefaults.findIndex((tab) => {
             return tab.name === 'Relationships';
         });
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueryBuilderForm.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueryBuilderForm.tsx
index b6780881c6df9..940c302e88828 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueryBuilderForm.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/QueryBuilderForm.tsx
@@ -77,7 +77,7 @@ export default function QueryBuilderForm({ state, updateState }: Props) {
                     autoFocus
                     value={state.title}
                     onChange={(newTitle) => updateTitle(newTitle.target.value)}
-                    placeholder="Join Transactions and Users Tables"
+                    placeholder="ERModelRelation Transactions and Users Tables"
                 />
             </Form.Item>
             <Form.Item label={<Typography.Text strong>Description</Typography.Text>}>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationshipsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationshipsTab.tsx
index ad1aad5569c93..8fcdcfc6aaaa9 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationshipsTab.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationshipsTab.tsx
@@ -4,7 +4,7 @@ import styled from 'styled-components';
 import { ExclamationCircleFilled, LoadingOutlined, PlusOutlined, SearchOutlined } from '@ant-design/icons';
 import { useBaseEntity } from '../../../EntityContext';
 import './RelationshipsTab.less';
-import { EntityType, Join } from '../../../../../../types.generated';
+import { EntityType, ErModelRelation } from '../../../../../../types.generated';
 import { useGetSearchResultsQuery } from '../../../../../../graphql/search.generated';
 import {
     GetDatasetQuery,
@@ -13,8 +13,8 @@ import {
 } from '../../../../../../graphql/dataset.generated';
 import { useGetEntityWithSchema } from '../Schema/useGetEntitySchema';
 import closeIcon from '../../../../../../images/close_dark.svg';
-import { CreateJoinModal } from '../../../components/styled/Join/CreateJoinModal';
-import { JoinPreview } from '../../../components/styled/Join/JoinPreview';
+import { CreateERModelRelationModal } from '../../../components/styled/ERModelRelation/CreateERModelRelationModal';
+import { ERModelRelationPreview } from '../../../components/styled/ERModelRelation/ERModelRelationPreview';
 import { SearchSelectModal } from '../../../components/styled/search/SearchSelectModal';
 import { ANTD_GRAY } from '../../../constants';
 
@@ -36,7 +36,7 @@ const ThinDivider = styled(Divider)`
     margin-left: -70px;
     margin-bottom: 0px;
 `;
-const NoJoins = styled(Empty)`
+const NoERModelRelations = styled(Empty)`
     color: ${ANTD_GRAY[6]};
     padding-top: 60px;
 `;
@@ -48,22 +48,22 @@ export const RelationshipsTab = () => {
     // Dynamically load the schema + editable schema information.
     const { entityWithSchema } = useGetEntityWithSchema();
     const [modalVisible, setModalVisible] = useState(false);
-    const [joinModalVisible, setjoinModalVisible] = useState(false);
+    const [ermodelrelationModalVisible, setermodelrelationModalVisible] = useState(false);
     const tabs = [
         {
-            key: 'joinsTab',
-            tab: 'Joins',
+            key: 'ermodelrelationsTab',
+            tab: 'ERModelRelations',
         },
     ];
     const {
-        data: joins,
-        loading: loadingJoin,
-        error: errorJoin,
+        data: ermodelrelations,
+        loading: loadingERModelRelation,
+        error: errorERModelRelation,
         refetch,
     } = useGetSearchResultsQuery({
         variables: {
             input: {
-                type: EntityType.Join,
+                type: EntityType.Ermodelrelation,
                 query: `${filterText ? `${filterText}` : ''}`,
                 orFilters: [
                     {
@@ -84,29 +84,29 @@ export const RelationshipsTab = () => {
                     },
                 ],
                 start: currentPage * pageSize,
-                count: pageSize, // all matched joins
+                count: pageSize, // all matched ermodelrelations
             },
         },
     });
-    const totalResults = joins?.search?.total || 0;
-    let joinData: Join[] = [];
-    if (loadingJoin) {
-        joinData = [{}] as Join[];
+    const totalResults = ermodelrelations?.search?.total || 0;
+    let ermodelrelationData: ErModelRelation[] = [];
+    if (loadingERModelRelation) {
+        ermodelrelationData = [{}] as ErModelRelation[];
     }
 
-    if (!loadingJoin && joins?.search && joins?.search?.searchResults?.length > 0 && !errorJoin) {
-        joinData = joins.search.searchResults.map((r) => r.entity as Join);
+    if (!loadingERModelRelation && ermodelrelations?.search && ermodelrelations?.search?.searchResults?.length > 0 && !errorERModelRelation) {
+        ermodelrelationData = ermodelrelations.search.searchResults.map((r) => r.entity as ErModelRelation);
     }
 
     const contentListNoTitle: Record<string, React.ReactNode> = {
-        joinsTab:
-            joinData.length > 0 && !loadingJoin ? (
-                joinData.map((record) => {
+        ermodelrelationsTab:
+            ermodelrelationData.length > 0 && !loadingERModelRelation ? (
+                ermodelrelationData.map((record) => {
                     return (
                         <>
                             <div>
-                                <JoinPreview
-                                    joinData={record}
+                                <ERModelRelationPreview
+                                    ermodelrelationData={record}
                                     baseEntityUrn={baseEntity?.dataset?.urn}
                                     prePageType="Dataset"
                                     refetch={refetch}
@@ -118,20 +118,20 @@ export const RelationshipsTab = () => {
                 })
             ) : (
                 <>
-                    {!loadingJoin && (
+                    {!loadingERModelRelation && (
                         <div>
-                            <NoJoins />
+                            <NoERModelRelations />
                         </div>
                     )}
-                    {loadingJoin && (
+                    {loadingERModelRelation && (
                         <div>
-                            Joins <LoadingOutlined />
+                            ERModelRelations <LoadingOutlined />
                         </div>
                     )}
                 </>
             ),
     };
-    const [activeTabKey, setActiveTabKey] = useState<string>('joinsTab');
+    const [activeTabKey, setActiveTabKey] = useState<string>('ermodelrelationsTab');
     const onTabChange = (key: string) => {
         setActiveTabKey(key);
     };
@@ -156,7 +156,7 @@ export const RelationshipsTab = () => {
                 <div>
                     <ThinDivider />
                     <p className="msg-div-inner">
-                        A schema was not ingested for the dataset selected. Join cannot be created.
+                        A schema was not ingested for the dataset selected. ERModelRelation cannot be created.
                     </p>
                     <ThinDivider />
                 </div>
@@ -180,13 +180,13 @@ export const RelationshipsTab = () => {
             table2LazySchema?.dataset !== undefined &&
             table2LazySchema?.dataset?.schemaMetadata?.fields !== undefined
         ) {
-            setjoinModalVisible(false);
+            setermodelrelationModalVisible(false);
             setModalVisible(true);
         }
     }, [table2LazySchema]);
     return (
         <>
-            {joinModalVisible && (
+            {ermodelrelationModalVisible && (
                 <SearchSelectModal
                     titleText="Select Table 2"
                     continueText="Submit"
@@ -202,14 +202,14 @@ export const RelationshipsTab = () => {
                             },
                         });
                     }}
-                    onCancel={() => setjoinModalVisible(false)}
+                    onCancel={() => setermodelrelationModalVisible(false)}
                     fixedEntityTypes={[EntityType.Dataset]}
                     singleSelect
                     hideToolbar
                 />
             )}
             {baseEntity !== undefined && (
-                <CreateJoinModal
+                <CreateERModelRelationModal
                     table1={baseEntity}
                     table1Schema={entityWithSchema}
                     table2={table2LazyDataset}
@@ -234,7 +234,7 @@ export const RelationshipsTab = () => {
                 <div className="search-header-div">
                     <StyledInput
                         defaultValue={filterText}
-                        placeholder="Find join..."
+                        placeholder="Find ermodelrelation..."
                         onChange={(e) => setFilterText(e.target.value)}
                         allowClear
                         autoFocus
@@ -245,10 +245,10 @@ export const RelationshipsTab = () => {
                         className="add-btn-link"
                         hidden={entityWithSchema?.schemaMetadata?.fields === undefined}
                         onClick={() => {
-                            setjoinModalVisible(true);
+                            setermodelrelationModalVisible(true);
                         }}
                     >
-                        <PlusOutlined /> Add Join
+                        <PlusOutlined /> Add ERModelRelation
                     </Button>
                 </div>{' '}
                 <br />
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.less b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.less
similarity index 94%
rename from datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.less
rename to datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.less
index 03057a35f0491..bb8f2f04f6a37 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.less
+++ b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.less
@@ -1,6 +1,6 @@
 @import "../../../../../../node_modules/antd/dist/antd.less";
 
-.JoinTab {
+.ERModelRelationTab {
     .add-btn-link {
         padding-left: 1155px !important;
         padding-top: 16px !important;
@@ -37,7 +37,7 @@
         background: #FFFFFF;
         box-shadow: 0px 2px 6px rgba(0, 0, 0, 0.05);
     }
-    .join-preview-div {
+    .ermodelrelation-preview-div {
         padding-left: 22px;
         padding-top: 34px;
     }
diff --git a/datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.tsx
new file mode 100644
index 0000000000000..f6f527840d3f0
--- /dev/null
+++ b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelation/ERModelRelationTab.tsx
@@ -0,0 +1,21 @@
+import React from 'react';
+import { Divider } from 'antd';
+import './ERModelRelationTab.less';
+import { ERModelRelationPreview } from '../../components/styled/ERModelRelation/ERModelRelationPreview';
+import { useEntityData, useRefetch } from '../../EntityContext';
+
+export const ERModelRelationTab = () => {
+    const { entityData } = useEntityData();
+    const refetch = useRefetch();
+    const ermodelrelationView = (ermodelrelationData?: any): JSX.Element => {
+        return <ERModelRelationPreview ermodelrelationData={ermodelrelationData} prePageType="ERModelRelation" refetch={refetch} />;
+    };
+    return (
+        <>
+            <div className="ERModelRelationTab">
+                <div className="ermodelrelation-preview-div">{ermodelrelationView(entityData)}</div>
+                <Divider className="thin-divider" />
+            </div>
+        </>
+    );
+};
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.tsx
deleted file mode 100644
index b02964f953206..0000000000000
--- a/datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.tsx
+++ /dev/null
@@ -1,21 +0,0 @@
-import React from 'react';
-import { Divider } from 'antd';
-import './JoinTab.less';
-import { JoinPreview } from '../../components/styled/Join/JoinPreview';
-import { useEntityData, useRefetch } from '../../EntityContext';
-
-export const JoinTab = () => {
-    const { entityData } = useEntityData();
-    const refetch = useRefetch();
-    const joinView = (joinData?: any): JSX.Element => {
-        return <JoinPreview joinData={joinData} prePageType="Join" refetch={refetch} />;
-    };
-    return (
-        <>
-            <div className="JoinTab">
-                <div className="join-preview-div">{joinView(entityData)}</div>
-                <Divider className="thin-divider" />
-            </div>
-        </>
-    );
-};
diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx
index 99dfe512eaa32..fd325c558e45d 100644
--- a/datahub-web-react/src/appConfigContext.tsx
+++ b/datahub-web-react/src/appConfigContext.tsx
@@ -48,7 +48,7 @@ export const DEFAULT_APP_CONFIG = {
         showSearchFiltersV2: true,
         showBrowseV2: true,
         showAcrylInfo: false,
-        joinFeatureEnabled: false,
+        ermodelrelationFeatureEnabled: false,
         showAccessManagement: false,
         nestedDomainsEnabled: true,
     },
diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql
index 21f24da665bb2..ffd3708a7523b 100644
--- a/datahub-web-react/src/graphql/app.graphql
+++ b/datahub-web-react/src/graphql/app.graphql
@@ -63,7 +63,7 @@ query appConfig {
             showSearchFiltersV2
             showBrowseV2
             showAcrylInfo
-            joinFeatureEnabled
+            ermodelrelationFeatureEnabled
             showAccessManagement
             nestedDomainsEnabled
         }
diff --git a/datahub-web-react/src/graphql/join.graphql b/datahub-web-react/src/graphql/ermodelrelation.graphql
similarity index 53%
rename from datahub-web-react/src/graphql/join.graphql
rename to datahub-web-react/src/graphql/ermodelrelation.graphql
index 6fc2f2afaf498..60a0b74751cbc 100644
--- a/datahub-web-react/src/graphql/join.graphql
+++ b/datahub-web-react/src/graphql/ermodelrelation.graphql
@@ -1,13 +1,13 @@
-query getJoin($urn: String!) {
-    join(urn: $urn) {
+query getERModelRelation($urn: String!) {
+    ermodelrelation(urn: $urn) {
         urn
         type
-        joinId
+        ermodelrelationId
         properties {
-            ...joinPropertiesFields
+            ...ermodelrelationPropertiesFields
         }
         editableProperties {
-            ...joinEditablePropertiesFields
+            ...ermodelrelationEditablePropertiesFields
         }
         institutionalMemory {
             ...institutionalMemoryFields
@@ -25,7 +25,7 @@ query getJoin($urn: String!) {
             ...glossaryTerms
         }
         outgoing: relationships(
-            input: { types: ["joinA", "joinB"], direction: OUTGOING, start: 0, count: 100 }
+            input: { types: ["ermodelrelationA", "ermodelrelationB"], direction: OUTGOING, start: 0, count: 100 }
         ) {
             start
             count
@@ -34,7 +34,7 @@ query getJoin($urn: String!) {
                 type
                 direction
                 entity {
-                    ... on Join {
+                    ... on ERModelRelation {
                         urn
                     }
 
@@ -44,14 +44,14 @@ query getJoin($urn: String!) {
     }
 }
 
-mutation updateJoin($urn: String!, $input: JoinUpdateInput!) {
-    updateJoin(urn: $urn, input: $input)
+mutation updateERModelRelation($urn: String!, $input: ERModelRelationUpdateInput!) {
+    updateERModelRelation(urn: $urn, input: $input)
 }
 
-mutation createJoin($input: JoinUpdateInput!) {
-    createJoin(input: $input) {
+mutation createERModelRelation($input: ERModelRelationUpdateInput!) {
+    createERModelRelation(input: $input) {
         urn
         type
-        joinId
+        ermodelrelationId
     }
 }
\ No newline at end of file
diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql
index 1a72390a90c4c..0defbcc1e8d00 100644
--- a/datahub-web-react/src/graphql/fragments.graphql
+++ b/datahub-web-react/src/graphql/fragments.graphql
@@ -1150,16 +1150,16 @@ fragment entityDisplayNameFields on Entity {
     }
 }
 
-fragment joinPropertiesFields on JoinProperties {
+fragment ermodelrelationPropertiesFields on ERModelRelationProperties {
     name
     datasetA {
-        ...datasetJoinFields
+        ...datasetERModelRelationFields
     }
     datasetB {
-        ...datasetJoinFields
+        ...datasetERModelRelationFields
     }
-    joinFieldMapping {
-        ...joinFieldMappingFields
+    ermodelrelationFieldMapping {
+        ...ermodelrelationFieldMappingFields
     }
     createdTime
     createdActor {
@@ -1167,19 +1167,19 @@ fragment joinPropertiesFields on JoinProperties {
     }
 }
 
-fragment joinFieldMappingFields on JoinFieldMapping {
+fragment ermodelrelationFieldMappingFields on ERModelRelationFieldMapping {
     fieldMappings {
         afield
         bfield
     }
 }
 
-fragment joinEditablePropertiesFields on JoinEditableProperties {
+fragment ermodelrelationEditablePropertiesFields on ERModelRelationEditableProperties {
     description
     name
 }
 
-fragment datasetJoinFields on Dataset {
+fragment datasetERModelRelationFields on Dataset {
     urn
     name
     properties {
diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql
index f14bb8abbde67..153b633d13378 100644
--- a/datahub-web-react/src/graphql/lineage.graphql
+++ b/datahub-web-react/src/graphql/lineage.graphql
@@ -249,15 +249,15 @@ fragment lineageNodeProperties on EntityWithRelationships {
     ... on MLPrimaryKey {
         ...nonRecursiveMLPrimaryKey
     }
-    ... on Join {
+    ... on ERModelRelation {
         urn
         type
-        joinId
+        ermodelrelationId
         properties {
-            ...joinPropertiesFields
+            ...ermodelrelationPropertiesFields
         }
         editableProperties {
-            ...joinEditablePropertiesFields
+            ...ermodelrelationEditablePropertiesFields
         }
         ownership {
             ...ownershipFields
diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql
index 56d12de6e21ec..815d008ce8a0e 100644
--- a/datahub-web-react/src/graphql/search.graphql
+++ b/datahub-web-react/src/graphql/search.graphql
@@ -801,15 +801,15 @@ fragment searchResultFields on Entity {
     ... on DataProduct {
         ...dataProductSearchFields
     }
-    ... on Join {
+    ... on ERModelRelation {
         urn
         type
-        joinId
+        ermodelrelationId
         properties {
-            ...joinPropertiesFields
+            ...ermodelrelationPropertiesFields
         }
         editableProperties {
-            ...joinEditablePropertiesFields
+            ...ermodelrelationEditablePropertiesFields
         }
         ownership {
             ...ownershipFields
diff --git a/datahub-web-react/src/images/joinIcon.svg b/datahub-web-react/src/images/ermodelrelationIcon.svg
similarity index 100%
rename from datahub-web-react/src/images/joinIcon.svg
rename to datahub-web-react/src/images/ermodelrelationIcon.svg
diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
index 56785d4ac95ff..528e82c5aa76b 100644
--- a/docker/docker-compose.dev.yml
+++ b/docker/docker-compose.dev.yml
@@ -45,7 +45,7 @@ services:
     - SEARCH_SERVICE_ENABLE_CACHE=false
     - LINEAGE_SEARCH_CACHE_ENABLED=false
     - SHOW_BROWSE_V2=true
-    - JOIN_FEATURE_ENABLED=false
+    - ERMODELRELATION_FEATURE_ENABLED=false
     volumes:
     - ./datahub-gms/start.sh:/datahub/datahub-gms/scripts/start.sh
     - ./datahub-gms/jetty.xml:/datahub/datahub-gms/scripts/jetty.xml
diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md
index cd524dd35325b..5c07b7a01bcb1 100644
--- a/docs/deploy/environment-vars.md
+++ b/docs/deploy/environment-vars.md
@@ -14,7 +14,7 @@ DataHub works.
 | `UI_INGESTION_ENABLED`                           | `true`  | boolean   | [`GMS`, `MCE Consumer`]                 | Enable UI based ingestion.                                                                                                  |
 | `DATAHUB_ANALYTICS_ENABLED`                      | `true`  | boolean   | [`Frontend`, `GMS`]                     | Collect DataHub usage to populate the analytics dashboard.                                                                  |
 | `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true`  | boolean   | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. |
-| `JOIN_FEATURE_ENABLED`                           | `false` | boolean   | [`Frontend`, `GMS`]                     | Enable Join Feature that shows Relationships Tab within a Dataset UI.                                                       |
+| `ERMODELRELATION_FEATURE_ENABLED`                | `false` | boolean   | [`Frontend`, `GMS`]                     | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI.                                                       |
 
 
 ## Ingestion
@@ -91,4 +91,4 @@ Simply replace the dot, `.`, with an underscore, `_`, and convert to uppercase.
 | `AUTH_OIDC_GROUPS_CLAIM`           | `groups` | string    | [`Frontend`]  | Claim to use as the user's group.                                                                                                                                                                                                                  |
 | `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` | `false`  | boolean   | [`Frontend`]  | Auto-provision the group from the user's group claim.                                                                                                                                                                                              |
 | `AUTH_SESSION_TTL_HOURS`           | `24`     | string    | [`Frontend`]  | The number of hours a user session is valid. After this many hours the actor cookie will be expired by the browser and the user will be prompted to login again.                                                                                   |
-| `MAX_SESSION_TOKEN_AGE`            | `24h`    | string    | [`Frontend`]  | The maximum age of the session token. [User session tokens are stateless and will become invalid after this time](https://www.playframework.com/documentation/2.8.x/SettingsSession#Session-Timeout-/-Expiration) requiring a user to login again. | 
\ No newline at end of file
+| `MAX_SESSION_TOKEN_AGE`            | `24h`    | string    | [`Frontend`]  | The maximum age of the session token. [User session tokens are stateless and will become invalid after this time](https://www.playframework.com/documentation/2.8.x/SettingsSession#Session-Timeout-/-Expiration) requiring a user to login again. | 
diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java
index 8d7d064bb89ce..65c4186ff97e9 100644
--- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java
+++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java
@@ -56,7 +56,7 @@ public class Constants {
   public static final String TAG_ENTITY_NAME = "tag";
   public static final String CONTAINER_ENTITY_NAME = "container";
   public static final String DOMAIN_ENTITY_NAME = "domain";
-  public static final String JOIN_ENTITY_NAME = "join";
+  public static final String ERMODELRELATION_ENTITY_NAME = "ermodelrelation";
   public static final String ASSERTION_ENTITY_NAME = "assertion";
   public static final String INGESTION_SOURCE_ENTITY_NAME = "dataHubIngestionSource";
   public static final String SECRETS_ENTITY_NAME = "dataHubSecret";
@@ -239,10 +239,10 @@ public class Constants {
 
   public static final String DOMAIN_CREATED_TIME_INDEX_FIELD_NAME = "createdTime";
 
-  // Join
-  public static final String JOIN_KEY_ASPECT_NAME = "joinKey";
-  public static final String JOIN_PROPERTIES_ASPECT_NAME = "joinProperties";
-  public static final String EDITABLE_JOIN_PROPERTIES_ASPECT_NAME = "editableJoinProperties";
+  // ERModelRelation
+  public static final String ERMODELRELATION_KEY_ASPECT_NAME = "ermodelrelationKey";
+  public static final String ERMODELRELATION_PROPERTIES_ASPECT_NAME = "ermodelrelationProperties";
+  public static final String EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME = "editableERModelRelationProperties";
 
   // Assertion
   public static final String ASSERTION_KEY_ASPECT_NAME = "assertionKey";
diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/ERModelRelationUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/ERModelRelationUrn.java
new file mode 100644
index 0000000000000..fe9fb2dc53daa
--- /dev/null
+++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/ERModelRelationUrn.java
@@ -0,0 +1,66 @@
+package com.linkedin.common.urn;
+
+import com.linkedin.data.template.Custom;
+import com.linkedin.data.template.DirectCoercer;
+import com.linkedin.data.template.TemplateOutputCastException;
+import java.net.URISyntaxException;
+
+
+public class ERModelRelationUrn extends Urn {
+  public static final String ENTITY_TYPE = "ermodelrelation";
+
+  private final String _ermodelrelationId;
+
+  public ERModelRelationUrn(String ermodelrelationId) {
+    super(ENTITY_TYPE, TupleKey.create(ermodelrelationId));
+    this._ermodelrelationId = ermodelrelationId;
+  }
+
+  public String getERModelRelationIdEntity() {
+    return _ermodelrelationId;
+  }
+
+  public static ERModelRelationUrn createFromString(String rawUrn) throws URISyntaxException {
+    return createFromUrn(Urn.createFromString(rawUrn));
+  }
+
+  public static ERModelRelationUrn createFromUrn(Urn urn) throws URISyntaxException {
+    if (!"li".equals(urn.getNamespace())) {
+      throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'.");
+    } else if (!ENTITY_TYPE.equals(urn.getEntityType())) {
+      throw new URISyntaxException(urn.toString(), "Urn entity type should be 'ermodelrelation'.");
+    } else {
+      TupleKey key = urn.getEntityKey();
+      if (key.size() != 1) {
+        throw new URISyntaxException(urn.toString(), "Invalid number of keys.");
+      } else {
+        try {
+          return new ERModelRelationUrn((String) key.getAs(0, String.class));
+        } catch (Exception var3) {
+          throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + var3.getMessage());
+        }
+      }
+    }
+  }
+
+  public static ERModelRelationUrn deserialize(String rawUrn) throws URISyntaxException {
+    return createFromString(rawUrn);
+  }
+
+  static {
+    Custom.initializeCustomClass(ERModelRelationUrn.class);
+    Custom.registerCoercer(new DirectCoercer<ERModelRelationUrn>() {
+      public Object coerceInput(ERModelRelationUrn object) throws ClassCastException {
+        return object.toString();
+      }
+
+      public ERModelRelationUrn coerceOutput(Object object) throws TemplateOutputCastException {
+        try {
+          return ERModelRelationUrn.createFromString((String) object);
+        } catch (URISyntaxException e) {
+          throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e);
+        }
+      }
+    }, ERModelRelationUrn.class);
+  }
+}
diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java
deleted file mode 100644
index 6b7833ca99032..0000000000000
--- a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package com.linkedin.common.urn;
-
-import com.linkedin.data.template.Custom;
-import com.linkedin.data.template.DirectCoercer;
-import com.linkedin.data.template.TemplateOutputCastException;
-import java.net.URISyntaxException;
-
-
-public class JoinUrn extends Urn {
-  public static final String ENTITY_TYPE = "join";
-
-  private final String _joinId;
-
-  public JoinUrn(String joinId) {
-    super(ENTITY_TYPE, TupleKey.create(joinId));
-    this._joinId = joinId;
-  }
-
-  public String getJoinIdEntity() {
-    return _joinId;
-  }
-
-  public static JoinUrn createFromString(String rawUrn) throws URISyntaxException {
-    return createFromUrn(Urn.createFromString(rawUrn));
-  }
-
-  public static JoinUrn createFromUrn(Urn urn) throws URISyntaxException {
-    if (!"li".equals(urn.getNamespace())) {
-      throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'.");
-    } else if (!ENTITY_TYPE.equals(urn.getEntityType())) {
-      throw new URISyntaxException(urn.toString(), "Urn entity type should be 'join'.");
-    } else {
-      TupleKey key = urn.getEntityKey();
-      if (key.size() != 1) {
-        throw new URISyntaxException(urn.toString(), "Invalid number of keys.");
-      } else {
-        try {
-          return new JoinUrn((String) key.getAs(0, String.class));
-        } catch (Exception var3) {
-          throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + var3.getMessage());
-        }
-      }
-    }
-  }
-
-  public static JoinUrn deserialize(String rawUrn) throws URISyntaxException {
-    return createFromString(rawUrn);
-  }
-
-  static {
-    Custom.initializeCustomClass(JoinUrn.class);
-    Custom.registerCoercer(new DirectCoercer<JoinUrn>() {
-      public Object coerceInput(JoinUrn object) throws ClassCastException {
-        return object.toString();
-      }
-
-      public JoinUrn coerceOutput(Object object) throws TemplateOutputCastException {
-        try {
-          return JoinUrn.createFromString((String) object);
-        } catch (URISyntaxException e) {
-          throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e);
-        }
-      }
-    }, JoinUrn.class);
-  }
-}
diff --git a/li-utils/src/main/pegasus/com/linkedin/common/ERModelRelationUrn.pdl b/li-utils/src/main/pegasus/com/linkedin/common/ERModelRelationUrn.pdl
new file mode 100644
index 0000000000000..2adeca20d6c24
--- /dev/null
+++ b/li-utils/src/main/pegasus/com/linkedin/common/ERModelRelationUrn.pdl
@@ -0,0 +1,24 @@
+namespace com.linkedin.common
+
+/**
+ * Standardized ermodelrelation identifier.
+ */
+@java.class = "com.linkedin.common.urn.ERModelRelationUrn"
+@validate.`com.linkedin.common.validator.TypedUrnValidator` = {
+  "accessible" : true,
+  "owningTeam" : "urn:li:internalTeam:datahub",
+  "entityType" : "ermodelrelation",
+  "constructable" : true,
+  "namespace" : "li",
+  "name" : "ERModelRelation",
+  "doc" : "Standardized ERModelRelation identifier.",
+  "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ],
+  "fields" : [ {
+    "name" : "ermodelrelationId",
+    "doc" : "ERModelRelation native name e.g. <db>.<table>, /dir/subdir/<name>, or <name>",
+    "type" : "string",
+    "maxLength" : 284
+  }],
+  "maxLength" : 284
+}
+typeref ERModelRelationUrn = string
\ No newline at end of file
diff --git a/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl b/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl
deleted file mode 100644
index abfb5ba9a34cb..0000000000000
--- a/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl
+++ /dev/null
@@ -1,24 +0,0 @@
-namespace com.linkedin.common
-
-/**
- * Standardized join identifier.
- */
-@java.class = "com.linkedin.common.urn.JoinUrn"
-@validate.`com.linkedin.common.validator.TypedUrnValidator` = {
-  "accessible" : true,
-  "owningTeam" : "urn:li:internalTeam:datahub",
-  "entityType" : "join",
-  "constructable" : true,
-  "namespace" : "li",
-  "name" : "Join",
-  "doc" : "Standardized Join identifier.",
-  "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ],
-  "fields" : [ {
-    "name" : "joinId",
-    "doc" : "Join native name e.g. <db>.<table>, /dir/subdir/<name>, or <name>",
-    "type" : "string",
-    "maxLength" : 284
-  }],
-  "maxLength" : 284
-}
-typeref JoinUrn = string
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationFieldMapping.pdl b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationFieldMapping.pdl
new file mode 100644
index 0000000000000..5dfccb101bcf1
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationFieldMapping.pdl
@@ -0,0 +1,15 @@
+namespace com.linkedin.ermodelrelation
+
+import com.linkedin.dataset.SchemaFieldPath
+
+/**
+ * Field Mapping about a ermodelrelation between two datasets
+ */
+record ERModelRelationFieldMapping {
+
+  /**
+   * All fields from dataset A that are required for the ermodelrelation to dataset B
+   */
+  fieldMappings: array[FieldMap]
+
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl
similarity index 69%
rename from metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl
rename to metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl
index 95574b84889c6..0f516bd25f1bc 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl
@@ -1,19 +1,19 @@
-namespace com.linkedin.join
+namespace com.linkedin.ermodelrelation
 
 import com.linkedin.common.AuditStamp
 import com.linkedin.common.DatasetUrn
 import com.linkedin.common.CustomProperties
 
 /**
- * Properties associated with a Join
+ * Properties associated with a ERModelRelation
  */
 @Aspect = {
-  "name": "joinProperties"
+  "name": "ermodelrelationProperties"
 }
-record JoinProperties includes CustomProperties {
+record ERModelRelationProperties includes CustomProperties {
 
   /**
-   * Name of the Join
+   * Name of the ERModelRelation
    */
   @Searchable = {
     "fieldType": "TEXT_PARTIAL",
@@ -23,10 +23,10 @@ record JoinProperties includes CustomProperties {
   name: string
 
   /**
-  * First dataset in the join (no directionality)
+  * First dataset in the ermodelrelation (no directionality)
   */
    @Relationship = {
-     "name": "joinA",
+     "name": "ermodelrelationA",
      "entityTypes": [ "dataset" ]
    }
   @Searchable = {
@@ -37,10 +37,10 @@ record JoinProperties includes CustomProperties {
   datasetA: DatasetUrn
 
   /**
-  * Second dataset in the join (no directionality)
+  * Second dataset in the ermodelrelation (no directionality)
   */
    @Relationship = {
-     "name": "joinB",
+     "name": "ermodelrelationB",
      "entityTypes": [ "dataset" ]
    }
   @Searchable = {
@@ -51,9 +51,9 @@ record JoinProperties includes CustomProperties {
   datasetB: DatasetUrn
 
   /**
-   * JoinFieldMapping (in future we can make it an array)
+   * ERModelRelationFieldMapping (in future we can make it an array)
    */
-  joinFieldMapping: JoinFieldMapping
+  ermodelrelationFieldMapping: ERModelRelationFieldMapping
 
   /**
    * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)
diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/EditableERModelRelationProperties.pdl
similarity index 54%
rename from metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl
rename to metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/EditableERModelRelationProperties.pdl
index beba0d7d13c6f..ff10f8d8b2da0 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/EditableERModelRelationProperties.pdl
@@ -1,18 +1,18 @@
-namespace com.linkedin.join
+namespace com.linkedin.ermodelrelation
 
 import com.linkedin.common.ChangeAuditStamps
 
 
 /**
- * EditableJoinProperties stores editable changes made to join properties. This separates changes made from
+ * EditableERModelRelationProperties stores editable changes made to ermodelrelation properties. This separates changes made from
  * ingestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines
  */
 @Aspect = {
-  "name": "editableJoinProperties"
+  "name": "editableERModelRelationProperties"
 }
-record EditableJoinProperties includes ChangeAuditStamps {
+record EditableERModelRelationProperties includes ChangeAuditStamps {
   /**
-   * Documentation of the join
+   * Documentation of the ermodelrelation
    */
   @Searchable = {
     "fieldType": "TEXT",
@@ -21,7 +21,7 @@ record EditableJoinProperties includes ChangeAuditStamps {
   description: optional string
 
   /**
-   * Display name of the Join
+   * Display name of the ERModelRelation
    */
   @Searchable = {
     "fieldType": "TEXT_PARTIAL",
diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/FieldMap.pdl
similarity index 54%
rename from metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl
rename to metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/FieldMap.pdl
index 0c534b0d744ef..862a0c10459c9 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/FieldMap.pdl
@@ -1,4 +1,4 @@
-namespace com.linkedin.join
+namespace com.linkedin.ermodelrelation
 
 import com.linkedin.dataset.SchemaFieldPath
 
@@ -7,7 +7,7 @@ import com.linkedin.dataset.SchemaFieldPath
  */
 record FieldMap {
   /**
-   * All fields from dataset A that are required for the join, maps to bFields 1:1
+   * All fields from dataset A that are required for the ermodelrelation, maps to bFields 1:1
    */
   @Searchable = {
     "fieldType": "TEXT"
@@ -15,7 +15,7 @@ record FieldMap {
   afield: SchemaFieldPath
 
   /**
-   * All fields from dataset B that are required for the join, maps to aFields 1:1
+   * All fields from dataset B that are required for the ermodelrelation, maps to aFields 1:1
    */
   @Searchable = {
     "fieldType": "TEXT"
diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl
deleted file mode 100644
index c3d34fcbe7c40..0000000000000
--- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl
+++ /dev/null
@@ -1,15 +0,0 @@
-namespace com.linkedin.join
-
-import com.linkedin.dataset.SchemaFieldPath
-
-/**
- * Field Mapping about a join between two datasets
- */
-record JoinFieldMapping {
-
-  /**
-   * All fields from dataset A that are required for the join to dataset B
-   */
-  fieldMappings: array[FieldMap]
-
-}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/ERModelRelationKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/ERModelRelationKey.pdl
new file mode 100644
index 0000000000000..9f6d6201e0dcb
--- /dev/null
+++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/ERModelRelationKey.pdl
@@ -0,0 +1,14 @@
+namespace com.linkedin.metadata.key
+
+/**
+ * Key for a ERModelRelation
+ */
+@Aspect = {
+  "name": "ermodelrelationKey"
+}
+record ERModelRelationKey {
+  /*
+  * Unique guid for ERModelRelation
+  */
+  ermodelrelationId: string
+}
\ No newline at end of file
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl
deleted file mode 100644
index e83e7c514b219..0000000000000
--- a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace com.linkedin.metadata.key
-
-/**
- * Key for a Join
- */
-@Aspect = {
-  "name": "joinKey"
-}
-record JoinKey {
-  /*
-  * Unique guid for Join
-  */
-  joinId: string
-}
\ No newline at end of file
diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml
index 3f4b339fc201c..e01e5b2061259 100644
--- a/metadata-models/src/main/resources/entity-registry.yml
+++ b/metadata-models/src/main/resources/entity-registry.yml
@@ -434,12 +434,12 @@ entities:
     keyAspect: dataHubViewKey
     aspects:
       - dataHubViewInfo
-  - name: join
-    doc: Join Dataset Fields
-    keyAspect: joinKey
+  - name: ermodelrelation
+    doc: ER Model Relation of  Dataset Fields
+    keyAspect: ermodelrelationKey
     aspects:
-      - joinProperties
-      - editableJoinProperties
+      - erModelRelationProperties
+      - editableERModelRelationProperties
       - institutionalMemory
       - ownership
       - status
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
index 9dda29a83f7e5..d0a5228425378 100644
--- a/metadata-service/configuration/src/main/resources/application.yml
+++ b/metadata-service/configuration/src/main/resources/application.yml
@@ -320,7 +320,7 @@ featureFlags:
   preProcessHooks:
     uiEnabled: ${PRE_PROCESS_HOOKS_UI_ENABLED:true} # Circumvents Kafka for processing index updates for UI changes sourced from GraphQL to avoid processing delays
   showAcrylInfo: ${SHOW_ACRYL_INFO:false} # Show different CTAs within DataHub around moving to Managed DataHub. Set to true for the demo site.
-  joinFeatureEnabled: ${JOIN_FEATURE_ENABLED:false} # Enable Join Tables Feature and show within Dataset view as Relations
+  ermodelrelationFeatureEnabled: ${ERMODELRELATION_FEATURE_ENABLED:false} # Enable Join Tables Feature and show within Dataset view as Relations
   nestedDomainsEnabled: ${NESTED_DOMAINS_ENABLED:true} # Enables the nested Domains feature that allows users to have sub-Domains. If this is off, Domains appear "flat" again
 
 entityChangeEvents:
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/join/JoinServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java
similarity index 72%
rename from metadata-service/factories/src/main/java/com/linkedin/gms/factory/join/JoinServiceFactory.java
rename to metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java
index 226c49b5ca101..158cee1122bbd 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/join/JoinServiceFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java
@@ -1,9 +1,9 @@
-package com.linkedin.gms.factory.join;
+package com.linkedin.gms.factory.ermodelrelation;
 
 import com.datahub.authentication.Authentication;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
 import com.linkedin.metadata.client.JavaEntityClient;
-import com.linkedin.metadata.service.JoinService;
+import com.linkedin.metadata.service.ERModelRelationService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Bean;
@@ -16,7 +16,7 @@
 
 @Configuration
 @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class)
-public class JoinServiceFactory {
+public class ERModelRelationServiceFactory {
   @Autowired
   @Qualifier("javaEntityClient")
   private JavaEntityClient _javaEntityClient;
@@ -25,10 +25,10 @@ public class JoinServiceFactory {
   @Qualifier("systemAuthentication")
   private Authentication _authentication;
 
-  @Bean(name = "joinService")
+  @Bean(name = "ermodelrelationService")
   @Scope("singleton")
   @Nonnull
-  protected JoinService getInstance() throws Exception {
-    return new JoinService(_javaEntityClient, _authentication);
+  protected ERModelRelationService getInstance() throws Exception {
+    return new ERModelRelationService(_javaEntityClient, _authentication);
   }
 }
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java
index 120653eddb2e6..c951598da8e82 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java
@@ -31,7 +31,7 @@
 import com.linkedin.metadata.service.DataProductService;
 import com.linkedin.metadata.service.OwnershipTypeService;
 import com.linkedin.metadata.service.QueryService;
-import com.linkedin.metadata.service.JoinService;
+import com.linkedin.metadata.service.ERModelRelationService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
 import com.linkedin.metadata.service.LineageService;
@@ -164,8 +164,8 @@ public class GraphQLEngineFactory {
   private QueryService _queryService;
 
   @Autowired
-  @Qualifier("joinService")
-  private JoinService _joinService;
+  @Qualifier("ermodelrelationService")
+  private ERModelRelationService _ermodelrelationService;
 
   @Autowired
   @Qualifier("dataProductService")
@@ -214,7 +214,7 @@ protected GraphQLEngine getInstance() {
     args.setSettingsService(_settingsService);
     args.setLineageService(_lineageService);
     args.setQueryService(_queryService);
-    args.setJoinService(_joinService);
+    args.setErmodelrelationService(_ermodelrelationService);
     args.setFeatureFlags(_configProvider.getFeatureFlags());
     args.setDataProductService(_dataProductService);
     return new GmsGraphQLEngine(
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/JoinService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/ERModelRelationService.java
similarity index 64%
rename from metadata-service/services/src/main/java/com/linkedin/metadata/service/JoinService.java
rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/ERModelRelationService.java
index 0097f11827c8d..2365dbfa659ca 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/JoinService.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/ERModelRelationService.java
@@ -25,15 +25,15 @@
  *
  */
 @Slf4j
-public class JoinService extends BaseService {
+public class ERModelRelationService extends BaseService {
 
-  public JoinService(@Nonnull EntityClient entityClient, @Nonnull Authentication systemAuthentication) {
+  public ERModelRelationService(@Nonnull EntityClient entityClient, @Nonnull Authentication systemAuthentication) {
     super(entityClient, systemAuthentication);
   }
   static final Set<String> ASPECTS_TO_RESOLVE = ImmutableSet.of(
-          JOIN_KEY_ASPECT_NAME,
-          JOIN_PROPERTIES_ASPECT_NAME,
-          EDITABLE_JOIN_PROPERTIES_ASPECT_NAME,
+          ERMODELRELATION_KEY_ASPECT_NAME,
+          ERMODELRELATION_PROPERTIES_ASPECT_NAME,
+          EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME,
           INSTITUTIONAL_MEMORY_ASPECT_NAME,
           OWNERSHIP_ASPECT_NAME,
           STATUS_ASPECT_NAME,
@@ -41,27 +41,27 @@ public JoinService(@Nonnull EntityClient entityClient, @Nonnull Authentication s
           GLOSSARY_TERMS_ASPECT_NAME
   );
   /**
-   * Returns an instance of {@link EntityResponse} for the specified Join urn,
+   * Returns an instance of {@link EntityResponse} for the specified ERModelRelation urn,
    * or null if one cannot be found.
    *
-   * @param joinUrn the urn of the Query
+   * @param ermodelrelationUrn the urn of the Query
    * @param authentication the authentication to use
    *
-   * @return an instance of {@link EntityResponse} for the Join, null if it does not exist.
+   * @return an instance of {@link EntityResponse} for the ERModelRelation, null if it does not exist.
    */
   @Nullable
-  public EntityResponse getJoinResponse(@Nonnull final Urn joinUrn, @Nonnull final Authentication authentication) {
-    Objects.requireNonNull(joinUrn, "joinUrn must not be null");
+  public EntityResponse getERModelRelationResponse(@Nonnull final Urn ermodelrelationUrn, @Nonnull final Authentication authentication) {
+    Objects.requireNonNull(ermodelrelationUrn, "ermodelrelationUrn must not be null");
     Objects.requireNonNull(authentication, "authentication must not be null");
     try {
       return this.entityClient.getV2(
-          Constants.JOIN_ENTITY_NAME,
-          joinUrn,
+          Constants.ERMODELRELATION_ENTITY_NAME,
+          ermodelrelationUrn,
           ASPECTS_TO_RESOLVE,
           authentication
       );
     } catch (Exception e) {
-      throw new RuntimeException(String.format("Failed to retrieve Query with urn %s", joinUrn), e);
+      throw new RuntimeException(String.format("Failed to retrieve Query with urn %s", ermodelrelationUrn), e);
     }
   }
-}
\ No newline at end of file
+}
diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json
index 505bd540ddf13..dfd0a762106f3 100644
--- a/metadata-service/war/src/main/resources/boot/policies.json
+++ b/metadata-service/war/src/main/resources/boot/policies.json
@@ -226,7 +226,7 @@
         "GET_TIMELINE_PRIVILEGE",
         "PRODUCE_PLATFORM_EVENT_PRIVILEGE",
         "MANAGE_DATA_PRODUCTS",
-        "CREATE_JOIN_PRIVILEGE",
+        "CREATE_ERMODELRELATION_PRIVILEGE",
         "DELETE_ENTITY"
       ],
       "displayName":"Admins - Metadata Policy",
diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java
index a57614487cff2..cdd3732644c29 100644
--- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java
+++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java
@@ -222,10 +222,10 @@ public class PoliciesConfig {
       "Edit Embedded Content",
       "The ability to edit the embedded content for an entity.");
 
-  public static final Privilege CREATE_JOIN_PRIVILEGE = Privilege.of(
-      "CREATE_ENTITY_JOIN",
-      "Create join",
-      "The ability to add join on a dataset.");
+  public static final Privilege CREATE_ERMODELRELATION_PRIVILEGE = Privilege.of(
+      "CREATE_ENTITY_ERMODELRELATION",
+      "Create ermodelrelation",
+      "The ability to add ermodelrelation on a dataset.");
 
   public static final List<Privilege> COMMON_ENTITY_PRIVILEGES = ImmutableList.of(
       VIEW_ENTITY_PAGE_PRIVILEGE,
@@ -388,7 +388,7 @@ public class PoliciesConfig {
               EDIT_LINEAGE_PRIVILEGE,
               EDIT_ENTITY_EMBED_PRIVILEGE,
               EDIT_QUERIES_PRIVILEGE,
-              CREATE_JOIN_PRIVILEGE))
+              CREATE_ERMODELRELATION_PRIVILEGE))
           .flatMap(Collection::stream)
           .collect(Collectors.toList())
   );
@@ -541,11 +541,11 @@ public class PoliciesConfig {
           EDIT_ENTITY_PRIVILEGE)
   );
 
-  // Join Privileges
-  public static final ResourcePrivileges JOIN_PRIVILEGES = ResourcePrivileges.of(
-          "join",
-          "Join",
-          "update privileges for joins",
+  // ERModelRelation Privileges
+  public static final ResourcePrivileges ERMODELRELATION_PRIVILEGES = ResourcePrivileges.of(
+          "ermodelrelation",
+          "ERModelRelation",
+          "update privileges for ermodelrelations",
           COMMON_ENTITY_PRIVILEGES
   );
   public static final List<ResourcePrivileges> ENTITY_RESOURCE_PRIVILEGES = ImmutableList.of(
@@ -563,7 +563,7 @@ public class PoliciesConfig {
       CORP_USER_PRIVILEGES,
       NOTEBOOK_PRIVILEGES,
       DATA_PRODUCT_PRIVILEGES,
-      JOIN_PRIVILEGES
+      ERMODELRELATION_PRIVILEGES
   );
 
   // Merge all entity specific resource privileges to create a superset of all resource privileges

From d12299603fbefecc3929051ad07ec532fdc7d14d Mon Sep 17 00:00:00 2001
From: Raj Tekal <varadaraj_tekal@optum.com>
Date: Fri, 19 Jan 2024 14:35:20 -0500
Subject: [PATCH 258/259] Datahub comes up.  Needs more UI testing

---
 .../com/linkedin/datahub/graphql/GmsGraphQLEngine.java     | 7 ++++---
 .../com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java | 2 +-
 .../ermodelrelation/CreateERModelRelationResolver.java     | 4 ++--
 .../linkedin/ermodelrelation/ERModelRelationProperties.pdl | 2 +-
 metadata-models/src/main/resources/entity-registry.yml     | 2 +-
 .../ermodelrelation/ERModelRelationServiceFactory.java     | 2 +-
 .../linkedin/gms/factory/graphql/GraphQLEngineFactory.java | 6 +++---
 7 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index 2953538078998..6097161b5482b 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -397,6 +397,7 @@ public class GmsGraphQLEngine {
     private final LineageService lineageService;
     private final QueryService queryService;
     private final DataProductService dataProductService;
+    private final ERModelRelationService eRModelRelationService;
 
     private final FeatureFlags featureFlags;
 
@@ -445,7 +446,7 @@ public class GmsGraphQLEngine {
     private final QueryType queryType;
     private final DataProductType dataProductType;
     private final OwnershipType ownershipType;
-    private final ERModelRelationService ermodelrelationService;
+
 
     /**
      * A list of GraphQL Plugins that extend the core engine
@@ -511,7 +512,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
         this.settingsService = args.settingsService;
         this.lineageService = args.lineageService;
         this.queryService = args.queryService;
-        this.ermodelrelationService = args.ermodelrelationService;
+        this.eRModelRelationService = args.eRModelRelationService;
         this.dataProductService = args.dataProductService;
 
         this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration);
@@ -937,7 +938,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType))
             .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType))
             .dataFetcher("updateERModelRelation", new UpdateERModelRelationResolver(this.entityClient))
-            .dataFetcher("createERModelRelation", new CreateERModelRelationResolver(this.entityClient, this.ermodelrelationService))
+            .dataFetcher("createERModelRelation", new CreateERModelRelationResolver(this.entityClient, this.eRModelRelationService))
             .dataFetcher("addTag", new AddTagResolver(entityService))
             .dataFetcher("addTags", new AddTagsResolver(entityService))
             .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService))
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java
index c5e34151ae2d8..b0f5712f30c9a 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java
@@ -74,6 +74,6 @@ public class GmsGraphQLEngineArgs {
     QueryService queryService;
     FeatureFlags featureFlags;
     DataProductService dataProductService;
-    ERModelRelationService ermodelrelationService;
+    ERModelRelationService eRModelRelationService;
     //any fork specific args should go below this line
 }
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java
index 021bdd0603110..c6070957cdbc1 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelation/CreateERModelRelationResolver.java
@@ -32,7 +32,7 @@
 public class CreateERModelRelationResolver implements DataFetcher<CompletableFuture<ERModelRelation>> {
 
     private final EntityClient _entityClient;
-    private final ERModelRelationService _ermodelrelationService;
+    private final ERModelRelationService _eRModelRelationService;
 
     @Override
     public CompletableFuture<ERModelRelation> get(DataFetchingEnvironment environment) throws Exception {
@@ -79,7 +79,7 @@ public CompletableFuture<ERModelRelation> get(DataFetchingEnvironment environmen
                 } catch (RemoteInvocationException e) {
                     throw new RuntimeException("Failed to create ermodelrelation entity", e);
                 }
-                return ERModelRelationMapper.map(_ermodelrelationService.getERModelRelationResponse(Urn.createFromString(inputUrn.toString()), authentication));
+                return ERModelRelationMapper.map(_eRModelRelationService.getERModelRelationResponse(Urn.createFromString(inputUrn.toString()), authentication));
             } catch (Exception e) {
                 log.error("Failed to create ERModelRelation to resource with input {}, {}", input, e.getMessage());
                 throw new RuntimeException(String.format("Failed to create ermodelrelation to resource with input %s", input), e);
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl
index 0f516bd25f1bc..4cda44584d6dd 100644
--- a/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl
+++ b/metadata-models/src/main/pegasus/com/linkedin/ermodelrelation/ERModelRelationProperties.pdl
@@ -8,7 +8,7 @@ import com.linkedin.common.CustomProperties
  * Properties associated with a ERModelRelation
  */
 @Aspect = {
-  "name": "ermodelrelationProperties"
+  "name": "eRModelRelationProperties"
 }
 record ERModelRelationProperties includes CustomProperties {
 
diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml
index e01e5b2061259..a27bb8005bb3b 100644
--- a/metadata-models/src/main/resources/entity-registry.yml
+++ b/metadata-models/src/main/resources/entity-registry.yml
@@ -438,7 +438,7 @@ entities:
     doc: ER Model Relation of  Dataset Fields
     keyAspect: ermodelrelationKey
     aspects:
-      - erModelRelationProperties
+      - eRModelRelationProperties
       - editableERModelRelationProperties
       - institutionalMemory
       - ownership
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java
index 158cee1122bbd..93acb1fe411f6 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationServiceFactory.java
@@ -25,7 +25,7 @@ public class ERModelRelationServiceFactory {
   @Qualifier("systemAuthentication")
   private Authentication _authentication;
 
-  @Bean(name = "ermodelrelationService")
+  @Bean(name = "eRModelRelationService")
   @Scope("singleton")
   @Nonnull
   protected ERModelRelationService getInstance() throws Exception {
diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java
index c951598da8e82..01d166d49b46b 100644
--- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java
+++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java
@@ -164,8 +164,8 @@ public class GraphQLEngineFactory {
   private QueryService _queryService;
 
   @Autowired
-  @Qualifier("ermodelrelationService")
-  private ERModelRelationService _ermodelrelationService;
+  @Qualifier("eRModelRelationService")
+  private ERModelRelationService _eRModelRelationService;
 
   @Autowired
   @Qualifier("dataProductService")
@@ -214,7 +214,7 @@ protected GraphQLEngine getInstance() {
     args.setSettingsService(_settingsService);
     args.setLineageService(_lineageService);
     args.setQueryService(_queryService);
-    args.setErmodelrelationService(_ermodelrelationService);
+    args.setERModelRelationService(_eRModelRelationService);
     args.setFeatureFlags(_configProvider.getFeatureFlags());
     args.setDataProductService(_dataProductService);
     return new GmsGraphQLEngine(

From 8e727c385f85c39caf42fdc00284cf0b0cdeaa75 Mon Sep 17 00:00:00 2001
From: Raj Tekal <varadaraj_tekal@optum.com>
Date: Mon, 22 Jan 2024 13:58:10 -0500
Subject: [PATCH 259/259] Typo in entity name

---
 li-utils/src/main/java/com/linkedin/metadata/Constants.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java
index 65c4186ff97e9..cb0303ea2e3c9 100644
--- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java
+++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java
@@ -241,7 +241,7 @@ public class Constants {
 
   // ERModelRelation
   public static final String ERMODELRELATION_KEY_ASPECT_NAME = "ermodelrelationKey";
-  public static final String ERMODELRELATION_PROPERTIES_ASPECT_NAME = "ermodelrelationProperties";
+  public static final String ERMODELRELATION_PROPERTIES_ASPECT_NAME = "eRModelRelationProperties";
   public static final String EDITABLE_ERMODELRELATION_PROPERTIES_ASPECT_NAME = "editableERModelRelationProperties";
 
   // Assertion