feat(ingest/redshift): redshift lineage v2 (#9904)

docs/how/updating-datahub.md

@@ -35,6 +35,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
   `datahub delete --platform databricks --soft` and then reingesting with latest cli version.
 
 - #9601 - The Unity Catalog(UC) ingestion source config `include_hive_metastore` is now enabled by default. This requires config `warehouse_id` to be set. You can disable `include_hive_metastore` by setting it to `False` to avoid ingesting legacy hive metastore catalog in Databricks.
+- #9904 - The default Redshift `table_lineage_mode` is now MIXED, instead of `STL_SCAN_BASED`. Improved lineage generation is also available by enabling `use_lineaege_v2`. This v2 implementation will become the default in a future release.
 
 ### Potential Downtime
 

metadata-ingestion/src/datahub/configuration/common.py

@@ -309,8 +309,3 @@ class LineageConfig(ConfigModel):
         default=False,
         description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
     )
-
-    sql_parser_use_external_process: bool = Field(
-        default=False,
-        description="When enabled, sql parser will run in isolated in a separate process. This can affect processing time but can protect from sql parser's mem leak.",
-    )

metadata-ingestion/src/datahub/ingestion/source/redshift/config.py

@@ -84,7 +84,7 @@ class RedshiftConfig(
     scheme: str = Field(
         default="redshift+redshift_connector",
         description="",
-        hidden_from_schema=True,
+        hidden_from_docs=True,
     )
 
     _database_alias_removed = pydantic_removed_field("database_alias")
@@ -94,6 +94,11 @@ class RedshiftConfig(
         description="The default schema to use if the sql parser fails to parse the schema with `sql_based` lineage collector",
     )
 
+    use_lineage_v2: bool = Field(
+        default=False,
+        description="Whether to use the new SQL-based lineage and usage collector.",
+    )
+
     include_table_lineage: bool = Field(
         default=True, description="Whether table lineage should be ingested."
     )
@@ -113,11 +118,11 @@ class RedshiftConfig(
     )
 
     include_table_rename_lineage: bool = Field(
-        default=False,
+        default=True,
         description="Whether we should follow `alter table ... rename to` statements when computing lineage. ",
     )
-    table_lineage_mode: Optional[LineageMode] = Field(
-        default=LineageMode.STL_SCAN_BASED,
+    table_lineage_mode: LineageMode = Field(
+        default=LineageMode.MIXED,
         description="Which table lineage collector mode to use. Available modes are: [stl_scan_based, sql_based, mixed]",
     )
     extra_client_options: Dict[str, Any] = {}
@@ -138,7 +143,7 @@ class RedshiftConfig(
     )
 
     resolve_temp_table_in_lineage: bool = Field(
-        default=False,
+        default=True,
         description="Whether to resolve temp table appear in lineage to upstream permanent tables.",
     )
 

metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py

@@ -766,7 +766,7 @@ def get_lineage(
         table: Union[RedshiftTable, RedshiftView],
         dataset_urn: str,
         schema: RedshiftSchema,
-    ) -> Optional[Tuple[UpstreamLineageClass, Dict[str, str]]]:
+    ) -> Optional[UpstreamLineageClass]:
         upstream_lineage: List[UpstreamClass] = []
 
         cll_lineage: List[FineGrainedLineage] = []
@@ -811,11 +811,9 @@ def get_lineage(
         else:
             return None
 
-        return (
-            UpstreamLineage(
-                upstreams=upstream_lineage, fineGrainedLineages=cll_lineage or None
-            ),
-            {},
+        return UpstreamLineage(
+            upstreams=upstream_lineage,
+            fineGrainedLineages=cll_lineage or None,
         )
 
     def report_status(self, step: str, status: bool) -> None: