chore(ingest): remove legacy memory_leak_detector (#9158)

datahub-project · Nov 2, 2023 · ec97250 · ec97250
1 parent f42cb95
commit ec97250
Show file tree

Hide file tree

Showing 6 changed files with 10 additions and 139 deletions.
diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py
@@ -27,7 +27,6 @@
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
-from datahub.utilities import memory_leak_detector
 
 logger = logging.getLogger(__name__)
 
@@ -98,7 +97,6 @@ def ingest() -> None:
 @click.option(
     "--no-spinner", type=bool, is_flag=True, default=False, help="Turn off spinner"
 )
-@click.pass_context
 @telemetry.with_telemetry(
     capture_kwargs=[
         "dry_run",
@@ -109,9 +107,7 @@ def ingest() -> None:
         "no_spinner",
     ]
 )
-@memory_leak_detector.with_leak_detection
 def run(
-    ctx: click.Context,
     config: str,
     dry_run: bool,
     preview: bool,

diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py
@@ -70,21 +70,10 @@
     version=datahub_package.nice_version_name(),
     prog_name=datahub_package.__package_name__,
 )
-@click.option(
-    "-dl",
-    "--detect-memory-leaks",
-    type=bool,
-    is_flag=True,
-    default=False,
-    help="Run memory leak detection.",
-)
-@click.pass_context
 def datahub(
-    ctx: click.Context,
     debug: bool,
     log_file: Optional[str],
     debug_vars: bool,
-    detect_memory_leaks: bool,
 ) -> None:
     if debug_vars:
         # debug_vars implies debug. This option isn't actually used here, but instead
@@ -109,10 +98,6 @@ def datahub(
     _logging_configured = configure_logging(debug=debug, log_file=log_file)
     _logging_configured.__enter__()
 
-    # Setup the context for the memory_leak_detector decorator.
-    ctx.ensure_object(dict)
-    ctx.obj["detect_memory_leaks"] = detect_memory_leaks
-
 
 @datahub.command()
 @telemetry.with_telemetry()

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
@@ -121,7 +121,10 @@ class LookerCommonConfig(DatasetSourceConfigMixin):
         "discoverable. When disabled, adds this information to the description of the column.",
     )
     platform_name: str = Field(
-        "looker", description="Default platform name. Don't change."
+        # TODO: This shouldn't be part of the config.
+        "looker",
+        description="Default platform name.",
+        hidden_from_docs=True,
     )
     extract_column_level_lineage: bool = Field(
         True,
@@ -213,7 +216,6 @@ def external_url_defaults_to_api_config_base_url(
     def stateful_ingestion_should_be_enabled(
         cls, v: Optional[bool], *, values: Dict[str, Any], **kwargs: Dict[str, Any]
     ) -> Optional[bool]:
-
         stateful_ingestion: StatefulStaleMetadataRemovalConfig = cast(
             StatefulStaleMetadataRemovalConfig, values.get("stateful_ingestion")
         )

diff --git a/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py b/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py
diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py
@@ -565,5 +565,4 @@ def default_query_results(  # noqa: C901
                 "DOMAIN": "DATABASE",
             },
         ]
-    # Unreachable code
-    raise Exception(f"Unknown query {query}")
+    raise ValueError(f"Unexpected query: {query}")
diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py
@@ -368,8 +368,7 @@ def default_query_results(query):
             return [('{"roles":"","value":""}',)]
         elif query == "select current_warehouse()":
             return [("TEST_WAREHOUSE")]
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     connection_mock = MagicMock()
     cursor_mock = MagicMock()
@@ -397,8 +396,7 @@ def query_results(query):
             ]
         elif query == 'show grants to role "PUBLIC"':
             return []
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     config = {
         "username": "user",
@@ -441,8 +439,7 @@ def query_results(query):
             return [("", "USAGE", "DATABASE", "DB1")]
         elif query == 'show grants to role "PUBLIC"':
             return []
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     setup_mock_connect(mock_connect, query_results)
 
@@ -485,8 +482,7 @@ def query_results(query):
             ]
         elif query == 'show grants to role "PUBLIC"':
             return []
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     setup_mock_connect(mock_connect, query_results)
 
@@ -536,8 +532,7 @@ def query_results(query):
                 ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY"],
                 ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES"],
             ]
-        # Unreachable code
-        raise Exception()
+        raise ValueError(f"Unexpected query: {query}")
 
     setup_mock_connect(mock_connect, query_results)