fix(ingestion/looker): Add sqlglot dependency and remove unused sqlpa…

…rser (#10874)

metadata-ingestion/setup.py

@@ -163,8 +163,7 @@
     # LookML files with spaces between an item and the following comma.
     # See https://github.com/joshtemple/lkml/issues/73.
     "lkml>=1.3.4",
-    "sql-metadata==2.2.2",
-    *sqllineage_lib,
+    *sqlglot_lib,
     "GitPython>2",
     "python-liquid",
 }
@@ -372,7 +371,7 @@
     "kafka-connect": sql_common | {"requests", "JPype1"},
     "ldap": {"python-ldap>=2.4"},
     "looker": looker_common,
-    "lookml": looker_common | sqlglot_lib,
+    "lookml": looker_common,
     "metabase": {"requests"} | sqlglot_lib,
     "mlflow": {
         "mlflow-skinny>=2.3.0",

metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py

@@ -4,7 +4,7 @@
 from collections import OrderedDict
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import Dict, Iterable, List, Optional, Set, Tuple, Type
+from typing import Dict, Iterable, List, Optional, Set, Tuple
 
 import lkml
 import lkml.simple
@@ -22,7 +22,6 @@
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.registry import import_path
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceCapability
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import (
@@ -96,7 +95,6 @@
     SubTypesClass,
 )
 from datahub.sql_parsing.sqlglot_lineage import ColumnRef
-from datahub.utilities.sql_parser import SQLParser
 
 logger = logging.getLogger(__name__)
 
@@ -111,15 +109,6 @@ class LookerView:
     raw_file_content: str
     view_details: Optional[ViewProperties] = None
 
-    @classmethod
-    def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
-        assert "." in sql_parser_path, "sql_parser-path must contain a ."
-        parser_cls = import_path(sql_parser_path)
-
-        if not issubclass(parser_cls, SQLParser):
-            raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")
-        return parser_cls
-
     @classmethod
     def determine_view_file_path(
         cls, base_folder_path: str, absolute_file_path: str