Merge branch 'main' into fix-19333

ingestion/src/metadata/ingestion/source/dashboard/redash/client.py

@@ -13,6 +13,7 @@
 """
 
 from metadata.ingestion.ometa.client import REST, ClientConfig
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -28,8 +29,8 @@ class RedashApiClient:
     def __init__(self, config):
         self.config = config
         client_config = ClientConfig(
-            base_url=str(config.hostPort),
-            api_version="",
+            base_url=clean_uri(config.hostPort),
+            api_version="api",
             access_token=config.apiKey.get_secret_value(),
             auth_header="Authorization",
             auth_token_mode="Key",
@@ -41,16 +42,11 @@ def dashboards(self, page=1, page_size=25):
         """GET api/dashboards"""
 
         params_data = {"page": page, "page_size": page_size}
-        return self.client.get(path="api/dashboards", data=params_data)
+        return self.client.get(path="/dashboards", data=params_data)
 
-    def get_dashboard(self, slug):
-        """GET api/dashboards/<slug>"""
-
-        # The API changed from redash v9 onwards
-        # legacy=true allows us to get the results in the old way
-        return self.client.get(
-            f"api/dashboards/{slug}?legacy=true",
-        )
+    def get_dashboard(self, dashboard_id: int):
+        """GET api/dashboards/<id>"""
+        return self.client.get(f"/dashboards/{dashboard_id}")
 
     def paginate(self, resource, page=1, page_size=25, **kwargs):
         """Load all items of a paginated resource"""

ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py

@@ -115,7 +115,7 @@ def get_dashboard_name(self, dashboard: dict) -> str:
         return dashboard["name"]
 
     def get_dashboard_details(self, dashboard: dict) -> dict:
-        return self.client.get_dashboard(dashboard["slug"])
+        return self.client.get_dashboard(dashboard["id"])
 
     def get_owner_ref(self, dashboard_details) -> Optional[EntityReferenceList]:
         """
@@ -160,9 +160,9 @@ def yield_dashboard(
             dashboard_request = CreateDashboardRequest(
                 name=EntityName(str(dashboard_details["id"])),
                 displayName=dashboard_details.get("name"),
-                description=Markdown(dashboard_description)
-                if dashboard_description
-                else None,
+                description=(
+                    Markdown(dashboard_description) if dashboard_description else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -275,19 +275,23 @@ def yield_dashboard_chart(
                 yield Either(
                     right=CreateChartRequest(
                         name=EntityName(str(widgets["id"])),
-                        displayName=chart_display_name
-                        if visualization and visualization["query"]
-                        else "",
+                        displayName=(
+                            chart_display_name
+                            if visualization and visualization["query"]
+                            else ""
+                        ),
                         chartType=get_standard_chart_type(
                             visualization["type"] if visualization else ""
                         ),
                         service=FullyQualifiedEntityName(
                             self.context.get().dashboard_service
                         ),
                         sourceUrl=SourceUrl(self.get_dashboard_url(dashboard_details)),
-                        description=Markdown(visualization["description"])
-                        if visualization
-                        else None,
+                        description=(
+                            Markdown(visualization["description"])
+                            if visualization
+                            else None
+                        ),
                     )
                 )
             except Exception as exc:

ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py

@@ -227,6 +227,17 @@ def _clearn_column_datatype(self, datatype: str) -> str:
         """clean datatype of column fetched from superset"""
         return datatype.replace("()", "")
 
+    def parse_array_data_type(self, col_parse: dict) -> Optional[str]:
+        """
+        Set arrayDataType to UNKNOWN for Snowflake table array columns
+        to prevent validation error requiring non-null arrayDataType
+        """
+        if col_parse["dataType"] == "ARRAY" and not col_parse.get("arrayDataType"):
+            return DataType.UNKNOWN
+        if col_parse.get("arrayDataType"):
+            return DataType(col_parse["arrayDataType"])
+        return None
+
     def get_column_info(
         self, data_source: List[Union[DataSourceResult, FetchColumn]]
     ) -> Optional[List[Column]]:
@@ -247,9 +258,7 @@ def get_column_info(
                     parsed_fields = Column(
                         dataTypeDisplay=field.type,
                         dataType=col_parse["dataType"],
-                        arrayDataType=DataType(col_parse["arrayDataType"])
-                        if col_parse.get("arrayDataType")
-                        else None,
+                        arrayDataType=self.parse_array_data_type(col_parse),
                         children=list(col_parse["children"])
                         if col_parse.get("children")
                         else None,

ingestion/tests/unit/test_column_type_parser.py

@@ -17,6 +17,7 @@
 from unittest import TestCase
 
 from metadata.generated.schema.entity.data.table import DataType
+from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.utils.datalake.datalake_utils import GenericDataFrameColumnParser
 
@@ -132,3 +133,16 @@ def test_check_datalake_type():
         assert assert_col_type_dict.get(
             column_name
         ) == GenericDataFrameColumnParser.fetch_col_types(df, column_name)
+
+
+def test_superset_parse_array_data_type():
+    """Test the parse_array_data_type method with different input scenarios"""
+    col_parse = {"dataType": "ARRAY", "arrayDataType": "STRING"}
+    result = SupersetSourceMixin.parse_array_data_type(None, col_parse)
+    assert result == DataType.STRING
+    col_parse = {"dataType": "ARRAY", "arrayDataType": None}
+    result = SupersetSourceMixin.parse_array_data_type(None, col_parse)
+    assert result == DataType.UNKNOWN
+    col_parse = {"dataType": "STRING", "arrayDataType": None}
+    result = SupersetSourceMixin.parse_array_data_type(None, col_parse)
+    assert result == None

openmetadata-docs/content/v1.6.x/connectors/database/doris/yaml.md

@@ -69,8 +69,6 @@ This is a sample config for Doris:
 
 {% codeInfo srNumber=2 %}
 
-{% codeInfo srNumber=2 %}
-
 **password**: Password to connect to Doris.
 
 {% /codeInfo %}

openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/doris/yaml.md

@@ -69,8 +69,6 @@ This is a sample config for Doris:
 
 {% codeInfo srNumber=2 %}
 
-{% codeInfo srNumber=2 %}
-
 **password**: Password to connect to Doris.
 
 {% /codeInfo %}