diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py index 30cc9e9cd6c8..40a0e925aade 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py @@ -13,6 +13,7 @@ """ from metadata.ingestion.ometa.client import REST, ClientConfig +from metadata.utils.helpers import clean_uri from metadata.utils.logger import utils_logger logger = utils_logger() @@ -28,8 +29,8 @@ class RedashApiClient: def __init__(self, config): self.config = config client_config = ClientConfig( - base_url=str(config.hostPort), - api_version="", + base_url=clean_uri(config.hostPort), + api_version="api", access_token=config.apiKey.get_secret_value(), auth_header="Authorization", auth_token_mode="Key", @@ -41,16 +42,11 @@ def dashboards(self, page=1, page_size=25): """GET api/dashboards""" params_data = {"page": page, "page_size": page_size} - return self.client.get(path="api/dashboards", data=params_data) + return self.client.get(path="/dashboards", data=params_data) - def get_dashboard(self, slug): - """GET api/dashboards/""" - - # The API changed from redash v9 onwards - # legacy=true allows us to get the results in the old way - return self.client.get( - f"api/dashboards/{slug}?legacy=true", - ) + def get_dashboard(self, dashboard_id: int): + """GET api/dashboards/""" + return self.client.get(f"/dashboards/{dashboard_id}") def paginate(self, resource, page=1, page_size=25, **kwargs): """Load all items of a paginated resource""" diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py index 94ae4a0cb7dc..1067e1afcc76 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py @@ -115,7 +115,7 @@ def get_dashboard_name(self, dashboard: dict) -> str: return dashboard["name"] def get_dashboard_details(self, dashboard: dict) -> dict: - return self.client.get_dashboard(dashboard["slug"]) + return self.client.get_dashboard(dashboard["id"]) def get_owner_ref(self, dashboard_details) -> Optional[EntityReferenceList]: """ @@ -160,9 +160,9 @@ def yield_dashboard( dashboard_request = CreateDashboardRequest( name=EntityName(str(dashboard_details["id"])), displayName=dashboard_details.get("name"), - description=Markdown(dashboard_description) - if dashboard_description - else None, + description=( + Markdown(dashboard_description) if dashboard_description else None + ), charts=[ FullyQualifiedEntityName( fqn.build( @@ -275,9 +275,11 @@ def yield_dashboard_chart( yield Either( right=CreateChartRequest( name=EntityName(str(widgets["id"])), - displayName=chart_display_name - if visualization and visualization["query"] - else "", + displayName=( + chart_display_name + if visualization and visualization["query"] + else "" + ), chartType=get_standard_chart_type( visualization["type"] if visualization else "" ), @@ -285,9 +287,11 @@ def yield_dashboard_chart( self.context.get().dashboard_service ), sourceUrl=SourceUrl(self.get_dashboard_url(dashboard_details)), - description=Markdown(visualization["description"]) - if visualization - else None, + description=( + Markdown(visualization["description"]) + if visualization + else None + ), ) ) except Exception as exc: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index 9eb49593dd46..718cffb0b833 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -227,6 +227,17 @@ def _clearn_column_datatype(self, datatype: str) -> str: """clean datatype of column fetched from superset""" return datatype.replace("()", "") + def parse_array_data_type(self, col_parse: dict) -> Optional[str]: + """ + Set arrayDataType to UNKNOWN for Snowflake table array columns + to prevent validation error requiring non-null arrayDataType + """ + if col_parse["dataType"] == "ARRAY" and not col_parse.get("arrayDataType"): + return DataType.UNKNOWN + if col_parse.get("arrayDataType"): + return DataType(col_parse["arrayDataType"]) + return None + def get_column_info( self, data_source: List[Union[DataSourceResult, FetchColumn]] ) -> Optional[List[Column]]: @@ -247,9 +258,7 @@ def get_column_info( parsed_fields = Column( dataTypeDisplay=field.type, dataType=col_parse["dataType"], - arrayDataType=DataType(col_parse["arrayDataType"]) - if col_parse.get("arrayDataType") - else None, + arrayDataType=self.parse_array_data_type(col_parse), children=list(col_parse["children"]) if col_parse.get("children") else None, diff --git a/ingestion/tests/unit/test_column_type_parser.py b/ingestion/tests/unit/test_column_type_parser.py index 33fbc47a0367..5cdd9bad6802 100644 --- a/ingestion/tests/unit/test_column_type_parser.py +++ b/ingestion/tests/unit/test_column_type_parser.py @@ -17,6 +17,7 @@ from unittest import TestCase from metadata.generated.schema.entity.data.table import DataType +from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser from metadata.utils.datalake.datalake_utils import GenericDataFrameColumnParser @@ -132,3 +133,16 @@ def test_check_datalake_type(): assert assert_col_type_dict.get( column_name ) == GenericDataFrameColumnParser.fetch_col_types(df, column_name) + + +def test_superset_parse_array_data_type(): + """Test the parse_array_data_type method with different input scenarios""" + col_parse = {"dataType": "ARRAY", "arrayDataType": "STRING"} + result = SupersetSourceMixin.parse_array_data_type(None, col_parse) + assert result == DataType.STRING + col_parse = {"dataType": "ARRAY", "arrayDataType": None} + result = SupersetSourceMixin.parse_array_data_type(None, col_parse) + assert result == DataType.UNKNOWN + col_parse = {"dataType": "STRING", "arrayDataType": None} + result = SupersetSourceMixin.parse_array_data_type(None, col_parse) + assert result == None diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/doris/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/doris/yaml.md index d09035fc7ef9..dde4fb06f9b5 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/database/doris/yaml.md +++ b/openmetadata-docs/content/v1.6.x/connectors/database/doris/yaml.md @@ -69,8 +69,6 @@ This is a sample config for Doris: {% codeInfo srNumber=2 %} -{% codeInfo srNumber=2 %} - **password**: Password to connect to Doris. {% /codeInfo %} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/doris/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/doris/yaml.md index cd17be2b1297..bf2dccdc653a 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/doris/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/doris/yaml.md @@ -69,8 +69,6 @@ This is a sample config for Doris: {% codeInfo srNumber=2 %} -{% codeInfo srNumber=2 %} - **password**: Password to connect to Doris. {% /codeInfo %} diff --git a/openmetadata-docs/images/v1.5/connectors/doris/add-new-service.png b/openmetadata-docs/images/v1.5/connectors/doris/add-new-service.png new file mode 100644 index 000000000000..71b1daec8039 Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/doris/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.5/connectors/doris/select-service.png b/openmetadata-docs/images/v1.5/connectors/doris/select-service.png new file mode 100644 index 000000000000..1944326d4471 Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/doris/select-service.png differ diff --git a/openmetadata-docs/images/v1.5/connectors/doris/service-connection.png b/openmetadata-docs/images/v1.5/connectors/doris/service-connection.png new file mode 100644 index 000000000000..4f81ec8f04b5 Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/doris/service-connection.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/doris/add-new-service.png b/openmetadata-docs/images/v1.6/connectors/doris/add-new-service.png new file mode 100644 index 000000000000..71b1daec8039 Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/doris/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/doris/select-service.png b/openmetadata-docs/images/v1.6/connectors/doris/select-service.png new file mode 100644 index 000000000000..1944326d4471 Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/doris/select-service.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/doris/service-connection.png b/openmetadata-docs/images/v1.6/connectors/doris/service-connection.png new file mode 100644 index 000000000000..4f81ec8f04b5 Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/doris/service-connection.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/doris/add-new-service.png b/openmetadata-docs/images/v1.7/connectors/doris/add-new-service.png new file mode 100644 index 000000000000..71b1daec8039 Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/doris/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/doris/select-service.png b/openmetadata-docs/images/v1.7/connectors/doris/select-service.png new file mode 100644 index 000000000000..1944326d4471 Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/doris/select-service.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/doris/service-connection.png b/openmetadata-docs/images/v1.7/connectors/doris/service-connection.png new file mode 100644 index 000000000000..4f81ec8f04b5 Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/doris/service-connection.png differ