fix(ingestion/powerbi): fix issue with broken report lineage

metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Iterable, Optional
 
 from pydantic.fields import Field
@@ -18,6 +19,8 @@
 from datahub.specific.dashboard import DashboardPatchBuilder
 from datahub.specific.dataset import DatasetPatchBuilder
 
+logger = logging.getLogger(__name__)
+
 
 def convert_upstream_lineage_to_patch(
     urn: str,
@@ -76,8 +79,36 @@ def convert_dashboard_info_to_patch(
         for chartEdge in aspect.chartEdges:
             patch_builder.add_chart_edge(chartEdge)
 
+    if aspect.title:
+        patch_builder.set_title(aspect.title)
+
+    if aspect.description:
+        patch_builder.set_description(aspect.description)
+
+    if aspect.charts:
+        patch_builder.set_charts(aspect.charts)
+
+    if aspect.dashboardUrl:
+        patch_builder.set_dashboard_url(aspect.dashboardUrl)
+
+    if aspect.datasets:
+        patch_builder.set_datasets(aspect.datasets)
+
+    if aspect.access:
+        patch_builder.set_access(aspect.access)
+
+    if aspect.lastRefreshed:
+        patch_builder.set_last_refreshed(aspect.lastRefreshed)
+
+    if aspect.lastModified:
+        patch_builder.set_last_modified(last_modified=aspect.lastModified)
+
     values = patch_builder.build()
+
     if values:
+        logger.debug(
+            f"Generating patch DashboardInfo MetadataWorkUnit for dashboard {aspect.title}"
+        )
         mcp = next(iter(values))
         return MetadataWorkUnit(
             id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp

metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py

@@ -1197,8 +1197,7 @@ def report_to_datahub_work_units(
     ) -> Iterable[MetadataWorkUnit]:
         mcps: List[MetadataChangeProposalWrapper] = []
 
-        logger.debug(f"Converting dashboard={report.name} to datahub dashboard")
-
+        logger.debug(f"Converting report={report.name} to datahub dashboard")
         # Convert user to CorpUser
         user_mcps = self.to_datahub_users(report.users)
         # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization
@@ -1215,9 +1214,7 @@ def report_to_datahub_work_units(
         mcps.extend(chart_mcps)
         mcps.extend(report_mcps)
 
-        # Convert MCP to work_units
-        work_units = map(self._to_work_unit, mcps)
-        return work_units
+        return map(self._to_work_unit, mcps)
 
 
 @platform_name("PowerBI")

metadata-ingestion/src/datahub/specific/dashboard.py

@@ -3,7 +3,9 @@
 
 from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
 from datahub.metadata.schema_classes import (
+    AccessLevelClass,
     AuditStampClass,
+    ChangeAuditStampsClass,
     DashboardInfoClass as DashboardInfo,
     EdgeClass as Edge,
     GlobalTagsClass as GlobalTags,
@@ -405,3 +407,119 @@ def remove_custom_property(self, key: str) -> "DashboardPatchBuilder":
         """
         self.custom_properties_patch_helper.remove_property(key)
         return self
+
+    def set_title(self, title: str) -> "DashboardPatchBuilder":
+        assert title, "DashboardInfo title should not be None"
+        self._add_patch(
+            DashboardInfo.ASPECT_NAME,
+            "add",
+            path="/title",
+            value=title,
+        )
+
+        return self
+
+    def set_description(self, description: str) -> "DashboardPatchBuilder":
+        assert description, "DashboardInfo description should not be None"
+        self._add_patch(
+            DashboardInfo.ASPECT_NAME,
+            "add",
+            path="/description",
+            value=description,
+        )
+
+        return self
+
+    def add_custom_properties(
+        self, custom_properties: Optional[Dict[str, str]] = None
+    ) -> "DashboardPatchBuilder":
+
+        if custom_properties:
+            for key, value in custom_properties.items():
+                self.custom_properties_patch_helper.add_property(key, value)
+
+        return self
+
+    def set_external_url(self, external_url: Optional[str]) -> "DashboardPatchBuilder":
+        if external_url:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/externalUrl",
+                value=external_url,
+            )
+        return self
+
+    def set_charts(self, charts: Optional[List[str]]) -> "DashboardPatchBuilder":
+        if charts:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/charts",
+                value=charts,
+            )
+
+        return self
+
+    def set_datasets(self, datasets: Optional[List[str]]) -> "DashboardPatchBuilder":
+        if datasets:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/datasets",
+                value=datasets,
+            )
+
+        return self
+
+    def set_dashboard_url(
+        self, dashboard_url: Optional[str]
+    ) -> "DashboardPatchBuilder":
+        if dashboard_url:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/dashboardUrl",
+                value=dashboard_url,
+            )
+
+        return self
+
+    def set_access(
+        self, access: Union[None, Union[str, "AccessLevelClass"]] = None
+    ) -> "DashboardPatchBuilder":
+        if access:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/access",
+                value=access,
+            )
+
+        return self
+
+    def set_last_refreshed(
+        self, last_refreshed: Optional[int]
+    ) -> "DashboardPatchBuilder":
+        if last_refreshed:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/lastRefreshed",
+                value=last_refreshed,
+            )
+
+        return self
+
+    def set_last_modified(
+        self, last_modified: "ChangeAuditStampsClass"
+    ) -> "DashboardPatchBuilder":
+        if last_modified:
+            self._add_patch(
+                DashboardInfo.ASPECT_NAME,
+                "add",
+                path="/lastModified",
+                value=last_modified,
+            )
+
+        return self

metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json

@@ -312,6 +312,43 @@
                 "op": "add",
                 "path": "/customProperties/workspaceId",
                 "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts",
+                "value": [
+                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+                ]
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
             }
         ]
     },

metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json

@@ -1226,6 +1226,38 @@
                 "op": "add",
                 "path": "/customProperties/workspaceId",
                 "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts",
+                "value": [
+                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+                ]
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
             }
         ]
     },
@@ -1951,6 +1983,50 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dashboard",
+    "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",
+    "changeType": "PATCH",
+    "aspectName": "dashboardInfo",
+    "aspect": {
+        "json": [
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Acryl sales marketing report"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "SalesMarketing"
+            }
+        ]
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "powerbi-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "dashboard",
     "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)",

metadata-ingestion/tests/integration/powerbi/golden_test_cll.json

@@ -1276,6 +1276,43 @@
                 "op": "add",
                 "path": "/customProperties/workspaceId",
                 "value": "64ED5CAD-7C10-4684-8180-826122881108"
+            },
+            {
+                "op": "add",
+                "path": "/title",
+                "value": "test_dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/description",
+                "value": "Description of test dashboard"
+            },
+            {
+                "op": "add",
+                "path": "/charts",
+                "value": [
+                    "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
+                    "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
+                ]
+            },
+            {
+                "op": "add",
+                "path": "/dashboardUrl",
+                "value": "https://localhost/dashboards/web/1"
+            },
+            {
+                "op": "add",
+                "path": "/lastModified",
+                "value": {
+                    "created": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    },
+                    "lastModified": {
+                        "time": 0,
+                        "actor": "urn:li:corpuser:unknown"
+                    }
+                }
             }
         ]
     },