From 6c48007eeab9a861ac0d319271d00dfc5dae8d37 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Thu, 8 Feb 2024 00:13:26 +0530 Subject: [PATCH] Address review comments --- .../ingestion/source/qlik_sense/config.py | 9 + .../source/qlik_sense/data_classes.py | 49 +- .../ingestion/source/qlik_sense/qlik_api.py | 138 +++-- .../ingestion/source/qlik_sense/qlik_sense.py | 162 +++--- .../golden_test_platform_instance_ingest.json | 473 +++++++++++++----- .../golden_test_qlik_sense_ingest.json | 434 ++++++++++++---- .../integration/qlik_sense/test_qlik_sense.py | 311 +++++++++++- 7 files changed, 1230 insertions(+), 346 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index d3303b04a8cec..1bcd89dfcb491 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -31,6 +31,8 @@ class Constant: ID = "id" NAME = "name" TYPE = "type" + ITEMID = "itemId" + DATASETTYPE = "datasetType" CREATEDAT = "createdAt" UPDATEDAT = "updatedAt" SECUREQRI = "secureQri" @@ -52,10 +54,17 @@ class Constant: MODIFIEDDATE = "modifiedDate" RESOURCEID = "resourceId" DATASETSCHEMA = "datasetSchema" + GRAPH = "graph" + NODES = "nodes" + RESOURCES = "resources" + LINEAGE = "lineage" + TABLELABEL = "tableLabel" + TABLEQRI = "tableQRI" # Websocket response key constants QID = "qId" RESULT = "result" QRETURN = "qReturn" + QTYPE = "qType" QHANDLE = "qHandle" QLAYOUT = "qLayout" QMETA = "qMeta" diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py index d3caf80f2bb94..a7c6f76464b63 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -61,6 +61,12 @@ class SpaceType(Enum): DATA = "data" +# Qlik table box type +class BoxType(Enum): + LOADFILE = "load-file" # Table extracted from dataset + BLACKBOX = "blackbox" # Table extracted from data connection + + PERSONAL_SPACE_DICT = { "id": Constant.PERSONAL_SPACE_ID, "name": Constant.PERSONAL_SPACE_NAME, @@ -118,6 +124,7 @@ class QlikDataset(Item): type: str size: int rowCount: int + itemId: str datasetSchema: List[SchemaField] @root_validator(pre=True) @@ -140,9 +147,26 @@ def update_values(cls, values: Dict) -> Dict: return values +class AxisProperty(BaseModel): + Title: str = Field(alias="qFallbackTitle") + Min: str = Field(alias="qMin") + Max: str = Field(alias="qMax") + + class Chart(BaseModel): qId: str - qType: str + visualization: str + title: str + subtitle: str + qDimension: List[AxisProperty] + qMeasure: List[AxisProperty] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.QID] = values[Constant.QINFO][Constant.QID] + values["qDimension"] = values["qHyperCube"]["qDimensionInfo"] + values["qMeasure"] = values["qHyperCube"]["qMeasureInfo"] + return values class Sheet(BaseModel): @@ -165,28 +189,33 @@ def update_values(cls, values: Dict) -> Dict: return values -class QlikAppDataset(BaseModel): +class QlikTable(BaseModel): tableName: str - schemaName: str databaseName: str + type: BoxType = Field(alias="boxType") tableAlias: str dataconnectorid: str dataconnectorName: str dataconnectorPlatform: str + spaceId: str datasetSchema: List[SchemaField] = Field(alias="fields") + tableQri: Optional[str] = None + schemaName: Optional[str] = None @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: - values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][ - Constant.TABLEQUALIFIERS - ][0] - values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][ - Constant.TABLEQUALIFIERS - ][1] + if values["boxType"] == BoxType.BLACKBOX.value: + values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][ + Constant.TABLEQUALIFIERS + ][0] + values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][ + Constant.TABLEQUALIFIERS + ][1] values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID] values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][ Constant.SOURCECONNECTORID ] + values[Constant.SPACEID] = values[Constant.CONNECTIONINFO]["space"] return values @@ -195,7 +224,7 @@ class App(Item): qri: str qUsage: str sheets: List[Sheet] = [] - datasets: List[QlikAppDataset] = [] + tables: List[QlikTable] = [] @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index 3bc64d692766d..d5d559144ef16 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -1,6 +1,7 @@ import logging import sys from typing import Any, Dict, List, Optional +from urllib.parse import quote import requests @@ -10,8 +11,8 @@ App, Chart, Item, - QlikAppDataset, QlikDataset, + QlikTable, Sheet, Space, ) @@ -60,18 +61,20 @@ def get_spaces(self) -> List[Space]: self.spaces[PERSONAL_SPACE_DICT[Constant.ID]] = PERSONAL_SPACE_DICT[ Constant.NAME ] - except Exception: - self._log_http_error(message="Unable to fetch spaces") + except Exception as e: + self._log_http_error(message=f"Unable to fetch spaces. Exception: {e}") return spaces - def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: + def _get_dataset(self, dataset_id: str, item_id: str) -> Optional[QlikDataset]: try: response = self.session.get(f"{self.rest_api_url}/data-sets/{dataset_id}") response.raise_for_status() - return QlikDataset.parse_obj(response.json()) - except Exception: + response_dict = response.json() + response_dict[Constant.ITEMID] = item_id + return QlikDataset.parse_obj(response_dict) + except Exception as e: self._log_http_error( - message=f"Unable to fetch dataset with id {dataset_id}" + message=f"Unable to fetch dataset with id {dataset_id}. Exception: {e}" ) return None @@ -86,8 +89,28 @@ def get_user_name(self, user_id: str) -> Optional[str]: user_name = response.json()[Constant.NAME] self.users[user_id] = user_name return user_name - except Exception: - self._log_http_error(message=f"Unable to fetch user with id {user_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch user with id {user_id}. Exception: {e}" + ) + return None + + def _get_chart( + self, + websocket_connection: WebsocketConnection, + chart_id: str, + sheet_id: str, + ) -> Optional[Chart]: + try: + websocket_connection.websocket_send_request( + method="GetChild", params={"qId": chart_id} + ) + response = websocket_connection.websocket_send_request(method="GetLayout") + return Chart.parse_obj(response[Constant.QLAYOUT]) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch chart {chart_id} of sheet {sheet_id}. Exception: {e}" + ) return None def _get_sheet( @@ -102,33 +125,83 @@ def _get_sheet( response = websocket_connection.websocket_send_request(method="GetLayout") sheet_dict = response[Constant.QLAYOUT] sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) + i = 1 for chart_dict in sheet_dict[Constant.QCHILDLIST][Constant.QITEMS]: - sheet.charts.append(Chart.parse_obj(chart_dict[Constant.QINFO])) + chart = self._get_chart( + websocket_connection, + chart_dict[Constant.QINFO][Constant.QID], + sheet_id, + ) + if chart: + if not chart.title: + chart.title = f"Object {i}" + i += 1 + sheet.charts.append(chart) + websocket_connection.handle.pop() return sheet - except Exception: - self._log_http_error(message=f"Unable to fetch sheet with id {sheet_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheet with id {sheet_id}. Exception: {e}" + ) return None - def _get_app_used_datasets( + def _add_qri_of_tables(self, tables: List[QlikTable], app_id: str) -> None: + table_qri_dict: Dict[str, str] = {} + app_qri = quote(f"qri:app:sense://{app_id}", safe="") + try: + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={app_qri}&level=TABLE" + ) + response.raise_for_status() + for table_node_qri in response.json()[Constant.GRAPH][Constant.NODES]: + table_node_qri = quote(table_node_qri, safe="") + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={table_node_qri}&level=FIELD" + ) + response.raise_for_status() + field_node_qri = list( + response.json()[Constant.GRAPH][Constant.NODES].keys() + )[0] + response = self.session.post( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview", + json=[field_node_qri], + ) + response.raise_for_status() + for each_lineage in response.json()[Constant.RESOURCES][0][ + Constant.LINEAGE + ]: + table_qri_dict[each_lineage[Constant.TABLELABEL]] = each_lineage[ + Constant.TABLEQRI + ] + for table in tables: + if table.tableName in table_qri_dict: + table.tableQri = table_qri_dict[table.tableName] + except Exception as e: + self._log_http_error( + message=f"Unable to add QRI for tables of app {app_id}. Exception: {e}" + ) + + def _get_app_used_tables( self, websocket_connection: WebsocketConnection, app_id: str - ) -> List[QlikAppDataset]: - datasets: List[QlikAppDataset] = [] + ) -> List[QlikTable]: + tables: List[QlikTable] = [] try: - websocket_connection.websocket_send_request( + response = websocket_connection.websocket_send_request( method="GetObject", params=["LoadModel"], ) + if not response[Constant.QRETURN][Constant.QTYPE]: + return [] response = websocket_connection.websocket_send_request(method="GetLayout") for table_dict in response[Constant.QLAYOUT][Constant.TABLES]: - # Condition to Add connection based table only - if table_dict["boxType"] == "blackbox": - datasets.append(QlikAppDataset.parse_obj(table_dict)) + tables.append(QlikTable.parse_obj(table_dict)) websocket_connection.handle.pop() - except Exception: + self._add_qri_of_tables(tables, app_id) + except Exception as e: self._log_http_error( - message=f"Unable to fetch app used datasets for app {app_id}" + message=f"Unable to fetch tables used by app {app_id}. Exception: {e}" ) - return datasets + return tables def _get_app_sheets( self, websocket_connection: WebsocketConnection, app_id: str @@ -151,8 +224,10 @@ def _get_app_sheets( if sheet: sheets.append(sheet) websocket_connection.handle.pop() - except Exception: - self._log_http_error(message=f"Unable to fetch sheets for app {app_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheets for app {app_id}. Exception: {e}" + ) return sheets def _get_app(self, app_id: str) -> Optional[App]: @@ -169,11 +244,13 @@ def _get_app(self, app_id: str) -> Optional[App]: ) app = App.parse_obj(response[Constant.QLAYOUT]) app.sheets = self._get_app_sheets(websocket_connection, app_id) - app.datasets = self._get_app_used_datasets(websocket_connection, app_id) + app.tables = self._get_app_used_tables(websocket_connection, app_id) websocket_connection.close_websocket() return app - except Exception: - self._log_http_error(message=f"Unable to fetch app with id {app_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch app with id {app_id}. Exception: {e}" + ) return None def get_items(self) -> List[Item]: @@ -196,11 +273,12 @@ def get_items(self) -> List[Item]: items.append(app) elif resource_type == Constant.DATASET: dataset = self._get_dataset( - dataset_id=item[Constant.RESOURCEID] + dataset_id=item[Constant.RESOURCEID], + item_id=item[Constant.ID], ) if dataset: items.append(dataset) - except Exception: - self._log_http_error(message="Unable to fetch items") + except Exception as e: + self._log_http_error(message=f"Unable to fetch items. Exception: {e}") return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index 1cfd329c35b73..3cdc5c846dd79 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -37,9 +37,10 @@ KNOWN_DATA_PLATFORM_MAPPING, App, AppKey, + BoxType, Chart, - QlikAppDataset, QlikDataset, + QlikTable, SchemaField as QlikDatasetSchemaField, Sheet, Space, @@ -183,6 +184,7 @@ def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: owner_urn=builder.make_user_urn(owner_username) if self.config.ingest_owner and owner_username else None, + external_url=f"https://{self.config.tenant_hostname}/catalog?space_filter={space.id}", created=int(space.createdAt.timestamp() * 1000), last_modified=int(space.updatedAt.timestamp() * 1000), ) @@ -215,7 +217,9 @@ def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: name=dashboard_identifier, ) - def _gen_dashboard_info_workunit(self, sheet: Sheet) -> MetadataWorkUnit: + def _gen_dashboard_info_workunit( + self, sheet: Sheet, app_id: str + ) -> MetadataWorkUnit: dashboard_urn = self._gen_dashboard_urn(sheet.id) custom_properties: Dict[str, str] = {"chartCount": str(len(sheet.charts))} dashboard_info_cls = DashboardInfoClass( @@ -234,6 +238,7 @@ def _gen_dashboard_info_workunit(self, sheet: Sheet) -> MetadataWorkUnit: lastModified=self._get_audit_stamp(sheet.updatedAt, sheet.ownerId), ), customProperties=custom_properties, + dashboardUrl=f"https://{self.config.tenant_hostname}/sense/app/{app_id}/sheet/{sheet.id}/state/analysis", ) return MetadataChangeProposalWrapper( entityUrn=dashboard_urn, aspect=dashboard_info_cls @@ -252,17 +257,23 @@ def _gen_charts_workunit(self, charts: List[Chart]) -> Iterable[MetadataWorkUnit yield self._gen_entity_status_aspect(chart_urn) + custom_properties = { + "Dimension": str(chart.qDimension), + "Measure": str(chart.qMeasure), + } + yield MetadataChangeProposalWrapper( entityUrn=chart_urn, aspect=ChartInfoClass( - title=chart.qId, - description=chart.qType, + title=chart.title, + description=chart.visualization, lastModified=ChangeAuditStampsClass(), + customProperties=custom_properties, ), ).as_workunit() def _gen_sheets_workunit( - self, sheets: List[Sheet], space_id: str, app_id: str + self, sheets: List[Sheet], app_id: str ) -> Iterable[MetadataWorkUnit]: """ Map Qlik Sheet to Datahub dashboard @@ -272,7 +283,7 @@ def _gen_sheets_workunit( yield self._gen_entity_status_aspect(dashboard_urn) - yield self._gen_dashboard_info_workunit(sheet) + yield self._gen_dashboard_info_workunit(sheet, app_id) yield from add_entity_to_container( container_key=self._gen_app_key(app_id), @@ -290,75 +301,83 @@ def _gen_sheets_workunit( yield from self._gen_charts_workunit(sheet.charts) - def _gen_app_dataset_upstream_lineage( - self, dataset: QlikAppDataset - ) -> MetadataWorkUnit: - dataset_identifier = self._get_app_dataset_identifier(dataset) - dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) - upstream_dataset_platform_detail = ( - self.config.data_connection_to_platform_instance.get( - dataset.dataconnectorName, - PlatformDetail( - env=self.config.env, platform_instance=self.config.platform_instance + def _gen_app_table_upstream_lineage( + self, dataset_urn: str, table: QlikTable + ) -> Optional[MetadataWorkUnit]: + upstream_dataset_urn: Optional[str] = None + if table.type == BoxType.BLACKBOX: + upstream_dataset_platform_detail = ( + self.config.data_connection_to_platform_instance.get( + table.dataconnectorName, + PlatformDetail( + env=self.config.env, + platform_instance=self.config.platform_instance, + ), + ) + ) + upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( + name=f"{table.databaseName}.{table.schemaName}.{table.tableName}", + platform=KNOWN_DATA_PLATFORM_MAPPING.get( + table.dataconnectorPlatform, table.dataconnectorPlatform ), + env=upstream_dataset_platform_detail.env, + platform_instance=upstream_dataset_platform_detail.platform_instance, ) - ) - - upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( - name=dataset_identifier, - platform=KNOWN_DATA_PLATFORM_MAPPING.get( - dataset.dataconnectorPlatform, dataset.dataconnectorPlatform - ), - env=upstream_dataset_platform_detail.env, - platform_instance=upstream_dataset_platform_detail.platform_instance, - ) - - return MetadataChangeProposalWrapper( - entityUrn=dataset_urn, - aspect=UpstreamLineage( - upstreams=[ - Upstream(dataset=upstream_dataset_urn, type=DatasetLineageType.COPY) - ] - ), - ).as_workunit() - - def _gen_app_dataset_properties(self, dataset: QlikAppDataset) -> MetadataWorkUnit: - dataset_urn = self._gen_qlik_dataset_urn( - self._get_app_dataset_identifier(dataset) - ) + elif table.type == BoxType.LOADFILE: + upstream_dataset_urn = self._gen_qlik_dataset_urn( + f"{table.spaceId}.{table.databaseName}" + ) + if upstream_dataset_urn: + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineage( + upstreams=[ + Upstream( + dataset=upstream_dataset_urn, type=DatasetLineageType.COPY + ) + ] + ), + ).as_workunit() + else: + return None + def _gen_app_table_properties( + self, dataset_urn: str, table: QlikTable + ) -> MetadataWorkUnit: dataset_properties = DatasetProperties( - name=dataset.tableName, - qualifiedName=dataset.tableAlias, + name=table.tableName, + qualifiedName=table.tableAlias, ) dataset_properties.customProperties.update( { - Constant.DATACONNECTORID: dataset.dataconnectorid, - Constant.DATACONNECTORNAME: dataset.dataconnectorName, + Constant.TYPE: "Qlik Table", + Constant.DATACONNECTORID: table.dataconnectorid, + Constant.DATACONNECTORNAME: table.dataconnectorName, } ) - return MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=dataset_properties ).as_workunit() - def _get_app_dataset_identifier(self, dataset: QlikAppDataset) -> str: - return ( - f"{dataset.databaseName}.{dataset.schemaName}.{dataset.tableName}".lower() - ) + def _get_app_table_identifier(self, table: QlikTable) -> Optional[str]: + if table.tableQri: + return table.tableQri.lower() + return None - def _gen_app_dataset_workunit( - self, datasets: List[QlikAppDataset], app_id: str + def _gen_app_tables_workunit( + self, tables: List[QlikTable], app_id: str ) -> Iterable[MetadataWorkUnit]: - for dataset in datasets: - dataset_identifier = self._get_app_dataset_identifier(dataset) - dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + for table in tables: + table_identifier = self._get_app_table_identifier(table) + if not table_identifier: + continue + dataset_urn = self._gen_qlik_dataset_urn(table_identifier) yield self._gen_entity_status_aspect(dataset_urn) - yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) + yield self._gen_schema_metadata(table_identifier, table.datasetSchema) - yield self._gen_app_dataset_properties(dataset) + yield self._gen_app_table_properties(dataset_urn, table) yield from add_entity_to_container( container_key=self._gen_app_key(app_id), @@ -375,7 +394,11 @@ def _gen_app_dataset_workunit( aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), ).as_workunit() - yield self._gen_app_dataset_upstream_lineage(dataset) + upstream_lineage_workunit = self._gen_app_table_upstream_lineage( + dataset_urn, table + ) + if upstream_lineage_workunit: + yield upstream_lineage_workunit def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: """ @@ -392,11 +415,12 @@ def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: owner_urn=builder.make_user_urn(owner_username) if self.config.ingest_owner and owner_username else None, + external_url=f"https://{self.config.tenant_hostname}/sense/app/{app.id}/overview", created=int(app.createdAt.timestamp() * 1000), last_modified=int(app.updatedAt.timestamp() * 1000), ) - yield from self._gen_sheets_workunit(app.sheets, app.spaceId, app.id) - yield from self._gen_app_dataset_workunit(app.datasets, app.id) + yield from self._gen_sheets_workunit(app.sheets, app.id) + yield from self._gen_app_tables_workunit(app.tables, app.id) def _gen_qlik_dataset_urn(self, dataset_identifier: str) -> str: return builder.make_dataset_urn_with_platform_instance( @@ -459,13 +483,14 @@ def _gen_schema_metadata( entityUrn=dataset_urn, aspect=schema_metadata ).as_workunit() - def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: - dataset_urn = self._gen_qlik_dataset_urn(dataset.id) - + def _gen_dataset_properties( + self, dataset_urn: str, dataset: QlikDataset + ) -> MetadataWorkUnit: dataset_properties = DatasetProperties( name=dataset.name, description=dataset.description, qualifiedName=dataset.name, + externalUrl=f"https://{self.config.tenant_hostname}/dataset/{dataset.itemId}", created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), ) @@ -473,7 +498,8 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: { Constant.QRI: dataset.secureQri, Constant.SPACEID: dataset.spaceId, - Constant.TYPE: dataset.type, + Constant.TYPE: "Qlik Dataset", + Constant.DATASETTYPE: dataset.type, Constant.SIZE: str(dataset.size), Constant.ROWCOUNT: str(dataset.rowCount), } @@ -483,14 +509,18 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: entityUrn=dataset_urn, aspect=dataset_properties ).as_workunit() + def _get_qlik_dataset_identifier(self, dataset: QlikDataset) -> str: + return f"{dataset.spaceId}.{dataset.name}".lower() + def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: - dataset_urn = self._gen_qlik_dataset_urn(dataset.id) + dataset_identifier = self._get_qlik_dataset_identifier(dataset) + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) yield self._gen_entity_status_aspect(dataset_urn) - yield self._gen_schema_metadata(dataset.id, dataset.datasetSchema) + yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) - yield self._gen_dataset_properties(dataset) + yield self._gen_dataset_properties(dataset_urn, dataset) yield from add_entity_to_container( container_key=self._gen_space_key(dataset.spaceId), diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index 92415127d414e..7cfe225dddbd7 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -12,6 +12,7 @@ "space": "659d0e41d1b0ecce6eebc9b1", "type": "SpaceType.SHARED" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", "name": "test_space", "description": "", "created": { @@ -23,7 +24,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363253, + "lastObserved": 1707327546102, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -39,7 +40,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363254, + "lastObserved": 1707327546103, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -56,7 +57,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363255, + "lastObserved": 1707327546104, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -74,7 +75,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363256, + "lastObserved": 1707327546105, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -99,7 +100,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363256, + "lastObserved": 1707327546106, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -120,7 +121,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363257, + "lastObserved": 1707327546106, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -138,18 +139,19 @@ "space": "personal-space-id", "type": "SpaceType.PERSONAL" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", "name": "personal_space", "description": "", "created": { - "time": 1707047363158 + "time": 1707327546001 }, "lastModified": { - "time": 1707047363158 + "time": 1707327546001 } } }, "systemMetadata": { - "lastObserved": 1707047363264, + "lastObserved": 1707327546121, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -165,7 +167,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363265, + "lastObserved": 1707327546122, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -182,7 +184,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363266, + "lastObserved": 1707327546123, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -200,7 +202,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363266, + "lastObserved": 1707327546124, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -221,14 +223,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363267, + "lastObserved": 1707327546124, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -236,10 +238,11 @@ "customProperties": { "platform": "qlik-sense", "instance": "qlik_sense_platform", - "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", "usage": "ANALYTICS" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", "name": "IPL_Matches_2022", "description": "", "created": { @@ -251,14 +254,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363268, + "lastObserved": 1707327546126, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -267,14 +270,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363269, + "lastObserved": 1707327546127, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -284,14 +287,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363269, + "lastObserved": 1707327546127, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -302,14 +305,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363270, + "lastObserved": 1707327546128, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -327,14 +330,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363271, + "lastObserved": 1707327546129, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -343,14 +346,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363272, + "lastObserved": 1707327546130, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -368,7 +371,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363272, + "lastObserved": 1707327546130, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -384,7 +387,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363274, + "lastObserved": 1707327546132, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -414,11 +417,12 @@ "time": 1706511226868, "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" } - } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" } }, "systemMetadata": { - "lastObserved": 1707047363274, + "lastObserved": 1707327546132, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -430,11 +434,11 @@ "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } }, "systemMetadata": { - "lastObserved": 1707047363275, + "lastObserved": 1707327546133, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -451,7 +455,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363276, + "lastObserved": 1707327546134, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -476,7 +480,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363276, + "lastObserved": 1707327546135, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -498,14 +502,14 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363277, + "lastObserved": 1707327546136, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -521,7 +525,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363278, + "lastObserved": 1707327546137, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -533,9 +537,12 @@ "aspectName": "chartInfo", "aspect": { "json": { - "customProperties": {}, - "title": "QYUUb", - "description": "barchart", + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", "lastModified": { "created": { "time": 0, @@ -549,14 +556,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363279, + "lastObserved": 1707327546138, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -565,19 +572,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047363281, + "lastObserved": 1707327546139, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "harshal-playground-306419.test_dataset.test_table", + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -611,19 +618,20 @@ } }, "systemMetadata": { - "lastObserved": 1707047363282, + "lastObserved": 1707327546140, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { + "type": "Qlik Table", "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", "dataconnectorName": "Google_BigQuery_harshal-playground-306419" }, @@ -633,30 +641,30 @@ } }, "systemMetadata": { - "lastObserved": 1707047363283, + "lastObserved": 1707327546142, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } }, "systemMetadata": { - "lastObserved": 1707047363284, + "lastObserved": 1707327546142, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -666,14 +674,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363284, + "lastObserved": 1707327546143, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -684,14 +692,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363285, + "lastObserved": 1707327546144, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -709,14 +717,216 @@ } }, "systemMetadata": { - "lastObserved": 1707047363286, + "lastObserved": 1707327546144, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546145, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546149, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546150, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.IPL_Matches_2022.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546152, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -731,21 +941,21 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363287, + "lastObserved": 1707327546153, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -754,19 +964,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047363288, + "lastObserved": 1707327546155, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "659d8aef92df266ef3aa5a7c", + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -836,14 +1046,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363289, + "lastObserved": 1707327546155, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -851,10 +1061,12 @@ "customProperties": { "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", "spaceId": "659d0e41d1b0ecce6eebc9b1", - "type": "DELIMETED", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", "size": "38168", "rowCount": "74" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", "name": "IPL_Matches_2022.csv", "qualifiedName": "IPL_Matches_2022.csv", "description": "", @@ -868,14 +1080,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363291, + "lastObserved": 1707327546158, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -884,14 +1096,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363292, + "lastObserved": 1707327546159, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -901,14 +1113,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363293, + "lastObserved": 1707327546160, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -926,14 +1138,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363293, + "lastObserved": 1707327546161, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -944,14 +1156,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363294, + "lastObserved": 1707327546161, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -969,14 +1181,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363295, + "lastObserved": 1707327546162, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -985,19 +1197,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047363296, + "lastObserved": 1707327546163, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "65a137c849f82a37c625151b", + "schemaName": "personal-space-id.test_tabl", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -1031,14 +1243,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363297, + "lastObserved": 1707327546164, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -1046,10 +1258,12 @@ "customProperties": { "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", "spaceId": "personal-space-id", - "type": "CONNECTION_BASED_DATASET", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", "size": "0", "rowCount": "1" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", "name": "test_tabl", "qualifiedName": "test_tabl", "description": "", @@ -1063,14 +1277,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363298, + "lastObserved": 1707327546166, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1079,14 +1293,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363299, + "lastObserved": 1707327546167, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -1096,14 +1310,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363300, + "lastObserved": 1707327546167, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1121,14 +1335,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363300, + "lastObserved": 1707327546168, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1139,14 +1353,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363301, + "lastObserved": 1707327546169, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1164,7 +1378,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363302, + "lastObserved": 1707327546170, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1185,7 +1399,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363303, + "lastObserved": 1707327546170, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1206,14 +1420,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363303, + "lastObserved": 1707327546171, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1231,7 +1445,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363304, + "lastObserved": 1707327546172, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1253,21 +1467,50 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546172, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363305, + "lastObserved": 1707327546173, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1282,21 +1525,21 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363306, + "lastObserved": 1707327546174, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1314,14 +1557,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363306, + "lastObserved": 1707327546175, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1339,7 +1582,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363307, + "lastObserved": 1707327546176, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 2dd62c5e56940..32af5d2191de4 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -11,6 +11,7 @@ "space": "659d0e41d1b0ecce6eebc9b1", "type": "SpaceType.SHARED" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", "name": "test_space", "description": "", "created": { @@ -22,7 +23,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264838, + "lastObserved": 1707327442983, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -38,7 +39,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264839, + "lastObserved": 1707327442985, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -54,7 +55,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264840, + "lastObserved": 1707327442985, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -72,7 +73,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264840, + "lastObserved": 1707327442986, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -97,7 +98,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264841, + "lastObserved": 1707327442987, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -113,7 +114,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264842, + "lastObserved": 1707327442988, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -130,18 +131,19 @@ "space": "personal-space-id", "type": "SpaceType.PERSONAL" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", "name": "personal_space", "description": "", "created": { - "time": 1707047264739 + "time": 1707327442833 }, "lastModified": { - "time": 1707047264739 + "time": 1707327442833 } } }, "systemMetadata": { - "lastObserved": 1707047264849, + "lastObserved": 1707327443007, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -157,7 +159,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264850, + "lastObserved": 1707327443008, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -173,7 +175,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264850, + "lastObserved": 1707327443010, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -191,7 +193,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264851, + "lastObserved": 1707327443010, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -207,24 +209,25 @@ } }, "systemMetadata": { - "lastObserved": 1707047264852, + "lastObserved": 1707327443011, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { "platform": "qlik-sense", - "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", "usage": "ANALYTICS" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", "name": "IPL_Matches_2022", "description": "", "created": { @@ -236,14 +239,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264853, + "lastObserved": 1707327443012, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -252,14 +255,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264854, + "lastObserved": 1707327443014, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -268,14 +271,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264854, + "lastObserved": 1707327443014, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -286,14 +289,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264855, + "lastObserved": 1707327443015, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -311,14 +314,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264856, + "lastObserved": 1707327443016, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -327,14 +330,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264856, + "lastObserved": 1707327443017, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -348,7 +351,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264857, + "lastObserved": 1707327443018, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -364,7 +367,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264858, + "lastObserved": 1707327443019, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -394,11 +397,12 @@ "time": 1706511226868, "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" } - } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" } }, "systemMetadata": { - "lastObserved": 1707047264859, + "lastObserved": 1707327443020, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -410,11 +414,11 @@ "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } }, "systemMetadata": { - "lastObserved": 1707047264860, + "lastObserved": 1707327443022, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -439,7 +443,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264861, + "lastObserved": 1707327443023, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -457,14 +461,14 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264861, + "lastObserved": 1707327443024, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -480,7 +484,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264862, + "lastObserved": 1707327443025, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -492,9 +496,12 @@ "aspectName": "chartInfo", "aspect": { "json": { - "customProperties": {}, - "title": "QYUUb", - "description": "barchart", + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", "lastModified": { "created": { "time": 0, @@ -508,14 +515,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264863, + "lastObserved": 1707327443026, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -524,19 +531,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047264865, + "lastObserved": 1707327443028, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "harshal-playground-306419.test_dataset.test_table", + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -570,19 +577,20 @@ } }, "systemMetadata": { - "lastObserved": 1707047264866, + "lastObserved": 1707327443029, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { + "type": "Qlik Table", "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", "dataconnectorName": "Google_BigQuery_harshal-playground-306419" }, @@ -592,30 +600,30 @@ } }, "systemMetadata": { - "lastObserved": 1707047264867, + "lastObserved": 1707327443031, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } }, "systemMetadata": { - "lastObserved": 1707047264868, + "lastObserved": 1707327443032, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -626,14 +634,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264869, + "lastObserved": 1707327443033, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -651,14 +659,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264869, + "lastObserved": 1707327443034, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -669,21 +677,21 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264870, + "lastObserved": 1707327443035, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -692,19 +700,200 @@ } }, "systemMetadata": { - "lastObserved": 1707047264872, + "lastObserved": 1707327443037, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "659d8aef92df266ef3aa5a7c", + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443038, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443040, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443041, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443042, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.IPL_Matches_2022.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443043, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443045, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443048, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -774,14 +963,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264872, + "lastObserved": 1707327443050, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -789,10 +978,12 @@ "customProperties": { "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", "spaceId": "659d0e41d1b0ecce6eebc9b1", - "type": "DELIMETED", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", "size": "38168", "rowCount": "74" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", "name": "IPL_Matches_2022.csv", "qualifiedName": "IPL_Matches_2022.csv", "description": "", @@ -806,14 +997,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264874, + "lastObserved": 1707327443055, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -822,14 +1013,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264875, + "lastObserved": 1707327443057, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -847,14 +1038,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264876, + "lastObserved": 1707327443058, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -865,14 +1056,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264877, + "lastObserved": 1707327443060, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -886,14 +1077,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264878, + "lastObserved": 1707327443062, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -902,19 +1093,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047264880, + "lastObserved": 1707327443065, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "65a137c849f82a37c625151b", + "schemaName": "personal-space-id.test_tabl", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -948,14 +1139,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264881, + "lastObserved": 1707327443066, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -963,10 +1154,12 @@ "customProperties": { "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", "spaceId": "personal-space-id", - "type": "CONNECTION_BASED_DATASET", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", "size": "0", "rowCount": "1" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", "name": "test_tabl", "qualifiedName": "test_tabl", "description": "", @@ -980,14 +1173,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264882, + "lastObserved": 1707327443069, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -996,14 +1189,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264883, + "lastObserved": 1707327443071, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1021,14 +1214,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264884, + "lastObserved": 1707327443073, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1039,14 +1232,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264885, + "lastObserved": 1707327443073, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1060,7 +1253,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264886, + "lastObserved": 1707327443074, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1076,7 +1269,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264886, + "lastObserved": 1707327443076, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1092,14 +1285,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264887, + "lastObserved": 1707327443076, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1113,7 +1306,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264888, + "lastObserved": 1707327443078, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1131,21 +1324,46 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443079, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264888, + "lastObserved": 1707327443080, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1156,21 +1374,21 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264889, + "lastObserved": 1707327443081, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1184,14 +1402,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264890, + "lastObserved": 1707327443082, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1205,7 +1423,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264891, + "lastObserved": 1707327443082, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index a0bbdc4348e1b..c5a14c8d7432a 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Dict from unittest.mock import patch import pytest @@ -8,7 +8,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: - api_vs_response = { + api_vs_response: Dict[str, Dict] = { "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/api-keys": { "method": "GET", "status_code": 200, @@ -411,17 +411,156 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: }, }, }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647&level=TABLE": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "TABLE", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc": { + "label": "IPL_Matches_2022", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE": { + "label": "test_table", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#t91dFUED_ebvKHOjauxSOOCnlRZQTwEkNHv_bjUl7AY#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A": { + "label": "City", + "metadata": {"type": "FIELD"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ": { + "label": "Date", + "metadata": {"type": "FIELD"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_55m8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + }, + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_58X8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok": { + "label": "name", + "metadata": {"type": "FIELD"}, + } + }, + "edges": [ + { + "relation": "read", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + } + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/overview": { + "method": "POST", + "response_list": [ + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + "lineage": [ + { + "resourceLabel": "d7a6c03238b0c25b3d5c8631e1121807.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "tableLabel": "IPL_Matches_2022", + }, + ], + } + ] + }, + }, + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "lineage": [ + { + "resourceLabel": "3fcbce17de9e55774ddedc345532c419.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "tableLabel": "test_table", + } + ], + } + ] + }, + }, + ], + }, } api_vs_response.update(override_data) for url in api_vs_response.keys(): - request_mock.register_uri( - api_vs_response[url]["method"], - url, - json=api_vs_response[url]["json"], - status_code=api_vs_response[url]["status_code"], - ) + if api_vs_response[url].get("response_list"): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + response_list=api_vs_response[url]["response_list"], + ) + else: + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) def mock_websocket_response(*args, **kwargs): @@ -450,7 +589,7 @@ def mock_websocket_response(*args, **kwargs): return { "qLayout": { "qTitle": "IPL_Matches_2022", - "qFileName": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qFileName": "f0714ca7-7093-49e4-8b58-47bb38563647", "qLastReloadTime": "2024-01-15T11:06:53.070Z", "qHasScript": True, "qStateNames": [], @@ -460,7 +599,7 @@ def mock_websocket_response(*args, **kwargs): "qUnsupportedFeatures": [], "qUsage": "ANALYTICS", "encrypted": True, - "id": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", "published": False, "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", "ownerId": "657b5abe656297cec3d8b205", @@ -595,6 +734,108 @@ def mock_websocket_response(*args, **kwargs): elif request == { "jsonrpc": "2.0", "id": 6, + "handle": 2, + "method": "GetChild", + "params": {"qId": "QYUUb"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 3, + "qGenericType": "scatterplot", + "qGenericId": "QYUUb", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 7, + "handle": 3, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "QYUUb", "qType": "scatterplot"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qSelectionInfo": {}, + "qHyperCube": { + "qSize": {"qcx": 3, "qcy": 5}, + "qDimensionInfo": [ + { + "qFallbackTitle": "City", + "qApprMaxGlyphCount": 11, + "qCardinal": 5, + "qSortIndicator": "A", + "qGroupFallbackTitles": ["City"], + "qGroupPos": 0, + "qStateCounts": { + "qLocked": 0, + "qSelected": 0, + "qOption": 5, + "qDeselected": 0, + "qAlternative": 0, + "qExcluded": 0, + "qSelectedExcluded": 0, + "qLockedExcluded": 0, + }, + "qTags": [ + "$ascii", + "$text", + "$geoname", + "$relates_IPL_Matches_2022.City_GeoInfo", + ], + "qDimensionType": "D", + "qGrouping": "N", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qIsAutoFormat": True, + "qGroupFieldDefs": ["City"], + "qMin": "NaN", + "qMax": "NaN", + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qCardinalities": { + "qCardinal": 5, + "qHypercubeCardinal": 5, + "qAllValuesCardinal": -1, + }, + "autoSort": True, + "cId": "FdErA", + "othersLabel": "Others", + } + ], + "qMeasureInfo": [ + { + "qFallbackTitle": "Sum(Date)", + "qApprMaxGlyphCount": 7, + "qCardinal": 0, + "qSortIndicator": "D", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qMin": 89411, + "qMax": 2144260, + "qIsAutoFormat": True, + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qTrendLines": [], + "autoSort": True, + "cId": "PtTpyG", + "numFormatFromTemplate": True, + }, + ], + }, + "script": "", + "showTitles": True, + "title": "Test_chart", + "subtitle": "", + "footnote": "", + "disableNavMenu": False, + "showDetails": True, + "showDetailsExpression": False, + "visualization": "scatterplot", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 8, "handle": 1, "method": "GetObject", "params": ["LoadModel"], @@ -602,15 +843,15 @@ def mock_websocket_response(*args, **kwargs): return { "qReturn": { "qType": "GenericObject", - "qHandle": 3, + "qHandle": 4, "qGenericType": "LoadModel", "qGenericId": "LoadModel", } } elif request == { "jsonrpc": "2.0", - "id": 7, - "handle": 3, + "id": 9, + "handle": 4, "method": "GetLayout", "params": {}, }: @@ -689,15 +930,51 @@ def mock_websocket_response(*args, **kwargs): }, "selectStatement": "SELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", "caching": {"enabled": True, "type": "qvd"}, - } + }, + { + "dataconnectorName": "DataFiles", + "dataconnectorPrefix": "test_space:", + "boxType": "load-file", + "databaseName": "IPL_Matches_2022.csv", + "ownerName": "TYPE9_CSV", + "tableName": "IPL_Matches_2022", + "tableAlias": "IPL_Matches_2022", + "key": "DataFiles:IPL_Matches_2022.csv:TYPE9_CSV:IPL_Matches_2022", + "fields": [ + { + "id": "dsd.IPL_Matches_2022.City", + "name": "City", + "alias": "City", + "selected": True, + }, + { + "id": "dsd.IPL_Matches_2022.Date", + "name": "Date", + "alias": "Date", + "selected": True, + }, + ], + "connectionInfo": { + "type": {"isStorageProvider": False}, + "id": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "name": "DataFiles", + "typeName": "qix-datafiles.exe", + "sourceConnectorID": "qix-datafiles.exe", + "connectionString": 'CUSTOM CONNECT TO "provider=qix-datafiles.exe;path=test_space:datafiles;"', + "space": "659d0e41d1b0ecce6eebc9b1", + "dataconnectorPrefix": "test_space:", + "caching": {"enabled": True, "type": "qvd"}, + }, + "id": "dsd.IPL_Matches_2022", + "loadSelectStatement": "LOAD [ID] AS [ID],\n\t[City] AS [City],\n\t[Date] AS [Date],\n\t[Season] AS [Season],\n\t[MatchNumber] AS [MatchNumber],\n\t[Team1] AS [Team1],\n\t[Team2] AS [Team2],\n\t[Venue] AS [Venue],\n\t[TossWinner] AS [TossWinner],\n\t[TossDecision] AS [TossDecision],\n\t[SuperOver] AS [SuperOver],\n\t[WinningTeam] AS [WinningTeam],\n\t[WonBy] AS [WonBy],\n\t[Margin] AS [Margin],\n\t[method] AS [method],\n\t[Player_of_Match] AS [Player_of_Match],\n\t[Team1Players] AS [Team1Players],\n\t[Team2Players] AS [Team2Players],\n\t[Umpire1] AS [Umpire1],\n\t[Umpire2] AS [Umpire2]\nFROM [lib://DataFiles/IPL_Matches_2022.csv]\n(txt, codepage is 28591, embedded labels, delimiter is ',', msq);\n", + "formatSpec": "(txt, codepage is 28591, embedded labels, delimiter is ',', msq)", + "caching": {"enabled": True, "type": "qvd"}, + }, ], "schemaVersion": 2.1, } } else: - import pdb - - pdb.set_trace() return {}