From 4ad9e23f2f5261e11dad0829d551267ceea5cd7f Mon Sep 17 00:00:00 2001 From: Masterchen09 <13187726+Masterchen09@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:38:39 +0200 Subject: [PATCH] adjustments after code review --- .../docs/sources/sac/sac_pre.md | 2 +- .../src/datahub/ingestion/source/sac/sac.py | 372 +++++++-------- .../src/datahub/utilities/logging_manager.py | 4 +- .../tests/integration/sac/test_sac.py | 438 +++++++++--------- 4 files changed, 383 insertions(+), 433 deletions(-) diff --git a/metadata-ingestion/docs/sources/sac/sac_pre.md b/metadata-ingestion/docs/sources/sac/sac_pre.md index 2dc66f4c1f2f65..e99e59eaaf55da 100644 --- a/metadata-ingestion/docs/sources/sac/sac_pre.md +++ b/metadata-ingestion/docs/sources/sac/sac_pre.md @@ -23,7 +23,7 @@ connection_mapping: env: PROD ``` -The key in the connection mapping is the technical name of the connection resp. its id. +The key in the connection mapping dictionary represents the name of the connection created in SAP Analytics Cloud. ## Concept mapping diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py index 20bfcac5ea8439..e2fbc75ef0851b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py @@ -66,7 +66,6 @@ BrowsePathsClass, BrowsePathsV2Class, ChangeAuditStampsClass, - ChangeTypeClass, DashboardInfoClass, DataPlatformInstanceClass, DatasetLineageTypeClass, @@ -196,24 +195,7 @@ def __init__(self, config: SACSourceConfig, ctx: PipelineContext): self.config = config self.report = SACSourceReport() - self.session = OAuth2Session( - client_id=self.config.client_id, - client_secret=self.config.client_secret.get_secret_value(), - token_endpoint=config.token_url, - token_endpoint_auth_method="client_secret_post", - grant_type="client_credentials", - ) - - self.session.register_compliance_hook( - "protected_request", _add_sap_sac_custom_auth_header - ) - self.session.fetch_token() - - self.client = pyodata.Client( - url=f"{self.config.tenant_url}/api/v1", - connection=self.session, - config=pyodata.v2.model.Config(retain_null=True), - ) + self.session, self.client = SACSource.get_sac_connection(self.config) def close(self) -> None: self.session.close() @@ -231,36 +213,20 @@ def test_connection(config_dict: dict) -> TestConnectionReport: try: config = SACSourceConfig.parse_obj(config_dict) - session = OAuth2Session( - client_id=config.client_id, - client_secret=config.client_secret.get_secret_value(), - token_endpoint=config.token_url, - token_endpoint_auth_method="client_secret_post", - grant_type="client_credentials", - ) - - session.register_compliance_hook( - "protected_request", _add_sap_sac_custom_auth_header - ) - session.fetch_token() - - response = session.get(url=f"{config.tenant_url}/api/v1/$metadata") - response.raise_for_status() + # when creating the pyodata.Client, the metadata is automatically parsed and validated + session, _ = SACSource.get_sac_connection(config) + # test the Data Import Service Service separately here, because it requires specific properties when configuring the OAuth client response = session.get(url=f"{config.tenant_url}/api/v1/dataimport/models") response.raise_for_status() - test_report.basic_connectivity = CapabilityReport(capable=True) + session.close() + test_report.basic_connectivity = CapabilityReport(capable=True) except Exception as e: - logger.error(f"Failed to test connection due to {e}", exc_info=e) - if test_report.basic_connectivity is None: - test_report.basic_connectivity = CapabilityReport( - capable=False, failure_reason=f"{e}" - ) - else: - test_report.internal_failure = True - test_report.internal_failure_reason = f"{e}" + test_report.basic_connectivity = CapabilityReport( + capable=False, failure_reason=f"{e}" + ) return test_report @@ -281,64 +247,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: resources = self.get_resources() for resource in resources: - dashboard_urn = make_dashboard_urn( - platform=self.platform, - name=resource.resource_id, - platform_instance=self.config.platform_instance, - ) - - if resource.ancestor_path: - mcp = MetadataChangeProposalWrapper( - entityType="dashboard", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dashboard_urn, - aspectName="browsePaths", - aspect=BrowsePathsClass( - paths=[ - f"/{self.platform}/{resource.ancestor_path}", - ], - ), - ) - - yield MetadataWorkUnit( - id=f"dashboard-browse-paths-{dashboard_urn}", mcp=mcp - ) - - mcp = MetadataChangeProposalWrapper( - entityType="dashboard", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dashboard_urn, - aspectName="browsePathsV2", - aspect=BrowsePathsV2Class( - path=[ - BrowsePathEntryClass(id=folder_name) - for folder_name in resource.ancestor_path.split("/") - ], - ), - ) - - yield MetadataWorkUnit( - id=f"dashboard-browse-paths-v2-{dashboard_urn}", mcp=mcp - ) - - if self.config.platform_instance is not None: - mcp = MetadataChangeProposalWrapper( - entityType="dashboard", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dashboard_urn, - aspectName="dataPlatformInstance", - aspect=DataPlatformInstanceClass( - platform=make_data_platform_urn(self.platform), - instance=make_dataplatform_instance_urn( - self.platform, self.config.platform_instance - ), - ), - ) - - yield MetadataWorkUnit( - id=f"dashboard-data-platform-instance-{dashboard_urn}", mcp=mcp - ) - datasets = [] for resource_model in resource.resource_models: @@ -359,73 +267,110 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.get_model_workunits(dataset_urn, resource_model) - mcp = MetadataChangeProposalWrapper( - entityType="dashboard", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dashboard_urn, - aspectName="dashboardInfo", - aspect=DashboardInfoClass( - title=resource.name, - description=resource.description, - lastModified=ChangeAuditStampsClass( - created=AuditStampClass( - time=round(resource.created_time.timestamp() * 1000), - actor=make_user_urn(resource.created_by) - if resource.created_by - else "urn:li:corpuser:unknown", - ), - lastModified=AuditStampClass( - time=round(resource.modified_time.timestamp() * 1000), - actor=make_user_urn(resource.modified_by) - if resource.modified_by - else "urn:li:corpuser:unknown", - ), - ), - customProperties={ - "resourceType": resource.resource_type, - "resourceSubtype": resource.resource_subtype, - "storyId": resource.story_id, - "isMobile": str(resource.is_mobile), - }, - datasets=sorted(datasets) if datasets else None, - externalUrl=f"{self.config.tenant_url}{resource.open_url}", + yield from self.get_resource_workunits(resource, datasets) + + def get_report(self) -> SACSourceReport: + return self.report + + def get_resource_workunits( + self, resource: Resource, datasets: List[str] + ) -> Iterable[MetadataWorkUnit]: + dashboard_urn = make_dashboard_urn( + platform=self.platform, + name=resource.resource_id, + platform_instance=self.config.platform_instance, + ) + + if resource.ancestor_path: + mcp = MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, + aspect=BrowsePathsClass( + paths=[ + f"/{self.platform}/{resource.ancestor_path}", + ], + ), + ) + + yield mcp.as_workunit() + + mcp = MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, + aspect=BrowsePathsV2Class( + path=[ + BrowsePathEntryClass(id=folder_name) + for folder_name in resource.ancestor_path.split("/") + ], + ), + ) + + yield mcp.as_workunit() + + if self.config.platform_instance is not None: + mcp = MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, + aspect=DataPlatformInstanceClass( + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn( + self.platform, self.config.platform_instance ), - ) + ), + ) - yield MetadataWorkUnit(id=f"dashboard-info-{dashboard_urn}", mcp=mcp) + yield mcp.as_workunit() - type_name: Optional[str] = None - if resource.resource_subtype == "": - type_name = BIAssetSubTypes.SAC_STORY - elif resource.resource_subtype == "APPLICATION": - type_name = BIAssetSubTypes.SAC_APPLICATION + mcp = MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, + aspect=DashboardInfoClass( + title=resource.name, + description=resource.description, + lastModified=ChangeAuditStampsClass( + created=AuditStampClass( + time=round(resource.created_time.timestamp() * 1000), + actor=make_user_urn(resource.created_by) + if resource.created_by + else "urn:li:corpuser:unknown", + ), + lastModified=AuditStampClass( + time=round(resource.modified_time.timestamp() * 1000), + actor=make_user_urn(resource.modified_by) + if resource.modified_by + else "urn:li:corpuser:unknown", + ), + ), + customProperties={ + "resourceType": resource.resource_type, + "resourceSubtype": resource.resource_subtype, + "storyId": resource.story_id, + "isMobile": str(resource.is_mobile), + }, + datasets=sorted(datasets) if datasets else None, + externalUrl=f"{self.config.tenant_url}{resource.open_url}", + ), + ) - if type_name: - mcp = MetadataChangeProposalWrapper( - entityType="dashboard", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dashboard_urn, - aspectName="subTypes", - aspect=SubTypesClass( - typeNames=[type_name], - ), - ) + yield mcp.as_workunit() - yield MetadataWorkUnit( - id=f"dashboard-subtype-{dashboard_urn}", mcp=mcp - ) + type_name: Optional[str] = None + if resource.resource_subtype == "": + type_name = BIAssetSubTypes.SAC_STORY + elif resource.resource_subtype == "APPLICATION": + type_name = BIAssetSubTypes.SAC_APPLICATION - def get_report(self) -> SACSourceReport: - return self.report + if type_name: + mcp = MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, + aspect=SubTypesClass( + typeNames=[type_name], + ), + ) + + yield mcp.as_workunit() def get_model_workunits( self, dataset_urn: str, model: ResourceModel ) -> Iterable[MetadataWorkUnit]: mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="datasetProperties", aspect=DatasetPropertiesClass( name=model.name, description=model.description, @@ -438,7 +383,7 @@ def get_model_workunits( ), ) - yield MetadataWorkUnit(id=f"dataset-properties-{dataset_urn}", mcp=mcp) + yield mcp.as_workunit() if model.is_import: primary_fields: List[str] = [] @@ -446,22 +391,11 @@ def get_model_workunits( columns = self.get_import_data_model_columns(model_id=model.model_id) for column in columns: - native_data_type = column.data_type - if column.data_type == "decimal": - native_data_type = ( - f"{column.data_type}({column.precision}, {column.scale})" - ) - elif column.data_type == "int32": - native_data_type = f"{column.data_type}({column.precision})" - elif column.max_length is not None: - native_data_type = f"{column.data_type}({column.max_length})" schema_field = SchemaFieldClass( fieldPath=column.name, - type=self.get_schema_field_data_type( - column.property_type, column.data_type - ), - nativeDataType=native_data_type, + type=self.get_schema_field_data_type(column), + nativeDataType=self.get_schema_field_native_data_type(column), description=column.description, isPartOfKey=column.is_key, ) @@ -472,10 +406,7 @@ def get_model_workunits( primary_fields.append(column.name) mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="schemaMetadata", aspect=SchemaMetadataClass( schemaName=model.model_id, platform=make_data_platform_urn(self.platform), @@ -487,9 +418,7 @@ def get_model_workunits( ), ) - yield MetadataWorkUnit( - id=f"dataset-upstream-lineage-{dataset_urn}", mcp=mcp - ) + yield mcp.as_workunit() if model.system_type in ("BW", "HANA") and model.external_id is not None: upstream_dataset_name: Optional[str] = None @@ -525,6 +454,10 @@ def get_model_workunits( platform_instance = model.connection_id env = DEFAULT_ENV + logger.info( + f"No connection mapping found for connection with id {model.connection_id}, connection id will be used as platform instance" + ) + upstream_dataset_urn = make_dataset_urn_with_platform_instance( platform=platform, name=upstream_dataset_name, @@ -534,26 +467,16 @@ def get_model_workunits( if upstream_dataset_urn not in self.ingested_upstream_dataset_keys: mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=upstream_dataset_urn, - aspectName="datasetKey", aspect=dataset_urn_to_key(upstream_dataset_urn), ) - yield MetadataWorkUnit( - id=f"dataset-key-{upstream_dataset_urn}", - mcp=mcp, - is_primary_source=False, - ) + yield mcp.as_workunit(is_primary_source=False) self.ingested_upstream_dataset_keys.add(upstream_dataset_urn) mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="upstreamLineage", aspect=UpstreamLineageClass( upstreams=[ UpstreamClass( @@ -564,37 +487,26 @@ def get_model_workunits( ), ) - yield MetadataWorkUnit( - id=f"dataset-upstream-lineage-{dataset_urn}", mcp=mcp - ) + yield mcp.as_workunit() else: - logger.warning( - f"Unknown upstream dataset for model with id {model.namespace}:{model.model_id} and external id {model.external_id}" - ) self.report.report_warning( "unknown-upstream-dataset", f"Unknown upstream dataset for model with id {model.namespace}:{model.model_id} and external id {model.external_id}", ) elif model.system_type is not None: - logger.warning( - f"Unknown system type {model.system_type} for model with id {model.namespace}:{model.model_id} and external id {model.external_id}" - ) self.report.report_warning( "unknown-system-type", f"Unknown system type {model.system_type} for model with id {model.namespace}:{model.model_id} and external id {model.external_id}", ) mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="status", aspect=StatusClass( removed=False, ), ) - yield MetadataWorkUnit(id=f"dataset-status-{dataset_urn}", mcp=mcp) + yield mcp.as_workunit() if model.external_id and model.connection_id and model.system_type: type_name = DatasetSubTypes.SAC_LIVE_DATA_MODEL @@ -604,32 +516,49 @@ def get_model_workunits( type_name = DatasetSubTypes.SAC_MODEL mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="subTypes", aspect=SubTypesClass( typeNames=[type_name], ), ) - yield MetadataWorkUnit(id=f"dataset-subtype-{dataset_urn}", mcp=mcp) + yield mcp.as_workunit() mcp = MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="dataPlatformInstance", aspect=DataPlatformInstanceClass( platform=make_data_platform_urn(self.platform), instance=self.config.platform_instance, ), ) - yield MetadataWorkUnit( - id=f"dataset-data-platform-instance-{dataset_urn}", mcp=mcp + yield mcp.as_workunit() + + @staticmethod + def get_sac_connection( + config: SACSourceConfig, + ) -> Tuple[OAuth2Session, pyodata.Client]: + session = OAuth2Session( + client_id=config.client_id, + client_secret=config.client_secret.get_secret_value(), + token_endpoint=config.token_url, + token_endpoint_auth_method="client_secret_post", + grant_type="client_credentials", + ) + + session.register_compliance_hook( + "protected_request", _add_sap_sac_custom_auth_header + ) + session.fetch_token() + + client = pyodata.Client( + url=f"{config.tenant_url}/api/v1", + connection=session, + config=pyodata.v2.model.Config(retain_null=True), ) + return session, client + def get_resources(self) -> Iterable[Resource]: import_data_model_ids = self.get_import_data_model_ids() @@ -787,19 +716,34 @@ def get_view_name(self, schema: str, namespace: Optional[str], view: str) -> str return f"{schema}.{view}" def get_schema_field_data_type( - self, property_type: str, data_type: str + self, column: ImportDataModelColumn ) -> SchemaFieldDataTypeClass: - if property_type == "DATE": + if column.property_type == "DATE": return SchemaFieldDataTypeClass(type=DateTypeClass()) else: - if data_type == "string": + if column.data_type == "string": return SchemaFieldDataTypeClass(type=StringTypeClass()) - elif data_type in ("decimal", "int32"): + elif column.data_type in ("decimal", "int32"): return SchemaFieldDataTypeClass(type=NumberTypeClass()) else: - logger.warning(f"Unknown data type {data_type} found") + self.report.report_warning( + "unknown-data-type", + f"Unknown data type {column.data_type} found", + ) + return SchemaFieldDataTypeClass(type=NullTypeClass()) + def get_schema_field_native_data_type(self, column: ImportDataModelColumn) -> str: + native_data_type = column.data_type + if column.data_type == "decimal": + native_data_type = f"{column.data_type}({column.precision}, {column.scale})" + elif column.data_type == "int32": + native_data_type = f"{column.data_type}({column.precision})" + elif column.max_length is not None: + native_data_type = f"{column.data_type}({column.max_length})" + + return native_data_type + def _add_sap_sac_custom_auth_header( url: str, headers: Dict[str, str], body: Any diff --git a/metadata-ingestion/src/datahub/utilities/logging_manager.py b/metadata-ingestion/src/datahub/utilities/logging_manager.py index 4e860d12a52dc3..75e0575c0f18b5 100644 --- a/metadata-ingestion/src/datahub/utilities/logging_manager.py +++ b/metadata-ingestion/src/datahub/utilities/logging_manager.py @@ -278,6 +278,4 @@ def configure_logging(debug: bool, log_file: Optional[str] = None) -> Iterator[N logging.getLogger("snowflake").setLevel(level=logging.WARNING) # logging.getLogger("botocore").setLevel(logging.INFO) # logging.getLogger("google").setLevel(logging.INFO) -logging.getLogger("pyodata.client").setLevel(logging.WARNING) -logging.getLogger("pyodata.model").setLevel(logging.WARNING) -logging.getLogger("pyodata.service").setLevel(logging.WARNING) +logging.getLogger("pyodata").setLevel(logging.WARNING) diff --git a/metadata-ingestion/tests/integration/sac/test_sac.py b/metadata-ingestion/tests/integration/sac/test_sac.py index 067bd5d7fc4215..2b6ca81700712e 100644 --- a/metadata-ingestion/tests/integration/sac/test_sac.py +++ b/metadata-ingestion/tests/integration/sac/test_sac.py @@ -21,28 +21,6 @@ def test_sac( requests_mock, mock_time, ): - def match_token_url(request, context): - form = parse_qs(request.text, strict_parsing=True) - - assert "grant_type" in form - assert len(form["grant_type"]) == 1 - assert form["grant_type"][0] == "client_credentials" - - assert "client_id" in form - assert len(form["client_id"]) == 1 - assert form["client_id"][0] == MOCK_CLIENT_ID - - assert "client_secret" in form - assert len(form["client_secret"]) == 1 - assert form["client_secret"][0] == MOCK_CLIENT_SECRET - - json = { - "access_token": MOCK_ACCESS_TOKEN, - "expires_in": 3599, - } - - return json - requests_mock.post( MOCK_TOKEN_URL, json=match_token_url, @@ -52,119 +30,16 @@ def match_token_url(request, context): with open(f"{test_resources_dir}/metadata.xml", mode="rb") as f: content = f.read() - - def match_metadata(request, context): - _check_authorization(request.headers) - - context.headers["content-type"] = "application/xml" - - return content - - requests_mock.get(f"{MOCK_TENANT_URL}/api/v1/$metadata", content=match_metadata) - - def match_resources(request, context): - _check_authorization(request.headers) - - json = { - "d": { - "results": [ - { - "__metadata": { - "type": "sap.fpa.services.search.internal.ResourcesType", - "uri": "/api/v1/Resources('LXTH4JCE36EOYLU41PIINLYPU9XRYM26')", - }, - "name": "Name of the story", - "description": "Description of the story", - "resourceId": "LXTH4JCE36EOYLU41PIINLYPU9XRYM26", - "resourceType": "STORY", - "resourceSubtype": "", - "storyId": "STORY:t.4:LXTH4JCE36EOYLU41PIINLYPU9XRYM26", - "createdTime": "/Date(1667544309783)/", - "createdBy": "JOHN_DOE", - "modifiedBy": "JOHN_DOE", - "modifiedTime": "/Date(1673067981272)/", - "isMobile": 0, - "openURL": "/sap/fpa/ui/tenants/3c44c/bo/story/LXTH4JCE36EOYLU41PIINLYPU9XRYM26", - "ancestorPath": '["Public","Folder 1","Folder 2"]', - }, - { - "__metadata": { - "type": "sap.fpa.services.search.internal.ResourcesType", - "uri": "/api/v1/Resources('EOYLU41PIILXTH4JCE36NLYPU9XRYM26')", - }, - "name": "Name of the application", - "description": "Description of the application", - "resourceId": "EOYLU41PIILXTH4JCE36NLYPU9XRYM26", - "resourceType": "STORY", - "resourceSubtype": "APPLICATION", - "storyId": "STORY:t.4:EOYLU41PIILXTH4JCE36NLYPU9XRYM26", - "createdTime": "/Date(1673279404272)/", - "createdBy": "SYSTEM", - "modifiedBy": "$DELETED_USER$", - "modifiedTime": "/Date(1673279414272)/", - "isMobile": 0, - "openURL": "/sap/fpa/ui/tenants/3c44c/bo/story/EOYLU41PIILXTH4JCE36NLYPU9XRYM26", - "ancestorPath": '["Public","Folder 1","Folder 2"]', - }, - ], - }, - } - - return json + requests_mock.get( + f"{MOCK_TENANT_URL}/api/v1/$metadata", + content=partial(match_metadata, content=content), + ) requests_mock.get( f"{MOCK_TENANT_URL}/api/v1/Resources?$format=json&$filter=isTemplate eq 0 and isSample eq 0 and isPublic eq 1 and ((resourceType eq 'STORY' and resourceSubtype eq '') or (resourceType eq 'STORY' and resourceSubtype eq 'APPLICATION'))&$select=resourceId,resourceType,resourceSubtype,storyId,name,description,createdTime,createdBy,modifiedBy,modifiedTime,openURL,ancestorPath,isMobile", json=match_resources, ) - def match_resource(request, context, resource_id): - _check_authorization(request.headers) - - json = { - "d": { - "results": [ - { - "__metadata": { - "type": "sap.fpa.services.search.internal.ModelsType", - "uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.ANL8Q577BA2F73KU3VELDXGWZK%3AANL8Q577BA2F73KU3VELDXGWZK')", - }, - "modelId": "t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK", - "name": "Name of the first model (BW)", - "description": "Description of the first model which has a connection to a BW query", - "externalId": "query:[][][QUERY_TECHNICAL_NAME]", - "connectionId": "BW", - "systemType": "BW", - }, - { - "__metadata": { - "type": "sap.fpa.services.search.internal.ModelsType", - "uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.K73U3VELDXGWZKANL8Q577BA2F%3AK73U3VELDXGWZKANL8Q577BA2F')", - }, - "modelId": "t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F", - "name": "Name of the second model (HANA)", - "description": "Description of the second model which has a connection to a HANA view", - "externalId": "view:[SCHEMA][NAMESPACE.SCHEMA][VIEW]", - "connectionId": "HANA", - "systemType": "HANA", - }, - { - "__metadata": { - "type": "sap.fpa.services.search.internal.ModelsType", - "uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.DXGWZKANLK73U3VEL8Q577BA2F%3ADXGWZKANLK73U3VEL8Q577BA2F')", - }, - "modelId": "t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F", - "name": "Name of the third model (Import)", - "description": "Description of the third model which was imported", - "externalId": "", - "connectionId": "", - "systemType": None, - }, - ], - }, - } - - return json - requests_mock.get( f"{MOCK_TENANT_URL}/api/v1/Resources%28%27LXTH4JCE36EOYLU41PIINLYPU9XRYM26%27%29/resourceModels?$format=json&$select=modelId,name,description,externalId,connectionId,systemType", json=partial(match_resource, resource_id="LXTH4JCE36EOYLU41PIINLYPU9XRYM26"), @@ -175,96 +50,11 @@ def match_resource(request, context, resource_id): json=partial(match_resource, resource_id="EOYLU41PIILXTH4JCE36NLYPU9XRYM26"), ) - def match_models(request, context): - _check_authorization(request.headers) - - json = { - "models": [ - { - "modelID": "DXGWZKANLK73U3VEL8Q577BA2F", - "modelName": "Name of the third model (Import)", - "modelDescription": "Description of the third model which was imported", - "modelURL": f"{MOCK_TENANT_URL}/api/v1/dataimport/models/DXGWZKANLK73U3VEL8Q577BA2F", - }, - ], - } - - return json - requests_mock.get( f"{MOCK_TENANT_URL}/api/v1/dataimport/models", json=match_models, ) - def match_model_metadata(request, context): - _check_authorization(request.headers) - - json = { - "factData": { - "keys": [ - "Account", - "FIELD1", - "FIELD2", - "FIELD3", - "Version", - ], - "columns": [ - { - "columnName": "Account", - "columnDataType": "string", - "maxLength": 256, - "isKey": True, - "propertyType": "PROPERTY", - "descriptionName": "Account", - }, - { - "columnName": "FIELD1", - "columnDataType": "string", - "maxLength": 256, - "isKey": True, - "propertyType": "PROPERTY", - "descriptionName": "FIELD1", - }, - { - "columnName": "FIELD2", - "columnDataType": "string", - "maxLength": 256, - "isKey": True, - "propertyType": "PROPERTY", - "descriptionName": "FIELD2", - }, - { - "columnName": "FIELD3", - "columnDataType": "string", - "maxLength": 256, - "isKey": True, - "propertyType": "DATE", - "descriptionName": "FIELD3", - }, - { - "columnName": "Version", - "columnDataType": "string", - "maxLength": 300, - "isKey": True, - "propertyType": "PROPERTY", - "descriptionName": "Version", - }, - { - "columnName": "SignedData", - "columnDataType": "decimal", - "maxLength": 32, - "precision": 31, - "scale": 7, - "isKey": False, - "propertyType": "PROPERTY", - "descriptionName": "SignedData", - }, - ], - }, - } - - return json - requests_mock.get( f"{MOCK_TENANT_URL}/api/v1/dataimport/models/DXGWZKANLK73U3VEL8Q577BA2F/metadata", json=match_model_metadata, @@ -300,9 +90,227 @@ def match_model_metadata(request, context): ) -def _check_authorization(headers: Dict) -> None: +def match_token_url(request, context): + form = parse_qs(request.text, strict_parsing=True) + + assert "grant_type" in form + assert len(form["grant_type"]) == 1 + assert form["grant_type"][0] == "client_credentials" + + assert "client_id" in form + assert len(form["client_id"]) == 1 + assert form["client_id"][0] == MOCK_CLIENT_ID + + assert "client_secret" in form + assert len(form["client_secret"]) == 1 + assert form["client_secret"][0] == MOCK_CLIENT_SECRET + + json = { + "access_token": MOCK_ACCESS_TOKEN, + "expires_in": 3599, + } + + return json + + +def check_authorization(headers: Dict[str, str]) -> None: assert "Authorization" in headers assert headers["Authorization"] == f"Bearer {MOCK_ACCESS_TOKEN}" assert "x-sap-sac-custom-auth" in headers assert headers["x-sap-sac-custom-auth"] == "true" + + +def match_metadata(request, context, content): + check_authorization(request.headers) + + context.headers["content-type"] = "application/xml" + + return content + + +def match_resources(request, context): + check_authorization(request.headers) + + json = { + "d": { + "results": [ + { + "__metadata": { + "type": "sap.fpa.services.search.internal.ResourcesType", + "uri": "/api/v1/Resources('LXTH4JCE36EOYLU41PIINLYPU9XRYM26')", + }, + "name": "Name of the story", + "description": "Description of the story", + "resourceId": "LXTH4JCE36EOYLU41PIINLYPU9XRYM26", + "resourceType": "STORY", + "resourceSubtype": "", + "storyId": "STORY:t.4:LXTH4JCE36EOYLU41PIINLYPU9XRYM26", + "createdTime": "/Date(1667544309783)/", + "createdBy": "JOHN_DOE", + "modifiedBy": "JOHN_DOE", + "modifiedTime": "/Date(1673067981272)/", + "isMobile": 0, + "openURL": "/sap/fpa/ui/tenants/3c44c/bo/story/LXTH4JCE36EOYLU41PIINLYPU9XRYM26", + "ancestorPath": '["Public","Folder 1","Folder 2"]', + }, + { + "__metadata": { + "type": "sap.fpa.services.search.internal.ResourcesType", + "uri": "/api/v1/Resources('EOYLU41PIILXTH4JCE36NLYPU9XRYM26')", + }, + "name": "Name of the application", + "description": "Description of the application", + "resourceId": "EOYLU41PIILXTH4JCE36NLYPU9XRYM26", + "resourceType": "STORY", + "resourceSubtype": "APPLICATION", + "storyId": "STORY:t.4:EOYLU41PIILXTH4JCE36NLYPU9XRYM26", + "createdTime": "/Date(1673279404272)/", + "createdBy": "SYSTEM", + "modifiedBy": "$DELETED_USER$", + "modifiedTime": "/Date(1673279414272)/", + "isMobile": 0, + "openURL": "/sap/fpa/ui/tenants/3c44c/bo/story/EOYLU41PIILXTH4JCE36NLYPU9XRYM26", + "ancestorPath": '["Public","Folder 1","Folder 2"]', + }, + ], + }, + } + + return json + + +def match_resource(request, context, resource_id): + check_authorization(request.headers) + + json = { + "d": { + "results": [ + { + "__metadata": { + "type": "sap.fpa.services.search.internal.ModelsType", + "uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.ANL8Q577BA2F73KU3VELDXGWZK%3AANL8Q577BA2F73KU3VELDXGWZK')", + }, + "modelId": "t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK", + "name": "Name of the first model (BW)", + "description": "Description of the first model which has a connection to a BW query", + "externalId": "query:[][][QUERY_TECHNICAL_NAME]", + "connectionId": "BW", + "systemType": "BW", + }, + { + "__metadata": { + "type": "sap.fpa.services.search.internal.ModelsType", + "uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.K73U3VELDXGWZKANL8Q577BA2F%3AK73U3VELDXGWZKANL8Q577BA2F')", + }, + "modelId": "t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F", + "name": "Name of the second model (HANA)", + "description": "Description of the second model which has a connection to a HANA view", + "externalId": "view:[SCHEMA][NAMESPACE.SCHEMA][VIEW]", + "connectionId": "HANA", + "systemType": "HANA", + }, + { + "__metadata": { + "type": "sap.fpa.services.search.internal.ModelsType", + "uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.DXGWZKANLK73U3VEL8Q577BA2F%3ADXGWZKANLK73U3VEL8Q577BA2F')", + }, + "modelId": "t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F", + "name": "Name of the third model (Import)", + "description": "Description of the third model which was imported", + "externalId": "", + "connectionId": "", + "systemType": None, + }, + ], + }, + } + + return json + + +def match_models(request, context): + check_authorization(request.headers) + + json = { + "models": [ + { + "modelID": "DXGWZKANLK73U3VEL8Q577BA2F", + "modelName": "Name of the third model (Import)", + "modelDescription": "Description of the third model which was imported", + "modelURL": f"{MOCK_TENANT_URL}/api/v1/dataimport/models/DXGWZKANLK73U3VEL8Q577BA2F", + }, + ], + } + + return json + + +def match_model_metadata(request, context): + check_authorization(request.headers) + + json = { + "factData": { + "keys": [ + "Account", + "FIELD1", + "FIELD2", + "FIELD3", + "Version", + ], + "columns": [ + { + "columnName": "Account", + "columnDataType": "string", + "maxLength": 256, + "isKey": True, + "propertyType": "PROPERTY", + "descriptionName": "Account", + }, + { + "columnName": "FIELD1", + "columnDataType": "string", + "maxLength": 256, + "isKey": True, + "propertyType": "PROPERTY", + "descriptionName": "FIELD1", + }, + { + "columnName": "FIELD2", + "columnDataType": "string", + "maxLength": 256, + "isKey": True, + "propertyType": "PROPERTY", + "descriptionName": "FIELD2", + }, + { + "columnName": "FIELD3", + "columnDataType": "string", + "maxLength": 256, + "isKey": True, + "propertyType": "DATE", + "descriptionName": "FIELD3", + }, + { + "columnName": "Version", + "columnDataType": "string", + "maxLength": 300, + "isKey": True, + "propertyType": "PROPERTY", + "descriptionName": "Version", + }, + { + "columnName": "SignedData", + "columnDataType": "decimal", + "maxLength": 32, + "precision": 31, + "scale": 7, + "isKey": False, + "propertyType": "PROPERTY", + "descriptionName": "SignedData", + }, + ], + }, + } + + return json