Hide schema section for performance hub metrics and display caveats (#…

…943)

Add custom template for Performance Hub Metrics

Since performance hub metadata is not in prod yet and are subject to change,
I've not implemented Metrics as a distinct entity type that can be filtered
on. Instead,

1. expose a subtype attribute that determines what template to render
2. conditionally add extra caveats to metrics coming from
   performance-hub, to reflect their current state

This will need to be reviewed again if we decide to move forward with
performance hub metadata.

home/service/details.py

@@ -94,6 +94,8 @@ def __init__(self, urn: str):
 
         self.context = self._get_context()
 
+        self.template = self._get_template()
+
     def _get_context(self):
         split_datahub_url = urlsplit(
             os.getenv("CATALOGUE_URL", "https://test-catalogue.gov.uk")
@@ -112,6 +114,13 @@ def _get_context(self):
             ),
         }
 
+    def _get_template(self):
+        return (
+            "details_metric.html"
+            if "Metric" in self.table_metadata.subtypes
+            else "details_table.html"
+        )
+
     def has_lineage(self) -> bool:
         """
         Inspects the relationships property of the Table model to establish if a

home/views.py

@@ -34,8 +34,8 @@ def home_view(request):
 @cache_control(max_age=300, private=True)
 def details_view(request, result_type, urn):
     if result_type == "table":
-        context = dataset_details(urn)
-        return render(request, "details_table.html", context)
+        service = dataset_service(urn)
+        return render(request, service.template, service.context)
     if result_type == "database":
         context = database_details(urn)
         return render(request, "details_database.html", context)
@@ -58,15 +58,13 @@ def database_details(urn):
     return context
 
 
-def dataset_details(urn):
+def dataset_service(urn):
     try:
         service = DatasetDetailsService(urn)
     except EntityDoesNotExist:
         raise Http404("Asset does not exist")
 
-    context = service.context
-
-    return context
+    return service
 
 
 def chart_details(urn):

lib/datahub-client/data_platform_catalogue/client/datahub_client.py

@@ -3,6 +3,33 @@
 from importlib.resources import files
 from typing import Sequence
 
+from datahub.configuration.common import ConfigurationError
+from datahub.emitter import mce_builder
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
+from datahub.ingestion.source.common.subtypes import (
+    DatasetContainerSubTypes,
+    DatasetSubTypes,
+)
+from datahub.metadata import schema_classes
+from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance
+from datahub.metadata.schema_classes import (
+    ChangeTypeClass,
+    ContainerClass,
+    ContainerPropertiesClass,
+    DatasetPropertiesClass,
+    DomainPropertiesClass,
+    DomainsClass,
+    OtherSchemaClass,
+    OwnerClass,
+    OwnershipClass,
+    OwnershipTypeClass,
+    SchemaFieldClass,
+    SchemaFieldDataTypeClass,
+    SchemaMetadataClass,
+    SubTypesClass,
+)
+
 from data_platform_catalogue.client.exceptions import (
     AspectDoesNotExist,
     ConnectivityError,
@@ -20,6 +47,7 @@
     parse_owner,
     parse_properties,
     parse_relations,
+    parse_subtypes,
     parse_tags,
 )
 from data_platform_catalogue.client.search import SearchClient
@@ -43,32 +71,6 @@
     SearchResponse,
     SortOption,
 )
-from datahub.configuration.common import ConfigurationError
-from datahub.emitter import mce_builder
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
-from datahub.ingestion.source.common.subtypes import (
-    DatasetContainerSubTypes,
-    DatasetSubTypes,
-)
-from datahub.metadata import schema_classes
-from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance
-from datahub.metadata.schema_classes import (
-    ChangeTypeClass,
-    ContainerClass,
-    ContainerPropertiesClass,
-    DatasetPropertiesClass,
-    DomainPropertiesClass,
-    DomainsClass,
-    OtherSchemaClass,
-    OwnerClass,
-    OwnershipClass,
-    OwnershipTypeClass,
-    SchemaFieldClass,
-    SchemaFieldDataTypeClass,
-    SchemaMetadataClass,
-    SubTypesClass,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -286,6 +288,7 @@ def get_table_details(self, urn) -> Table:
             parent_relations_to_display = self.list_relations_to_display(
                 parent_relations
             )
+            subtypes = parse_subtypes(response)
 
             return Table(
                 urn=urn,
@@ -299,6 +302,7 @@ def get_table_details(self, urn) -> Table:
                     data_owner=owner,
                     data_stewards=[owner],
                 ),
+                subtypes=subtypes,
                 tags=tags,
                 glossary_terms=glossary_terms,
                 last_modified=modified,

lib/datahub-client/data_platform_catalogue/client/graphql/getDatasetDetails.graphql

@@ -3,6 +3,9 @@ query getDatasetDetails($urn: String!) {
     platform {
       name
     }
+    subTypes {
+      typeNames
+    }
     name # Deprecated - prefer properties.name
     domain {
       domain {
@@ -14,39 +17,35 @@ query getDatasetDetails($urn: String!) {
         }
       }
     }
-    downstream_lineage_relations: lineage (
-      input: {direction: DOWNSTREAM
-      start:0,
-      count:10}
+    downstream_lineage_relations: lineage(
+      input: { direction: DOWNSTREAM, start: 0, count: 10 }
     ) {
       total
-      relationships{
+      relationships {
         type
-        entity{
+        entity {
           urn
           ... on Dataset {
             name
-            properties{
+            properties {
               name
             }
           }
           type
         }
       }
     }
-    upstream_lineage_relations: lineage (
-      input: {direction: UPSTREAM
-      start:0,
-      count:10}
+    upstream_lineage_relations: lineage(
+      input: { direction: UPSTREAM, start: 0, count: 10 }
     ) {
       total
-      relationships{
+      relationships {
         type
-        entity{
+        entity {
           urn
           ... on Dataset {
             name
-            properties{
+            properties {
               name
             }
           }
@@ -74,7 +73,9 @@ query getDatasetDetails($urn: String!) {
             }
             tags {
               tags {
-                tag{urn}
+                tag {
+                  urn
+                }
               }
             }
             subTypes {

lib/datahub-client/data_platform_catalogue/client/graphql_helpers.py

@@ -359,5 +359,8 @@ def _make_user_email_from_urn(urn) -> str:
     return email
 
 
-def parse_refresh_period(entity: dict[str, Any]) -> str:
-    pass
+def parse_subtypes(entity: dict[str, Any]) -> list[str]:
+    subtypes = entity.get("subTypes", {})
+    if not subtypes:
+        return []
+    return subtypes.get("typeNames", [])

lib/datahub-client/data_platform_catalogue/entities.py

@@ -457,6 +457,15 @@ class Table(Entity):
         description="Unique identifier for the entity. Relates to Datahub's urn",
         examples=["urn:li:dataset:(urn:li:dataPlatform:redshift,public.table,DEV)"],
     )
+
+    subtypes: list[str] = Field(
+        description=(
+            "List of datahub subtypes. If a subtype is set, we still model the entity "
+            "as a table in Find MoJ data."
+        ),
+        examples=[["Metric"]],
+        default=[],
+    )
     column_details: list[Column] = Field(
         description=(
             "A list of objects which relate to columns in your data, each list item "

lib/datahub-client/tests/client/datahub/test_graphql_helpers.py

@@ -1,13 +1,15 @@
 from datetime import datetime, timezone
 
 import pytest
+
 from data_platform_catalogue.client.graphql_helpers import (
     _make_user_email_from_urn,
     parse_columns,
     parse_created_and_modified,
     parse_glossary_terms,
     parse_properties,
     parse_relations,
+    parse_subtypes,
     parse_tags,
 )
 from data_platform_catalogue.entities import (
@@ -531,3 +533,15 @@ def test_parse_glossary_terms():
 def test_make_user_email_from_urn(urn, expected_email):
     email = _make_user_email_from_urn(urn)
     assert email == expected_email
+
+
+def test_parse_subtypes():
+    result = parse_subtypes({"subTypes": {"typeNames": ["abc", "def"]}})
+
+    assert result == ["abc", "def"]
+
+
+def test_parse_missing_subtypes():
+    result = parse_subtypes({"subTypes": None})
+
+    assert result == []

templates/details_base.html

@@ -53,6 +53,8 @@ <h2 class="govuk-heading-s govuk-!-margin-top-3">
             {% translate "No description available." %}
           {% endif %}
         </div>
+        {% block extra_description %}
+        {% endblock extra_description %}
         {% block metadata_list %}
           <ul class="govuk-list govuk-body" id="metadata-property-list">
             {% if entity.created %}
@@ -93,7 +95,7 @@ <h2 class="govuk-heading-s govuk-!-margin-top-3">
       </div>
     </div>
     <div class="govuk-grid-column-one-third">
-      {% include "partial/contact_info.html" with data_owner=entity.governance.data_owner.display_name data_owner_email=entity.governance.data_owner.email slack_channel=entity.custom_properties.further_information access_requirements=entity.custom_properties.access_information.dc_access_requirements is_access_url=is_access_requirements_a_url%}
+      {% include "partial/contact_info.html" with data_owner=entity.governance.data_owner.display_name data_owner_email=entity.governance.data_owner.email slack_channel=entity.custom_properties.further_information access_requirements=entity.custom_properties.access_information.dc_access_requirements is_access_url=is_access_requirements_a_url platform=entity.platform %}
     </div>
   </div>
 

templates/details_metric.html

@@ -0,0 +1,9 @@
+{% extends "details_base.html" %}
+
+{% block extra_description %}
+{% if entity.platform.urn == 'performance-hub' %}
+<p>Data quality: the data is checked to be good enough to support the outcomes it is being used for. Data values
+  should be right, but there are other factors that help ensure data meets the needs of its users.</p>
+<p>Please remember that this data is for INTERNAL USE ONLY and not to be shared outside the organisation.</p>
+{% endif %}
+{% endblock extra_description %}

templates/partial/contact_info.html

@@ -37,6 +37,8 @@ <h2 class="govuk-heading-s govuk-!-margin-bottom-1">{% translate "IAO or Data Ow
   <p id="data_owner" class="govuk-body">
     {% if data_owner_email %}
       {{ data_owner_email|urlize }}
+    {% elif platform.urn == 'performance-hub' %}
+      No owner is listed as this data is undergoing a review of ownership.
     {% else %}
       {% blocktranslate %}Not provided - <a href="https://moj.enterprise.slack.com/archives/C06NPM2200N" class="govuk-link">contact the Data Catalogue team</a> about this data.{% endblocktranslate %}
     {% endif %}