CSV exports (#1020)

* refactor: remove wrapper functions for services

In the details view, expose the service object directly.

This will make it easier to add CSV representations.

* feat: add classes to generate table CSVs

* feat: add button for csv downloads

Use the secondary button style, as this is not a call to action, and
neither is it a link to another page.

* fixup: remove primary key from csv

We don't show this in the UI because it's not populated reliably, so
we shouldn't included it in the csv either.

* fixup: remove nullable from csv

home/service/details_csv.py

@@ -0,0 +1,65 @@
+from home.service.details import (
+    DashboardDetailsService,
+    DatabaseDetailsService,
+    DatasetDetailsService,
+)
+
+
+class DatasetDetailsCsvFormatter:
+    def __init__(self, details_service: DatasetDetailsService):
+        self.details_service = details_service
+
+    def data(self):
+        return [
+            (
+                column.name,
+                column.display_name,
+                column.type,
+                column.description,
+            )
+            for column in self.details_service.table_metadata.column_details
+        ]
+
+    def headers(self):
+        return [
+            "name",
+            "display_name",
+            "type",
+            "description",
+        ]
+
+
+class DatabaseDetailsCsvFormatter:
+    def __init__(self, details_service: DatabaseDetailsService):
+        self.details_service = details_service
+
+    def data(self):
+        return [
+            (
+                table.entity_ref.urn,
+                table.entity_ref.display_name,
+                table.description,
+            )
+            for table in self.details_service.entities_in_database
+        ]
+
+    def headers(self):
+        return [
+            "urn",
+            "display_name",
+            "description",
+        ]
+
+
+class DashboardDetailsCsvFormatter:
+    def __init__(self, details_service: DashboardDetailsService):
+        self.details_service = details_service
+
+    def data(self):
+        return [
+            (chart.entity_ref.urn, chart.entity_ref.display_name, chart.description)
+            for chart in self.details_service.children
+        ]
+
+    def headers(self):
+        return ["urn", "display_name", "description"]

home/urls.py

@@ -13,6 +13,11 @@
         views.metadata_specification_view,
         name="metadata_specification",
     ),
+    path(
+        "details/<str:result_type>/<str:urn>.csv",
+        views.details_view_csv,
+        name="details_csv",
+    ),
     path(
         "details/<str:result_type>/<str:urn>",
         views.details_view,

home/views.py

@@ -1,9 +1,10 @@
+import csv
 from urllib.parse import urlparse
 
 from data_platform_catalogue.client.exceptions import EntityDoesNotExist
 from data_platform_catalogue.search_types import DomainOption
 from django.conf import settings
-from django.http import Http404, HttpResponseBadRequest
+from django.http import Http404, HttpResponse, HttpResponseBadRequest
 from django.shortcuts import render
 from django.utils.translation import gettext as _
 from django.views.decorators.cache import cache_control
@@ -15,6 +16,11 @@
     DatabaseDetailsService,
     DatasetDetailsService,
 )
+from home.service.details_csv import (
+    DashboardDetailsCsvFormatter,
+    DatabaseDetailsCsvFormatter,
+    DatasetDetailsCsvFormatter,
+)
 from home.service.domain_fetcher import DomainFetcher
 from home.service.glossary import GlossaryService
 from home.service.metadata_specification import MetadataSpecificationService
@@ -33,60 +39,56 @@ def home_view(request):
 
 @cache_control(max_age=300, private=True)
 def details_view(request, result_type, urn):
-    if result_type == "table":
-        service = dataset_service(urn)
-        return render(request, service.template, service.context)
-    if result_type == "database":
-        context = database_details(urn)
-        return render(request, "details_database.html", context)
-    if result_type == "chart":
-        context = chart_details(urn)
-        return render(request, "details_chart.html", context)
-    if result_type == "dashboard":
-        context = dashboard_details(urn)
-        return render(request, "details_dashboard.html", context)
-
-
-def database_details(urn):
     try:
-        service = DatabaseDetailsService(urn)
-    except EntityDoesNotExist:
-        raise Http404("Asset does not exist")
-
-    context = service.context
-
-    return context
-
+        if result_type == "table":
+            service = DatasetDetailsService(urn)
+            template = service.template
+        elif result_type == "database":
+            service = DatabaseDetailsService(urn)
+            template = "details_database.html"
+        elif result_type == "chart":
+            service = ChartDetailsService(urn)
+            template = "details_chart.html"
+        elif result_type == "dashboard":
+            service = DashboardDetailsService(urn)
+            template = "details_dashboard.html"
+        else:
+            raise Http404("Invalid result type")
+
+        return render(request, template, service.context)
 
-def dataset_service(urn):
-    try:
-        service = DatasetDetailsService(urn)
     except EntityDoesNotExist:
-        raise Http404("Asset does not exist")
-
-    return service
+        raise Http404(f"{result_type} '{urn}' does not exist")
 
 
-def chart_details(urn):
-    try:
-        service = ChartDetailsService(urn)
-    except EntityDoesNotExist:
-        raise Http404("Asset does not exist")
-
-    context = service.context
-
-    return context
-
-
-def dashboard_details(urn):
-    try:
+@cache_control(max_age=300, private=True)
+def details_view_csv(request, result_type, urn) -> HttpResponse:
+    if result_type == "table":
+        service = DatasetDetailsService(urn)
+        csv_formatter = DatasetDetailsCsvFormatter(service)
+    elif result_type == "database":
+        service = DatabaseDetailsService(urn)
+        csv_formatter = DatabaseDetailsCsvFormatter(service)
+    elif result_type == "dashboard":
         service = DashboardDetailsService(urn)
-    except EntityDoesNotExist:
-        raise Http404("Asset does not exist")
+        csv_formatter = DashboardDetailsCsvFormatter(service)
+    else:
+        raise Http404("CSV not available")
+
+    # In case there are any quotes in the filename, remove them in order to
+    # not to break the header.
+    unsavoury_characters = str.maketrans({'"': ""})
+    filename = urn.translate(unsavoury_characters) + ".csv"
 
-    context = service.context
+    response = HttpResponse(
+        content_type="text/csv",
+        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+    )
+    writer = csv.writer(response)
+    writer.writerow(csv_formatter.headers())
+    writer.writerows(csv_formatter.data())
 
-    return context
+    return response
 
 
 @cache_control(max_age=60, private=True)

lib/datahub-client/data_platform_catalogue/entities.py

@@ -265,7 +265,7 @@ class EntitySummary(BaseModel):
     )
     description: str = Field(description="A description of the entity")
     entity_type: str = Field(
-        description="indicates the tpye of entity that is summarised"
+        description="indicates the type of entity that is summarised"
     )
     tags: list[TagRef] = Field(description="Any tags associated with the entity")
 

templates/details_dashboard.html

@@ -33,6 +33,11 @@
             {% endfor %}
           </tbody>
         </table>
+        <form action="{% url 'home:details_csv' result_type='dashboard' urn=entity.urn %}">
+          <button type="submit" class="govuk-button govuk-button--secondary" data-module="govuk-button">
+            {% translate 'Download chart descriptions (CSV format)' %}
+          </button>
+        </form>
       {% else %}
         <h2 class="govuk-heading-m">{% translate "Dashboard content" %}</h2>
         <p class="govuk-body">{% translate "This dashboard is missing chart information." %}</p>

templates/details_database.html

@@ -39,6 +39,11 @@
             {% endfor %}
           </tbody>
         </table>
+        <form action="{% url 'home:details_csv' result_type='database' urn=entity.urn %}">
+          <button type="submit" class="govuk-button govuk-button--secondary" data-module="govuk-button">
+            {% translate 'Download table descriptions (CSV format)' %}
+          </button>
+        </form>
       {% else %}
         <h2 class="govuk-heading-m">{% translate "Database content" %}</h2>
         <p class="govuk-body">{% translate "This database is missing table information." %}</p>

templates/details_table.html

@@ -40,6 +40,12 @@
             {% endfor %}
           </tbody>
         </table>
+
+        <form action="{% url 'home:details_csv' result_type='table' urn=entity.urn %}">
+          <button type="submit" class="govuk-button govuk-button--secondary" data-module="govuk-button">
+            {% translate 'Download table schema (CSV format)' %}
+          </button>
+        </form>
       {% else %}
         <h2 class="govuk-heading-m">{% translate "Table schema" %}</h2>
         <p class="govuk-body">{% translate "The schema for this table is not available." %}</p>

tests/home/service/test_details_csv.py

@@ -0,0 +1,116 @@
+from unittest.mock import MagicMock
+
+from data_platform_catalogue.entities import Column, EntityRef, EntitySummary
+
+from home.service.details import (
+    DashboardDetailsService,
+    DatabaseDetailsService,
+    DatasetDetailsService,
+)
+from home.service.details_csv import (
+    DashboardDetailsCsvFormatter,
+    DatabaseDetailsCsvFormatter,
+    DatasetDetailsCsvFormatter,
+)
+
+
+def test_dataset_details_csv_formatter(example_table):
+    details_service = MagicMock(spec=DatasetDetailsService)
+    columns = [
+        Column(
+            name="foo",
+            display_name="Foo",
+            type="string",
+            description="an example",
+            nullable=False,
+            is_primary_key=True,
+        ),
+        Column(
+            name="bar",
+            display_name="Bar",
+            type="integer",
+            description="another example",
+            nullable=True,
+            is_primary_key=False,
+        ),
+    ]
+    details_service.table_metadata = example_table
+    example_table.column_details = columns
+    csv_formatter = DatasetDetailsCsvFormatter(details_service)
+
+    assert csv_formatter.headers() == [
+        "name",
+        "display_name",
+        "type",
+        "description",
+    ]
+    assert csv_formatter.data() == [
+        (
+            "foo",
+            "Foo",
+            "string",
+            "an example",
+        ),
+        (
+            "bar",
+            "Bar",
+            "integer",
+            "another example",
+        ),
+    ]
+
+
+def test_database_details_csv_formatter(example_database):
+    tables = [
+        EntitySummary(
+            entity_ref=EntityRef(display_name="foo", urn="urn:foo"),
+            description="an example",
+            entity_type="Table",
+            tags=[],
+        ),
+        EntitySummary(
+            entity_ref=EntityRef(display_name="bar", urn="urn:bar"),
+            description="another example",
+            entity_type="Table",
+            tags=[],
+        ),
+    ]
+
+    details_service = MagicMock(spec=DatabaseDetailsService)
+    details_service.entities_in_database = tables
+
+    csv_formatter = DatabaseDetailsCsvFormatter(details_service)
+
+    assert csv_formatter.headers() == ["urn", "display_name", "description"]
+    assert csv_formatter.data() == [
+        ("urn:foo", "foo", "an example"),
+        ("urn:bar", "bar", "another example"),
+    ]
+
+
+def test_dashboard_details_csv_formatter(example_dashboard):
+    charts = [
+        EntitySummary(
+            entity_ref=EntityRef(display_name="foo", urn="urn:foo"),
+            description="an example",
+            entity_type="Chart",
+            tags=[],
+        ),
+        EntitySummary(
+            entity_ref=EntityRef(display_name="bar", urn="urn:bar"),
+            description="another example",
+            entity_type="Chart",
+            tags=[],
+        ),
+    ]
+
+    details_service = MagicMock(spec=DashboardDetailsService)
+    details_service.children = charts
+
+    csv_formatter = DashboardDetailsCsvFormatter(details_service)
+
+    assert csv_formatter.headers() == ["urn", "display_name", "description"]
+    assert csv_formatter.data() == [
+        ("urn:foo", "foo", "an example"),
+        ("urn:bar", "bar", "another example"),
+    ]