From 176253a207a082bcbeb11d0546d316d90572f18c Mon Sep 17 00:00:00 2001 From: John Tordoff <> Date: Fri, 23 Aug 2024 14:53:39 -0400 Subject: [PATCH] add institutional dashboard summary. --- api/institutions/renderers.py | 4 +- api/metrics/views.py | 1 + osf/metrics/reporters/__init__.py | 2 + .../institution_dashboard_summary.py | 77 ++++++++++++++++ osf/metrics/reports.py | 87 +++++++++++++++++++ 5 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 osf/metrics/reporters/institution_dashboard_summary.py diff --git a/api/institutions/renderers.py b/api/institutions/renderers.py index 353fcdfcce0a..08197841ac91 100644 --- a/api/institutions/renderers.py +++ b/api/institutions/renderers.py @@ -6,12 +6,14 @@ class MetricsCSVRenderer(CSVRenderer): CSVRenderer with updated render method to export `data` dictionary of API Response to CSV """ - def render(self, data, media_type=None, renderer_context={}, writer_opts=None): + def render(self, data, media_type=None, renderer_context=None, writer_opts=None): """ Overwrites CSVRenderer.render() to create a CSV with the data dictionary instead of the entire API response. This is necessary for results to be separated into different rows. """ + if not renderer_context: + renderer_context = {} data = data.get('data') return super().render(data, media_type=media_type, renderer_context=renderer_context, writer_opts=writer_opts) diff --git a/api/metrics/views.py b/api/metrics/views.py index 51556ddc89cb..4721942285c5 100644 --- a/api/metrics/views.py +++ b/api/metrics/views.py @@ -266,6 +266,7 @@ def get(self, request, *args, **kwargs): VIEWABLE_REPORTS = { 'download_count': reports.DownloadCountReport, 'institution_summary': reports.InstitutionSummaryReport, + 'institution_dashboard_summary': reports.InstitutionDashboardSummaryReport, 'node_summary': reports.NodeSummaryReport, 'osfstorage_file_count': reports.OsfstorageFileCountReport, 'preprint_summary': reports.PreprintSummaryReport, diff --git a/osf/metrics/reporters/__init__.py b/osf/metrics/reporters/__init__.py index b7a0f5e53635..f0da811e641e 100644 --- a/osf/metrics/reporters/__init__.py +++ b/osf/metrics/reporters/__init__.py @@ -2,6 +2,7 @@ from .storage_addon_usage import StorageAddonUsageReporter from .download_count import DownloadCountReporter from .institution_summary import InstitutionSummaryReporter +from .institution_dashboard_summary import InstitutionDashboardSummaryReport from .new_user_domain import NewUserDomainReporter from .node_count import NodeCountReporter from .osfstorage_file_count import OsfstorageFileCountReporter @@ -14,6 +15,7 @@ # ActiveUserReporter, DownloadCountReporter, InstitutionSummaryReporter, + InstitutionDashboardSummaryReport, NewUserDomainReporter, NodeCountReporter, OsfstorageFileCountReporter, diff --git a/osf/metrics/reporters/institution_dashboard_summary.py b/osf/metrics/reporters/institution_dashboard_summary.py new file mode 100644 index 000000000000..0847c1a30986 --- /dev/null +++ b/osf/metrics/reporters/institution_dashboard_summary.py @@ -0,0 +1,77 @@ +import logging + +from django.db.models import Q + +from osf.metrics.reports import ( + InstitutionDashboardSummaryReport, + RunningTotal, + NodeRunningTotals, + RegistrationRunningTotals, + FileRunningTotals, + DataRunningTotals +) +from osf.models import Institution +from ._base import DailyReporter + + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +class InstitutionDashboardSummaryReporter(DailyReporter): + def report(self, date): + institutions = Institution.objects.all() + reports = [] + + daily_query = Q(created__date=date) + public_query = Q(is_public=True) + private_query = Q(is_public=False) + + embargo_v2_query = Q(root__embargo__end_date__date__gt=date) + + for institution in institutions: + node_qs = institution.nodes.filter( + deleted__isnull=True, + created__date__lte=date, + ).exclude(type='osf.registration') + registration_qs = institution.nodes.filter( + deleted__isnull=True, + created__date__lte=date, + type='osf.registration', + ) + + report = InstitutionDashboardSummaryReport( + report_date=date, + institution_id=institution._id, + institution_name=institution.name, + users=RunningTotal( + total=institution.get_institution_users().filter(is_active=True).count(), + total_daily=institution.get_institution_users().filter(date_confirmed__date=date).count(), + ), + # Projects use get_roots to remove children + projects=NodeRunningTotals( + total=node_qs.get_roots().count(), + public=node_qs.filter(public_query).get_roots().count(), + private=node_qs.filter(private_query).get_roots().count(), + + total_daily=node_qs.filter(daily_query).get_roots().count(), + public_daily=node_qs.filter(public_query & daily_query).get_roots().count(), + private_daily=node_qs.filter(private_query & daily_query).get_roots().count(), + ), + registerations=RegistrationRunningTotals( + total=registration_qs.count(), + public=registration_qs.filter(public_query).count(), + embargoed=registration_qs.filter(private_query).count(), + embargoed_v2=registration_qs.filter(private_query & embargo_v2_query).count(), + + total_daily=registration_qs.filter(daily_query).count(), + public_daily=registration_qs.filter(public_query & daily_query).count(), + embargoed_daily=registration_qs.filter(private_query & daily_query).count(), + embargoed_v2_daily=registration_qs.filter(private_query & daily_query & embargo_v2_query).count(), + ), + files=FileRunningTotals(), + data=DataRunningTotals(), + ) + + reports.append(report) + return reports \ No newline at end of file diff --git a/osf/metrics/reports.py b/osf/metrics/reports.py index 609e79fc324b..426da6ea92c7 100644 --- a/osf/metrics/reports.py +++ b/osf/metrics/reports.py @@ -94,6 +94,7 @@ class RunningTotal(InnerDoc): total = metrics.Integer() total_daily = metrics.Integer() + class FileRunningTotals(InnerDoc): total = metrics.Integer() public = metrics.Integer() @@ -102,6 +103,16 @@ class FileRunningTotals(InnerDoc): public_daily = metrics.Integer() private_daily = metrics.Integer() + +class DataRunningTotals(InnerDoc): + total = metrics.Integer() + public = metrics.Integer() + private = metrics.Integer() + total_daily = metrics.Integer() + public_daily = metrics.Integer() + private_daily = metrics.Integer() + + class NodeRunningTotals(InnerDoc): total = metrics.Integer() total_excluding_spam = metrics.Integer() @@ -112,6 +123,18 @@ class NodeRunningTotals(InnerDoc): public_daily = metrics.Integer() private_daily = metrics.Integer() + +class ProjectRootRunningTotals(InnerDoc): + total = metrics.Integer() + total_excluding_spam = metrics.Integer() + public = metrics.Integer() + private = metrics.Integer() + total_daily = metrics.Integer() + total_daily_excluding_spam = metrics.Integer() + public_daily = metrics.Integer() + private_daily = metrics.Integer() + + class RegistrationRunningTotals(InnerDoc): total = metrics.Integer() public = metrics.Integer() @@ -124,6 +147,15 @@ class RegistrationRunningTotals(InnerDoc): embargoed_v2_daily = metrics.Integer() withdrawn_daily = metrics.Integer() + +class PreprintRunningTotals(InnerDoc): + total = metrics.Integer() + public = metrics.Integer() + withdrawn = metrics.Integer() + total_daily = metrics.Integer() + public_daily = metrics.Integer() + withdrawn_daily = metrics.Integer() + ##### END reusable inner objects ##### @@ -168,6 +200,61 @@ class InstitutionSummaryReport(DailyReport): registered_projects = metrics.Object(RegistrationRunningTotals) +class InstitutionDashboardSummaryReport(DailyReport): + """ + These are the following attributes necessary for the Institutional Dashboard Summary: + Counts + Users + Top-level Public projects* + Top-level Private projects* + Public registrations + Private registrations + preprints + Files + Data stored (both public and private; OSF storage only) + Tables + """ + + DAILY_UNIQUE_FIELD = 'institution_id' + + institution_id = metrics.Keyword() + institution_name = metrics.Keyword() + users = metrics.Object(RunningTotal) + projects = metrics.Object(NodeRunningTotals) + registrations = metrics.Object(RegistrationRunningTotals) + preprint = metrics.Object(PreprintRunningTotals) + files = metrics.Object(FileRunningTotals) + data = metrics.Object(DataRunningTotals) + + # Visualizations + ## Users by department pie chart + users_by_departments = metrics.Object() + + ## Line graph of recent user activity [Wip] + + ## Stacked bar chart showing types of OSF objects + types_of_resource = metrics.Object() + + ## simple bar chart showing public vs. private data + private_data = metrics.Integer() + public_data = metrics.Integer() + + ## Pie chart showing addons used + types_of_addons = metrics.Object(StorageAddonUsage) + + ## Pie chart of storage regions + storage_regions = metrics.Object() + + ## Pie chart of licenses + types_of_licenses = metrics.Object() + + ## FAIR assessment star plot (a star plot with each arm being the presence of a metadata element or identifier) or + # other FAIR metric? [WIP] + + ## Something to represent the size of projects … like a scatter plot? Each dot is a project and one axis is size of + # data, but not sure what the other axis would be + + class NewUserDomainReport(DailyReport): DAILY_UNIQUE_FIELD = 'domain_name'