diff --git a/api/institutions/views.py b/api/institutions/views.py index 5e7bd8ae5c8c..3f1bcad08817 100644 --- a/api/institutions/views.py +++ b/api/institutions/views.py @@ -1,5 +1,4 @@ -from django.db.models import Count, Q, F, Value, BooleanField, IntegerField -from django.db.models.functions import Coalesce +from django.db.models import Count, Q, F from rest_framework import generics from rest_framework import permissions as drf_permissions from rest_framework import exceptions @@ -7,7 +6,6 @@ from rest_framework.response import Response from rest_framework.settings import api_settings - from framework.auth.oauth_scopes import CoreScopes from osf.metrics import InstitutionProjectCounts @@ -545,43 +543,43 @@ class InstitutionDashboardUserList(JSONAPIBaseView, generics.ListAPIView, ListFi def get_default_queryset(self): institution = self.get_institution() - from django.db.models import OuterRef, Subquery, Count, Q, F, Value, BooleanField, IntegerField + from django.db.models import IntegerField, Value, BooleanField, BigIntegerField from django.db.models.functions import Coalesce - from django.db.models.expressions import RawSQL + from django.db.models.expressions import RawSQL, F return institution.get_institution_users().annotate( email_address=F('username'), department=F('institutionaffiliation__sso_department'), - # Count of public projects (assuming a related_name 'projects' from OSFUser to Project) - number_of_public_projects=Count( + # Count of public projects + public_projects=Count( 'nodes', filter=(Q(nodes__is_public=True) & Q(nodes__type='osf.node')), - distinct=True + distinct=True, ), - number_of_private_projects=Count( + private_projects=Count( 'nodes', filter=(Q(nodes__is_public=False) & Q(nodes__type='osf.node')), - distinct=True + distinct=True, ), # Count of public and private registrations - number_of_public_registrations=Count( + public_registrations=Count( 'nodes', filter=(Q(nodes__is_public=True) & Q(nodes__type='osf.registration')), - distinct=True + distinct=True, ), - number_of_private_registrations=Count( + embargoed_registrations=Count( 'nodes', filter=(Q(nodes__is_public=False) & Q(nodes__type='osf.registration')), - distinct=True + distinct=True, ), # Count of preprints - number_of_preprints=Count( + published_preprints=Count( 'preprints', filter=Q(preprints__is_public=True), - distinct=True + distinct=True, ), - # Count files associated with nodes - number_of_node_files=RawSQL( + # Count files associated with nodes using RawSQL + public_node_files=RawSQL( """ SELECT COUNT(f.id) FROM osf_basefilenode f @@ -589,14 +587,15 @@ def get_default_queryset(self): INNER JOIN django_content_type ct ON ct.id = f.target_content_type_id WHERE ct.model = 'abstractnode' AND n.type = 'osf.node' + AND n.is_public = TRUE AND f.type = 'osf.osfstoragefile' AND n.creator_id = osf_osfuser.id """, [], - output_field=IntegerField() + output_field=IntegerField(), ), # Count files associated with registrations using RawSQL - number_of_registration_files=RawSQL( + public_registration_files=RawSQL( """ SELECT COUNT(f.id) FROM osf_basefilenode f @@ -604,14 +603,15 @@ def get_default_queryset(self): INNER JOIN django_content_type ct ON ct.id = f.target_content_type_id WHERE ct.model = 'abstractnode' AND r.type = 'osf.registration' + AND r.is_public = TRUE AND f.type = 'osf.osfstoragefile' AND r.creator_id = osf_osfuser.id """, [], - output_field=IntegerField() + output_field=IntegerField(), ), # Count files associated with preprints using RawSQL - number_of_preprint_files=RawSQL( + public_preprint_files=RawSQL( """ SELECT COUNT(f.id) FROM osf_basefilenode f @@ -623,21 +623,49 @@ def get_default_queryset(self): AND p.creator_id = osf_osfuser.id """, [], - output_field=IntegerField() + output_field=IntegerField(), + ), + public_files=Coalesce( + F('public_node_files') + + F('public_registration_files') + + F('public_preprint_files'), + Value(0), + output_field=IntegerField(), + ), + storage_byte_count=RawSQL( + """ + SELECT COALESCE(SUM(version.size), 0) FROM osf_basefileversionsthrough AS obfnv + LEFT JOIN osf_basefilenode file ON obfnv.basefilenode_id = file.id + LEFT JOIN osf_fileversion version ON obfnv.fileversion_id = version.id + LEFT JOIN django_content_type type on file.target_content_type_id = type.id + WHERE file.provider = 'osfstorage' + AND type.model = 'abstractnode' + AND file.deleted_on IS NULL + AND file.target_object_id IN ( + SELECT id FROM osf_abstractnode + WHERE creator_id = osf_osfuser.id + ) + """, + [], + output_field=BigIntegerField(), ), - number_of_files=Coalesce( - F('number_of_node_files') + - F('number_of_registration_files') + - F('number_of_preprint_files'), + total_object_count=Coalesce( + F('public_projects') + + F('private_projects') + + F('public_registrations') + + F('embargoed_registrations') + + F('published_preprints'), Value(0), - output_field=IntegerField() + output_field=IntegerField(), ), + month_last_login=F('last_login'), + month_last_active=F('last_login'), has_orcid=Coalesce( Q(external_identity__has_key='ORCID'), Value(False), - output_field=BooleanField() + output_field=BooleanField(), ), - account_created_date=F('created') + account_created=F('created'), ) # overrides RetrieveAPIView @@ -659,18 +687,28 @@ def create_csv_response(self): response['Content-Disposition'] = 'attachment; filename="institution_users.csv"' writer = csv.writer(response) - writer.writerow(['ID', 'Email', 'Department', 'Public Projects', 'Private Projects', 'Public Registrations', - 'Private Registrations', 'Preprints']) + writer.writerow( + [ + 'ID', + 'Email', + 'Department', + 'Public Projects', + 'Private Projects', + 'Public Registrations', + 'Private Registrations', + 'Preprints', + ], + ) for user in queryset: writer.writerow([ user.id, user.email_address, user.department, - user.number_of_public_projects, - user.number_of_private_projects, - user.number_of_public_registrations, - user.number_of_private_registrations, - user.number_of_preprints, + user.public_projects, + user.private_projects, + user.public_registrations, + user.embargoed_registrations, + user.published_preprints, ]) return response diff --git a/api/users/serializers.py b/api/users/serializers.py index 0efb11507b70..e50f8149d5d1 100644 --- a/api/users/serializers.py +++ b/api/users/serializers.py @@ -57,15 +57,15 @@ class UserSerializer(JSONAPISerializer): filterable_fields = frozenset([ 'email_address', # For Institutional Dashboard only 'department', # For Institutional Dashboard only - 'number_of_public_projects', # For Institutional Dashboard only - 'number_of_private_projects', # For Institutional Dashboard only - 'number_of_public_registrations', # For Institutional Dashboard only - 'number_of_private_registrations', # For Institutional Dashboard only - 'number_of_preprints', # For Institutional Dashboard only - 'number_of_files', # For Institutional Dashboard only + 'public_projects', # For Institutional Dashboard only + 'private_projects', # For Institutional Dashboard only + 'public_registrations', # For Institutional Dashboard only + 'embargoed_registrations', # For Institutional Dashboard only + 'published_preprints', # For Institutional Dashboard only + 'public_files', # For Institutional Dashboard only 'has_orcid', # For Institutional Dashboard only - 'account_created_date', # For Institutional Dashboard only - 'last_log', # For Institutional Dashboard only + 'account_created', # For Institutional Dashboard only + 'last_login', # For Institutional Dashboard only 'full_name', 'given_name', 'middle_names', @@ -100,18 +100,20 @@ class UserSerializer(JSONAPISerializer): email_address = ser.CharField(required=False, allow_blank=True, help_text='For Institutional Dashboard only') department = ser.CharField(required=False, allow_blank=True, help_text='For Institutional Dashboard only') - number_of_public_projects = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_private_projects = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_public_registrations = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_private_registrations = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_preprints = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_node_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_registration_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_preprint_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') - number_of_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + public_projects = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + private_projects = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + public_registrations = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + embargoed_registrations = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + published_preprints = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + public_node_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + public_registration_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + public_preprint_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + public_files = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + storage_byte_count = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') + total_object_count = ser.IntegerField(required=False, help_text='For Institutional Dashboard only') has_orcid = ser.BooleanField(required=False, help_text='For Institutional Dashboard only') - account_created_date = VersionedDateTimeField(required=False, help_text='For Institutional Dashboard only') - last_log = VersionedDateTimeField(required=False, help_text='For Institutional Dashboard only') + account_created = VersionedDateTimeField(required=False, help_text='For Institutional Dashboard only') + last_login = VersionedDateTimeField(required=False, help_text='For Institutional Dashboard only') links = HideIfDisabled( LinksField( diff --git a/api_tests/institutions/views/test_institution_dashboard_user.py b/api_tests/institutions/views/test_institution_dashboard_user.py index 2edae097d9ef..9b105be95387 100644 --- a/api_tests/institutions/views/test_institution_dashboard_user.py +++ b/api_tests/institutions/views/test_institution_dashboard_user.py @@ -58,7 +58,7 @@ def users(self, institution): ) user_one.add_or_update_affiliated_institution( institution, - sso_department="Science Department" + sso_department='Science Department' ) user_two.add_or_update_affiliated_institution(institution) user_three.add_or_update_affiliated_institution(institution) @@ -123,16 +123,16 @@ def test_return_all_users(self, app, institution, users): ('[full_name]', 'Example', 3), # Multiple users should be returned here ('[email_address]', 'bob@example.com', 1), ('[department]', 'Science Department', 1), - ('[number_of_public_projects][lte]', '1', 2), - ('[number_of_private_projects][lt]', '1', 3), - ('[number_of_private_projects][gte]', '1', 3), - ('[number_of_public_registrations][lte]', '1', 1), - ('[number_of_private_registrations][lte]', '1', 3), - ('[number_of_preprints][lte]', '2', 1), - ('[number_of_files][lte]', '1', 0), + ('[public_projects][lte]', '1', 2), + ('[private_projects][lte]', '1', 3), + ('[private_projects][gte]', '1', 3), + ('[public_registrations][lte]', '1', 3), + ('[embargoed_registrations][lte]', '1', 3), + ('[published_preprints][lte]', '1', 2), + ('[public_files][lt]', '1', 1), + ('[public_files][lte]', '1', 2), ('[last_login][lte]', '2-11-2018', 0), - ('[last_log]', 'account_created', 0), - ('[account_created_date]', '2-11-2018', 0), + ('[account_created]', '2-11-2018', 0), ('[has_orcid]', 'True', 0), ]) def test_filter_users(self, app, institution, users, attribute, value, expected_count): diff --git a/osf/metrics/reports.py b/osf/metrics/reports.py index 8ea7ae285bed..609e79fc324b 100644 --- a/osf/metrics/reports.py +++ b/osf/metrics/reports.py @@ -94,7 +94,6 @@ class RunningTotal(InnerDoc): total = metrics.Integer() total_daily = metrics.Integer() - class FileRunningTotals(InnerDoc): total = metrics.Integer() public = metrics.Integer() @@ -113,9 +112,6 @@ class NodeRunningTotals(InnerDoc): public_daily = metrics.Integer() private_daily = metrics.Integer() - - - class RegistrationRunningTotals(InnerDoc): total = metrics.Integer() public = metrics.Integer()