From bd4b9a0cc56193ef05ec2218133ae895fb66234e Mon Sep 17 00:00:00 2001 From: John Tordoff <> Date: Wed, 18 Sep 2024 09:38:00 -0400 Subject: [PATCH] Merge branch 'feature/insti-dash-improv' of https://github.com/CenterForOpenScience/osf.io into new-summary-reports * 'feature/insti-dash-improv' of https://github.com/CenterForOpenScience/osf.io: (43 commits) fix: usable waffle-flag admin Fix task name for clear_expired_sessions in celery schedule Update CHANGELOG, bump version Revert "Add logging to post-commit handlers" filter out deleted preprint drafts (#10731) add debugging remove debugging remove debugging Revert "Add logging to post-commit handlers" Update commit to warning level Update commit to warning level Update commit to warning level Update commit to warning level Add logging to post-commit handlers Add logging to post-commit handlers revert resend confirmation to synchronous version add preprint draft relationship to user add PreprintDraftSerializer remove command file Move notification deletion to a dedicated view and remove obsolete duplicate notification handling logic. ... --- .docker-compose.env | 2 +- .../reporters/institution_summary_monthly.py | 134 ++++++------------ osf/metrics/reports.py | 29 ---- .../test_institutional_summary_reporter.py | 121 +++------------- 4 files changed, 65 insertions(+), 221 deletions(-) diff --git a/.docker-compose.env b/.docker-compose.env index 9cb7a59e274..4c77bb00804 100644 --- a/.docker-compose.env +++ b/.docker-compose.env @@ -19,4 +19,4 @@ BROKER_URL=amqp://guest:guest@192.168.168.167:5672/ REDIS_URL=redis://192.168.168.167:6379/1 EMBER_DOMAIN=192.168.168.167 -#PYTHONUNBUFFERED=0 # This when set to 0 will allow print statements to be visible in the Docker logs +PYTHONUNBUFFERED=0 # This when set to 0 will allow print statements to be visible in the Docker logs diff --git a/osf/metrics/reporters/institution_summary_monthly.py b/osf/metrics/reporters/institution_summary_monthly.py index f150ac29e4e..c44f25aa432 100644 --- a/osf/metrics/reporters/institution_summary_monthly.py +++ b/osf/metrics/reporters/institution_summary_monthly.py @@ -1,19 +1,13 @@ from django.contrib.contenttypes.models import ContentType -from django.db.models import Q, F, Sum, Count +from django.db.models import Q, F, Sum from osf.models import Institution, Preprint, AbstractNode, FileVersion from osf.models.spam import SpamStatus from addons.osfstorage.models import OsfStorageFile -from osf.metrics.reports import ( - InstitutionMonthlySummaryReport, - License as ReporterLicense, - Addon as ReporterAddon, - StorageRegion as ReporterStorageRegion, - Department as ReporterDepartment, -) +from osf.metrics.reports import InstitutionMonthlySummaryReport from osf.metrics.utils import YearMonth from ._base import MonthlyReporter -from website import settings +from datetime import datetime class InstitutionalSummaryMonthlyReporter(MonthlyReporter): @@ -28,19 +22,16 @@ def generate_report(self, institution, yearmonth): return InstitutionMonthlySummaryReport( institution_id=institution._id, - public_project_count=self._get_count(node_queryset, 'osf.node', is_public=True), private_project_count=self._get_count(node_queryset, 'osf.node', is_public=False), + public_project_count=self._get_count(node_queryset, 'osf.node', is_public=True), + user_count=institution.get_institution_users().count(), public_registration_count=self._get_count(node_queryset, 'osf.registration', is_public=True), embargoed_registration_count=self._get_count(node_queryset, 'osf.registration', is_public=False), - published_preprint_count=self.get_published_preprints(institution).count(), + preprint_count=self.get_published_preprints(institution).count(), + storage_byte_count=self.get_storage_size(node_queryset, institution), public_file_count=self.get_files(node_queryset, institution, is_public=True).count(), - private_file_count=self.get_files(node_queryset, institution, is_public=False).count(), - public_storage_count=self.get_storage_size(node_queryset, institution, is_public=True), - private_storage_count=self.get_storage_size(node_queryset, institution, is_public=False), - departments=self.get_department_count(institution), - licenses=self.get_license_count(node_queryset), - addons=self.get_addons_count(), - storage_regions=self.get_storage_region_count(node_queryset), + monthly_logged_in_user_count=self.get_monthly_logged_in_user_count(institution, yearmonth), + monthly_active_user_count=self.get_monthly_active_user_count(institution, yearmonth), ) def _get_count(self, node_queryset, node_type, is_public): @@ -53,9 +44,13 @@ def get_published_preprints(self, institution): affiliated_institutions=institution ).exclude(spam_status=SpamStatus.SPAM) - def get_files(self, node_queryset, institution, is_public): + def get_files(self, node_queryset, institution, is_public=None): + public_kwargs = {} + if is_public: + public_kwargs = {'is_public': is_public} + target_node_q = Q( - target_object_id__in=node_queryset.filter(is_public=is_public).values("pk"), + target_object_id__in=node_queryset.filter(**public_kwargs).values("pk"), target_content_type=ContentType.objects.get_for_model(AbstractNode), ) target_preprint_q = Q( @@ -66,76 +61,35 @@ def get_files(self, node_queryset, institution, is_public): deleted__isnull=True, purged__isnull=True ).filter(target_node_q | target_preprint_q) - def get_storage_size(self, node_queryset, institution, is_public): - files = self.get_files(node_queryset, institution, is_public) + def get_storage_size(self, node_queryset, institution): + files = self.get_files(node_queryset, institution) return FileVersion.objects.filter( - size__gt=0, purged__isnull=True, basefilenode__in=files + size__gt=0, + purged__isnull=True, + basefilenode__in=files ).aggregate(storage_bytes=Sum("size", default=0))["storage_bytes"] - def get_license_count(self, node_queryset): - licenses = node_queryset.exclude(node_license=None).values( - "node_license__node_license__name", "node_license___id" - ).annotate(total=Count("node_license")) - - license_list = [ - ReporterLicense( - id=license_data["node_license___id"], - name=license_data["node_license__node_license__name"], - total=license_data["total"], - ) - for license_data in licenses - ] - - license_list.append( - ReporterLicense( - id=None, - name="Default (No license selected)", - total=node_queryset.filter(node_license=None).count(), - ) - ) - - return license_list - - def get_addons_count(self): - storage_addons = [ - addon for addon in settings.ADDONS_AVAILABLE if "storage" in addon.categories - ] - - addon_counts = [] - for addon in storage_addons: - node_settings = addon.get_model("NodeSettings").objects.exclude( - owner__isnull=True, - owner__deleted__isnull=False, - owner__spam_status=SpamStatus.SPAM, - ) - is_oauth = hasattr(addon.get_model("NodeSettings"), "external_account") - filter_condition = Q(external_account__isnull=False) if is_oauth else Q() - count = node_settings.filter(filter_condition).count() - addon_counts.append(ReporterAddon(name=addon.short_name, total=count)) - - return addon_counts - - def get_storage_region_count(self, node_queryset): - region_counts = node_queryset.values( - "addons_osfstorage_node_settings__region___id", - "addons_osfstorage_node_settings__region__name", - ).annotate(total=Count("addons_osfstorage_node_settings__region___id")) - - return [ - ReporterStorageRegion( - id=region["addons_osfstorage_node_settings__region___id"], - name=region["addons_osfstorage_node_settings__region__name"], - total=region["total"], - ) - for region in region_counts - ] - - def get_department_count(self, institution): - departments = institution.institutionaffiliation_set.exclude( - sso_department__isnull=True - ).values("sso_department").annotate(total=Count("sso_department")) - - return [ - ReporterDepartment(name=dept["sso_department"], total=dept["total"]) - for dept in departments - ] + def get_month_start_end(self, yearmonth): + # Get the first day of the month + start_date = datetime(yearmonth.year, yearmonth.month, 1) + # Calculate the first day of the next month + if yearmonth.month == 12: + end_date = datetime(yearmonth.year + 1, 1, 1) + else: + end_date = datetime(yearmonth.year, yearmonth.month + 1, 1) + return start_date, end_date + + def get_monthly_logged_in_user_count(self, institution, yearmonth): + start_date, end_date = self.get_month_start_end(yearmonth) + return institution.get_institution_users().filter( + date_last_login__gte=start_date, + date_last_login__lte=end_date + ).count() + + def get_monthly_active_user_count(self, institution, yearmonth): + start_date, end_date = self.get_month_start_end(yearmonth) + return institution.get_institution_users().filter( + date_disabled__isnull=True, + date_last_login__gte=start_date, + date_last_login__lte=end_date + ).count() diff --git a/osf/metrics/reports.py b/osf/metrics/reports.py index 4f5242877af..03f9fd341b0 100644 --- a/osf/metrics/reports.py +++ b/osf/metrics/reports.py @@ -272,30 +272,6 @@ class InstitutionalUserReport(MonthlyReport): storage_byte_count = metrics.Long() -class Department(InnerDoc): - id = metrics.Keyword() - name = metrics.Keyword() - total = metrics.Long() - - -class License(InnerDoc): - id = metrics.Keyword() - name = metrics.Keyword() - total = metrics.Long() - - -class Addon(InnerDoc): - id = metrics.Keyword() - name = metrics.Keyword() - total = metrics.Long() - - -class StorageRegion(InnerDoc): - id = metrics.Keyword() - name = metrics.Keyword() - total = metrics.Long() - - class InstitutionMonthlySummaryReport(MonthlyReport): UNIQUE_TOGETHER_FIELDS = ('report_yearmonth', 'institution_id', ) institution_id = metrics.Keyword() @@ -308,8 +284,3 @@ class InstitutionMonthlySummaryReport(MonthlyReport): private_file_count = metrics.Long() public_storage_count = metrics.Long() private_storage_count = metrics.Long() - - departments = metrics.Object(Department, multi=True) - licenses = metrics.Object(License, multi=True) - addons = metrics.Object(Addon, multi=True) - storage_regions = metrics.Object(StorageRegion, multi=True) diff --git a/osf_tests/metrics/reporters/test_institutional_summary_reporter.py b/osf_tests/metrics/reporters/test_institutional_summary_reporter.py index 404aa4b8443..6cdbebfc185 100644 --- a/osf_tests/metrics/reporters/test_institutional_summary_reporter.py +++ b/osf_tests/metrics/reporters/test_institutional_summary_reporter.py @@ -9,12 +9,8 @@ RegistrationFactory, PreprintFactory, UserFactory, - NodeLicenseRecordFactory, - RegionFactory, AuthUserFactory, ) -from addons.github.tests.factories import GitHubAccountFactory -from framework.auth import Auth class TestInstiSummaryMonthlyReporter(TestCase): @@ -34,6 +30,9 @@ def setUpTestData(cls): cls._published_preprint = PreprintFactory(creator=UserFactory(), is_public=True) cls._published_preprint.affiliated_institutions.add(cls._institution) + cls._logged_in_user = cls._create_logged_in_user() + cls._active_user = cls._create_active_user() + @classmethod def _create_affiliated_project(cls, is_public): project = ProjectFactory(creator=UserFactory(), is_public=is_public) @@ -46,31 +45,21 @@ def _create_affiliated_registration(cls, is_public): registration.affiliated_institutions.add(cls._institution) return registration - def configure_addon(self): - - addon_user = AuthUserFactory() - auth_obj = Auth(user=addon_user) - node = ProjectFactory(creator=addon_user) - - addon_user.add_addon('github') - user_addon = addon_user.get_addon('github') - oauth_settings = GitHubAccountFactory(display_name='john') - oauth_settings.save() - addon_user.external_accounts.add(oauth_settings) - addon_user.save() - - node.add_addon('github', auth_obj) - node_addon = node.get_addon('github') - node_addon.user = 'john' - node_addon.repo = 'youre-my-best-friend' - node_addon.user_settings = user_addon - node_addon.external_account = oauth_settings - node_addon.save() - - user_addon.oauth_grants[node._id] = {oauth_settings._id: []} - user_addon.save() + @classmethod + def _create_logged_in_user(cls): + user = AuthUserFactory() + user.add_or_update_affiliated_institution(cls._institution) + user.date_last_login = cls._now + user.save() + return user - return node + @classmethod + def _create_active_user(cls): + user = AuthUserFactory() + user.add_or_update_affiliated_institution(cls._institution) + user.date_confirmed = cls._now - datetime.timedelta(days=1) + user.save() + return user def test_report_generation(self): reporter = InstitutionalSummaryMonthlyReporter() @@ -83,76 +72,6 @@ def test_report_generation(self): self.assertEqual(report.private_project_count, 1) self.assertEqual(report.public_registration_count, 1) self.assertEqual(report.embargoed_registration_count, 1) - self.assertEqual(report.published_preprint_count, 1 if hasattr(Preprint, 'affiliated_institutions') else 0) - - def test_license_counts(self): - license_record = NodeLicenseRecordFactory() - node_with_license = self._create_affiliated_project(is_public=True) - node_with_license.node_license = license_record - node_with_license.save() - - reporter = InstitutionalSummaryMonthlyReporter() - reports = list(reporter.report(self._yearmonth)) - report = reports[0] - - self.assertIsNotNone(report.licenses) - self.assertEqual(len(report.licenses), 2) - - default_license_count = next((l for l in report.licenses if l.name == 'Default (No license selected)'), None) - self.assertEqual(default_license_count['total'], 4) - - license_count = next((l for l in report.licenses if l.name == license_record.name), None) - self.assertEqual(license_count['total'], 1) - - def test_addons_count(self): - github_addon_node = self.configure_addon() - github_addon_node.affiliated_institutions.add(self._institution) - - reporter = InstitutionalSummaryMonthlyReporter() - reports = list(reporter.report(self._yearmonth)) - report = reports[0] - - osfstorage_addon_count = next((l for l in report.addons if l.name == 'osfstorage'), None) - self.assertTrue(osfstorage_addon_count['total'] == 7) - - github_addon_count = next((l for l in report.addons if l.name == 'github'), None) - self.assertTrue(github_addon_count['total'] == 1) - - def test_storage_region_count(self): - region = RegionFactory(_id='test1', name='Test Region #1') - region2 = RegionFactory(_id='test2', name='Test Region #2') - self._add_node_with_storage_region(region) - self._add_node_with_storage_region(region2) - self._add_node_with_storage_region(region2) - - reporter = InstitutionalSummaryMonthlyReporter() - reports = list(reporter.report(self._yearmonth)) - report = reports[0] - - self.assertIsNotNone(report.storage_regions) - self.assertEqual(len(report.storage_regions), 3) - - storage_region_count = next((l for l in report.storage_regions if l.name == 'Test Region #2'), None) - self.assertEqual(storage_region_count['total'], 2) - - def _add_node_with_storage_region(self, region): - project = self._create_affiliated_project(is_public=True) - addon = project.get_addon('osfstorage') - addon.region = region - addon.save() - self._institution.storage_regions.add(region) - - def test_department_count(self): - user = self._public_project.creator - user.add_or_update_affiliated_institution(self._institution, sso_department='Test Department #1') - user = self._private_project.creator - user.add_or_update_affiliated_institution(self._institution, sso_department='Test Department #2') - - reporter = InstitutionalSummaryMonthlyReporter() - reports = list(reporter.report(self._yearmonth)) - report = reports[0] - - self.assertEqual(len(report.departments), 2) - - departments_count = next((l for l in report.departments if l.name == 'Test Department #1'), None) - self.assertEqual(departments_count['total'], 1) + self.assertEqual(report.published_preprint_count, 1 if hasattr(Preprint, 'affiliated_institutions') else None) + self.assertEqual(report.monthly_logged_in_user_count, 1) + self.assertEqual(report.monthly_active_user_count, 1)