Skip to content

Commit

Permalink
Merge branch 'feature/insti-dash-improv' of https://github.com/Center…
Browse files Browse the repository at this point in the history 
…ForOpenScience/osf.io into new-summary-reports

* 'feature/insti-dash-improv' of https://github.com/CenterForOpenScience/osf.io: (43 commits)
  fix: usable waffle-flag admin
  Fix task name for clear_expired_sessions in celery schedule
  Update CHANGELOG, bump version
  Revert "Add logging to post-commit handlers"
  filter out deleted preprint drafts (CenterForOpenScience#10731)
  add debugging
  remove debugging
  remove debugging
  Revert "Add logging to post-commit handlers"
  Update commit to warning level
  Update commit to warning level
  Update commit to warning level
  Update commit to warning level
  Add logging to post-commit handlers
  Add logging to post-commit handlers
  revert resend confirmation to synchronous version
  add preprint draft relationship to user
  add PreprintDraftSerializer
  remove command file
  Move notification deletion to a dedicated view and remove obsolete duplicate notification handling logic.
  ...
  • Loading branch information
John Tordoff committed Sep 18, 2024
1 parent 26d50e9 commit bd4b9a0
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 221 deletions.
2 changes: 1 addition & 1 deletion .docker-compose.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ BROKER_URL=amqp://guest:[email protected]:5672/
REDIS_URL=redis://192.168.168.167:6379/1
EMBER_DOMAIN=192.168.168.167

#PYTHONUNBUFFERED=0 # This when set to 0 will allow print statements to be visible in the Docker logs
PYTHONUNBUFFERED=0 # This when set to 0 will allow print statements to be visible in the Docker logs
134 changes: 44 additions & 90 deletions osf/metrics/reporters/institution_summary_monthly.py
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q, F, Sum, Count
from django.db.models import Q, F, Sum

from osf.models import Institution, Preprint, AbstractNode, FileVersion
from osf.models.spam import SpamStatus
from addons.osfstorage.models import OsfStorageFile
from osf.metrics.reports import (
InstitutionMonthlySummaryReport,
License as ReporterLicense,
Addon as ReporterAddon,
StorageRegion as ReporterStorageRegion,
Department as ReporterDepartment,
)
from osf.metrics.reports import InstitutionMonthlySummaryReport
from osf.metrics.utils import YearMonth
from ._base import MonthlyReporter
from website import settings
from datetime import datetime


class InstitutionalSummaryMonthlyReporter(MonthlyReporter):
Expand All @@ -28,19 +22,16 @@ def generate_report(self, institution, yearmonth):

return InstitutionMonthlySummaryReport(
institution_id=institution._id,
public_project_count=self._get_count(node_queryset, 'osf.node', is_public=True),
private_project_count=self._get_count(node_queryset, 'osf.node', is_public=False),
public_project_count=self._get_count(node_queryset, 'osf.node', is_public=True),
user_count=institution.get_institution_users().count(),
public_registration_count=self._get_count(node_queryset, 'osf.registration', is_public=True),
embargoed_registration_count=self._get_count(node_queryset, 'osf.registration', is_public=False),
published_preprint_count=self.get_published_preprints(institution).count(),
preprint_count=self.get_published_preprints(institution).count(),
storage_byte_count=self.get_storage_size(node_queryset, institution),
public_file_count=self.get_files(node_queryset, institution, is_public=True).count(),
private_file_count=self.get_files(node_queryset, institution, is_public=False).count(),
public_storage_count=self.get_storage_size(node_queryset, institution, is_public=True),
private_storage_count=self.get_storage_size(node_queryset, institution, is_public=False),
departments=self.get_department_count(institution),
licenses=self.get_license_count(node_queryset),
addons=self.get_addons_count(),
storage_regions=self.get_storage_region_count(node_queryset),
monthly_logged_in_user_count=self.get_monthly_logged_in_user_count(institution, yearmonth),
monthly_active_user_count=self.get_monthly_active_user_count(institution, yearmonth),
)

def _get_count(self, node_queryset, node_type, is_public):
Expand All @@ -53,9 +44,13 @@ def get_published_preprints(self, institution):
affiliated_institutions=institution
).exclude(spam_status=SpamStatus.SPAM)

def get_files(self, node_queryset, institution, is_public):
def get_files(self, node_queryset, institution, is_public=None):
public_kwargs = {}
if is_public:
public_kwargs = {'is_public': is_public}

target_node_q = Q(
target_object_id__in=node_queryset.filter(is_public=is_public).values("pk"),
target_object_id__in=node_queryset.filter(**public_kwargs).values("pk"),
target_content_type=ContentType.objects.get_for_model(AbstractNode),
)
target_preprint_q = Q(
Expand All @@ -66,76 +61,35 @@ def get_files(self, node_queryset, institution, is_public):
deleted__isnull=True, purged__isnull=True
).filter(target_node_q | target_preprint_q)

def get_storage_size(self, node_queryset, institution, is_public):
files = self.get_files(node_queryset, institution, is_public)
def get_storage_size(self, node_queryset, institution):
files = self.get_files(node_queryset, institution)
return FileVersion.objects.filter(
size__gt=0, purged__isnull=True, basefilenode__in=files
size__gt=0,
purged__isnull=True,
basefilenode__in=files
).aggregate(storage_bytes=Sum("size", default=0))["storage_bytes"]

def get_license_count(self, node_queryset):
licenses = node_queryset.exclude(node_license=None).values(
"node_license__node_license__name", "node_license___id"
).annotate(total=Count("node_license"))

license_list = [
ReporterLicense(
id=license_data["node_license___id"],
name=license_data["node_license__node_license__name"],
total=license_data["total"],
)
for license_data in licenses
]

license_list.append(
ReporterLicense(
id=None,
name="Default (No license selected)",
total=node_queryset.filter(node_license=None).count(),
)
)

return license_list

def get_addons_count(self):
storage_addons = [
addon for addon in settings.ADDONS_AVAILABLE if "storage" in addon.categories
]

addon_counts = []
for addon in storage_addons:
node_settings = addon.get_model("NodeSettings").objects.exclude(
owner__isnull=True,
owner__deleted__isnull=False,
owner__spam_status=SpamStatus.SPAM,
)
is_oauth = hasattr(addon.get_model("NodeSettings"), "external_account")
filter_condition = Q(external_account__isnull=False) if is_oauth else Q()
count = node_settings.filter(filter_condition).count()
addon_counts.append(ReporterAddon(name=addon.short_name, total=count))

return addon_counts

def get_storage_region_count(self, node_queryset):
region_counts = node_queryset.values(
"addons_osfstorage_node_settings__region___id",
"addons_osfstorage_node_settings__region__name",
).annotate(total=Count("addons_osfstorage_node_settings__region___id"))

return [
ReporterStorageRegion(
id=region["addons_osfstorage_node_settings__region___id"],
name=region["addons_osfstorage_node_settings__region__name"],
total=region["total"],
)
for region in region_counts
]

def get_department_count(self, institution):
departments = institution.institutionaffiliation_set.exclude(
sso_department__isnull=True
).values("sso_department").annotate(total=Count("sso_department"))

return [
ReporterDepartment(name=dept["sso_department"], total=dept["total"])
for dept in departments
]
def get_month_start_end(self, yearmonth):
# Get the first day of the month
start_date = datetime(yearmonth.year, yearmonth.month, 1)
# Calculate the first day of the next month
if yearmonth.month == 12:
end_date = datetime(yearmonth.year + 1, 1, 1)
else:
end_date = datetime(yearmonth.year, yearmonth.month + 1, 1)
return start_date, end_date

def get_monthly_logged_in_user_count(self, institution, yearmonth):
start_date, end_date = self.get_month_start_end(yearmonth)
return institution.get_institution_users().filter(
date_last_login__gte=start_date,
date_last_login__lte=end_date
).count()

def get_monthly_active_user_count(self, institution, yearmonth):
start_date, end_date = self.get_month_start_end(yearmonth)
return institution.get_institution_users().filter(
date_disabled__isnull=True,
date_last_login__gte=start_date,
date_last_login__lte=end_date
).count()
29 changes: 0 additions & 29 deletions osf/metrics/reports.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,30 +272,6 @@ class InstitutionalUserReport(MonthlyReport):
storage_byte_count = metrics.Long()


class Department(InnerDoc):
id = metrics.Keyword()
name = metrics.Keyword()
total = metrics.Long()


class License(InnerDoc):
id = metrics.Keyword()
name = metrics.Keyword()
total = metrics.Long()


class Addon(InnerDoc):
id = metrics.Keyword()
name = metrics.Keyword()
total = metrics.Long()


class StorageRegion(InnerDoc):
id = metrics.Keyword()
name = metrics.Keyword()
total = metrics.Long()


class InstitutionMonthlySummaryReport(MonthlyReport):
UNIQUE_TOGETHER_FIELDS = ('report_yearmonth', 'institution_id', )
institution_id = metrics.Keyword()
Expand All @@ -308,8 +284,3 @@ class InstitutionMonthlySummaryReport(MonthlyReport):
private_file_count = metrics.Long()
public_storage_count = metrics.Long()
private_storage_count = metrics.Long()

departments = metrics.Object(Department, multi=True)
licenses = metrics.Object(License, multi=True)
addons = metrics.Object(Addon, multi=True)
storage_regions = metrics.Object(StorageRegion, multi=True)
121 changes: 20 additions & 101 deletions osf_tests/metrics/reporters/test_institutional_summary_reporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,8 @@
RegistrationFactory,
PreprintFactory,
UserFactory,
NodeLicenseRecordFactory,
RegionFactory,
AuthUserFactory,
)
from addons.github.tests.factories import GitHubAccountFactory
from framework.auth import Auth


class TestInstiSummaryMonthlyReporter(TestCase):
Expand All @@ -34,6 +30,9 @@ def setUpTestData(cls):
cls._published_preprint = PreprintFactory(creator=UserFactory(), is_public=True)
cls._published_preprint.affiliated_institutions.add(cls._institution)

cls._logged_in_user = cls._create_logged_in_user()
cls._active_user = cls._create_active_user()

@classmethod
def _create_affiliated_project(cls, is_public):
project = ProjectFactory(creator=UserFactory(), is_public=is_public)
Expand All @@ -46,31 +45,21 @@ def _create_affiliated_registration(cls, is_public):
registration.affiliated_institutions.add(cls._institution)
return registration

def configure_addon(self):

addon_user = AuthUserFactory()
auth_obj = Auth(user=addon_user)
node = ProjectFactory(creator=addon_user)

addon_user.add_addon('github')
user_addon = addon_user.get_addon('github')
oauth_settings = GitHubAccountFactory(display_name='john')
oauth_settings.save()
addon_user.external_accounts.add(oauth_settings)
addon_user.save()

node.add_addon('github', auth_obj)
node_addon = node.get_addon('github')
node_addon.user = 'john'
node_addon.repo = 'youre-my-best-friend'
node_addon.user_settings = user_addon
node_addon.external_account = oauth_settings
node_addon.save()

user_addon.oauth_grants[node._id] = {oauth_settings._id: []}
user_addon.save()
@classmethod
def _create_logged_in_user(cls):
user = AuthUserFactory()
user.add_or_update_affiliated_institution(cls._institution)
user.date_last_login = cls._now
user.save()
return user

return node
@classmethod
def _create_active_user(cls):
user = AuthUserFactory()
user.add_or_update_affiliated_institution(cls._institution)
user.date_confirmed = cls._now - datetime.timedelta(days=1)
user.save()
return user

def test_report_generation(self):
reporter = InstitutionalSummaryMonthlyReporter()
Expand All @@ -83,76 +72,6 @@ def test_report_generation(self):
self.assertEqual(report.private_project_count, 1)
self.assertEqual(report.public_registration_count, 1)
self.assertEqual(report.embargoed_registration_count, 1)
self.assertEqual(report.published_preprint_count, 1 if hasattr(Preprint, 'affiliated_institutions') else 0)

def test_license_counts(self):
license_record = NodeLicenseRecordFactory()
node_with_license = self._create_affiliated_project(is_public=True)
node_with_license.node_license = license_record
node_with_license.save()

reporter = InstitutionalSummaryMonthlyReporter()
reports = list(reporter.report(self._yearmonth))
report = reports[0]

self.assertIsNotNone(report.licenses)
self.assertEqual(len(report.licenses), 2)

default_license_count = next((l for l in report.licenses if l.name == 'Default (No license selected)'), None)
self.assertEqual(default_license_count['total'], 4)

license_count = next((l for l in report.licenses if l.name == license_record.name), None)
self.assertEqual(license_count['total'], 1)

def test_addons_count(self):
github_addon_node = self.configure_addon()
github_addon_node.affiliated_institutions.add(self._institution)

reporter = InstitutionalSummaryMonthlyReporter()
reports = list(reporter.report(self._yearmonth))
report = reports[0]

osfstorage_addon_count = next((l for l in report.addons if l.name == 'osfstorage'), None)
self.assertTrue(osfstorage_addon_count['total'] == 7)

github_addon_count = next((l for l in report.addons if l.name == 'github'), None)
self.assertTrue(github_addon_count['total'] == 1)

def test_storage_region_count(self):
region = RegionFactory(_id='test1', name='Test Region #1')
region2 = RegionFactory(_id='test2', name='Test Region #2')
self._add_node_with_storage_region(region)
self._add_node_with_storage_region(region2)
self._add_node_with_storage_region(region2)

reporter = InstitutionalSummaryMonthlyReporter()
reports = list(reporter.report(self._yearmonth))
report = reports[0]

self.assertIsNotNone(report.storage_regions)
self.assertEqual(len(report.storage_regions), 3)

storage_region_count = next((l for l in report.storage_regions if l.name == 'Test Region #2'), None)
self.assertEqual(storage_region_count['total'], 2)

def _add_node_with_storage_region(self, region):
project = self._create_affiliated_project(is_public=True)
addon = project.get_addon('osfstorage')
addon.region = region
addon.save()
self._institution.storage_regions.add(region)

def test_department_count(self):
user = self._public_project.creator
user.add_or_update_affiliated_institution(self._institution, sso_department='Test Department #1')
user = self._private_project.creator
user.add_or_update_affiliated_institution(self._institution, sso_department='Test Department #2')

reporter = InstitutionalSummaryMonthlyReporter()
reports = list(reporter.report(self._yearmonth))
report = reports[0]

self.assertEqual(len(report.departments), 2)

departments_count = next((l for l in report.departments if l.name == 'Test Department #1'), None)
self.assertEqual(departments_count['total'], 1)
self.assertEqual(report.published_preprint_count, 1 if hasattr(Preprint, 'affiliated_institutions') else None)
self.assertEqual(report.monthly_logged_in_user_count, 1)
self.assertEqual(report.monthly_active_user_count, 1)

0 comments on commit bd4b9a0

Please sign in to comment.