From 51ffdb5a2018327bd771162404627d44507711f9 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Mon, 1 Jul 2024 10:25:01 +0530 Subject: [PATCH 01/15] add event tracking --- commcare_connect/events/__init__.py | 0 commcare_connect/events/apps.py | 6 + commcare_connect/events/models.py | 161 +++++++++++++++++++ commcare_connect/events/tasks.py | 36 +++++ commcare_connect/opportunity/tasks.py | 5 + commcare_connect/opportunity/views.py | 4 + commcare_connect/opportunity/visit_import.py | 3 + config/settings/base.py | 1 + 8 files changed, 216 insertions(+) create mode 100644 commcare_connect/events/__init__.py create mode 100644 commcare_connect/events/apps.py create mode 100644 commcare_connect/events/models.py create mode 100644 commcare_connect/events/tasks.py diff --git a/commcare_connect/events/__init__.py b/commcare_connect/events/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/commcare_connect/events/apps.py b/commcare_connect/events/apps.py new file mode 100644 index 00000000..c79a1afa --- /dev/null +++ b/commcare_connect/events/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class EventsAppConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "commcare_connect.events" diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py new file mode 100644 index 00000000..dcc7f5c6 --- /dev/null +++ b/commcare_connect/events/models.py @@ -0,0 +1,161 @@ +from abc import ABCMeta, abstractproperty +from dataclasses import dataclass, field, fields +from datetime import datetime + +from django.db import models +from django.utils.translation import gettext as _ + +from commcare_connect.users.models import User + +EVENT_TYPES = {"invite_sent": {}} + + +@dataclass(frozen=True) +class EventTypeChoice: + INVITE_SENT: tuple[str, str] = field(default=("invite_sent", _("Invite Sent"))) + RECORDS_APPROVED: tuple[str, str] = field(default=("records_approved", _("Records Approved"))) + RECORDS_FLAGGED: tuple[str, str] = field(default=("records_flagged", _("Records Flagged"))) + RECORDS_REJECTED: tuple[str, str] = field(default=("records_rejected", _("Records Rejected"))) + PAYMENT_APPROVED: tuple[str, str] = field(default=("payment_approved", _("Payment Approved"))) + PAYMENT_ACCRUED: tuple[str, str] = field(default=("payment_accrued", _("Payment Accrued"))) + PAYMENT_TRANSFERRED: tuple[str, str] = field(default=("payment_transferred", _("Payment Transferred"))) + NOTIFICATIONS_SENT: tuple[str, str] = field(default=("notifications_sent", _("Notifications Sent"))) + ADDITIONAL_BUDGET_ADDED: tuple[str, str] = field(default=("additional_budget_added", _("Additional Budget Added"))) + + @classmethod + @property + def choices(cls): + return [(field.default[0], field.default[1]) for field in fields(cls)] + + +class Event(models.Model): + # this allows referring to Event.Type without importing EventTypeChoice separately + Type = EventTypeChoice + + date_created = models.DateTimeField(auto_now_add=True, db_index=True) + event_type = models.CharField(max_length=40, choices=Type.choices) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) + opportunity = models.ForeignKey("Opportunity", on_delete=models.PROTECT, null=True) + + @classmethod + def track(cls, use_async=True): + """ + To track an event instantiate the object and call this method, + instead of calling save directly. + + If use_async is True, the event is queued in Redis and saved + via celery, otherwise it's saved directly. + """ + from commcare_connect.events.tasks import track_event + + track_event(cls, use_async=use_async) + + +@dataclass +class InferredEvent: + from commcare_connect.opportunity.models import Opportunity + + user: User + opportunity: Opportunity + date_created: datetime + event_type: str + + +class InferredEventSpec(metaclass=ABCMeta): + """ + Use this to define an Event that can be inferred + based on other models. + """ + + @abstractproperty + def model_cls(self): + """ + The source model class to infer the event from + for e.g. UserVisit + """ + raise NotImplementedError + + @abstractproperty + def event_type(self): + """ + Should be a tuple to indicate the name + for e.g. "RECORDS_FLAGGED", gettext("Records Flagged") + """ + raise NotImplementedError + + @abstractproperty + def event_filters(self): + """ + Should be a dict of the queryset filters + for e.g. {'flagged': True} for RecordsFlagged for UserVisit + """ + raise NotImplementedError + + @abstractproperty + def user(self): + """ + The field corresponding to user on source model. + """ + raise NotImplementedError + + @abstractproperty + def date_created(self): + """ + The field corresponding to user date_created. + """ + raise NotImplementedError + + @abstractproperty + def opportunity(self): + """ + The field corresponding to user opportunity, could be None. + """ + raise NotImplementedError + + def get_events(self, user=None, from_date=None, to_date=None): + filters = {} + filters.update(self.event_filters) + + if user: + filters.update({self.user: user}) + if from_date: + filters.update({f"{self.date_created}__gte": from_date}) + if to_date: + filters.update({f"{self.date_created}__lte": to_date}) + + events = ( + self.model_cls.objects.filter(**filters).values(self.user, self.opportunity, self.date_created).iterator() + ) + for event in events: + yield InferredEvent( + event_type=self.event_type[0], + user=event[self.user], + date_created=event[self.date_created], + opportunity=event.get(self.opportunity, None), + ) + + +class RecordsFlagged(InferredEventSpec): + event_type = ("RECORDS_FLAGGED", _("Records Flagged")) + model_cls = "UserVisit" + event_filters = {"flagged": True} + user = "user" + date_created = "visit_date" + opportunity = "opportunity" + + +INFERRED_EVENT_SPECS = [RecordsFlagged()] + + +def get_events(user=None, from_date=None, to_date=None): + filters = { + "user": user, + "date_created__gte": from_date, + "date_created__lte": to_date, + } + filters = {k: v for k, v in filters.items() if v is not None} + raw_events = Event.objects.filter(**filters).all() + inferred_events = [] + for event_spec in INFERRED_EVENT_SPECS: + inferred_events += list(event_spec.get_events(user=user, from_date=from_date, to_date=to_date)) + return list(raw_events) + inferred_events diff --git a/commcare_connect/events/tasks.py b/commcare_connect/events/tasks.py new file mode 100644 index 00000000..585629fb --- /dev/null +++ b/commcare_connect/events/tasks.py @@ -0,0 +1,36 @@ +import pickle +from datetime import datetime + +from django.db import transaction +from django_redis import get_redis_connection + +from commcare_connect.events.models import Event +from config import celery_app + +REDIS_EVENTS_QUEUE = "events_queue" + + +@celery_app.task +def process_events_batch(): + redis_conn = get_redis_connection("default") + events = redis_conn.lrange(REDIS_EVENTS_QUEUE, 0, -1) + if not events: + return + + with transaction.atomic(): + event_objs = [] + for event in events: + event_objs.append(pickle.loads(event)) + Event.objects.bulk_create(event_objs) + + redis_conn.ltrim(REDIS_EVENTS_QUEUE, len(events), -1) + + +def track_event(event_obj, use_async=True): + event_obj.date_created = datetime.now() + if use_async: + redis_conn = get_redis_connection("default") + serialized_event = pickle.dumps(event_obj) + redis_conn.rpush(REDIS_EVENTS_QUEUE, serialized_event) + else: + event_obj.save() diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index 86265516..4e3632cc 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -101,6 +101,11 @@ def invite_user(user_id, opportunity_access_id): ), ) send_message(message) + from commcare_connect.events.models import Event + + Event(event_type=Event.Type.INVITE_SENT, user=user, opportunity=opportunity_access.opportunity).track( + use_async=False + ) @celery_app.task() diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py index 42192784..0d7b0cd1 100644 --- a/commcare_connect/opportunity/views.py +++ b/commcare_connect/opportunity/views.py @@ -20,6 +20,7 @@ from django_tables2.export import TableExport from geopy import distance +from commcare_connect.events.models import Event from commcare_connect.form_receiver.serializers import XFormSerializer from commcare_connect.opportunity.forms import ( AddBudgetExistingUsersForm, @@ -382,6 +383,7 @@ def add_budget_existing_users(request, org_slug=None, pk=None): opportunity.total_budget += ocl.payment_unit.amount * additional_visits opportunity.save() return redirect("opportunity:detail", org_slug, pk) + Event(event_type=Event.Type.ADDITIONAL_BUDGET_ADDED, opportunity=opportunity).track() return render( request, @@ -750,6 +752,7 @@ def approve_visit(request, org_slug=None, pk=None): opp_id = user_visit.opportunity_id access = OpportunityAccess.objects.get(user_id=user_visit.user_id, opportunity_id=opp_id) update_payment_accrued(opportunity=access.opportunity, users=[access.user]) + Event(event_type=Event.Type.RECORDS_APPROVED, user=user_visit.user, opportunity=access.opportunity).track() return redirect("opportunity:user_visits_list", org_slug=org_slug, opp_id=user_visit.opportunity.id, pk=access.id) @@ -760,6 +763,7 @@ def reject_visit(request, org_slug=None, pk=None): user_visit.save() access = OpportunityAccess.objects.get(user_id=user_visit.user_id, opportunity_id=user_visit.opportunity_id) update_payment_accrued(opportunity=access.opportunity, users=[access.user]) + Event(event_type=Event.Type.RECORDS_REJECTED, user=user_visit.user, opportunity=access.opportunity).track() return redirect("opportunity:user_visits_list", org_slug=org_slug, opp_id=user_visit.opportunity_id, pk=access.id) diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py index d7e50e43..930d34fa 100644 --- a/commcare_connect/opportunity/visit_import.py +++ b/commcare_connect/opportunity/visit_import.py @@ -7,6 +7,7 @@ from django.db import transaction from tablib import Dataset +from commcare_connect.events.models import Event from commcare_connect.opportunity.models import ( CompletedWork, CompletedWorkStatus, @@ -143,6 +144,7 @@ def update_payment_accrued(opportunity: Opportunity, users): access.payment_accrued += approved_count * completed_work.payment_unit.amount completed_work.save() access.save() + Event(event_type=Event.Type.PAYMENT_ACCRUED, user=access.user, opportunity=access.opportunity).track() def get_status_by_visit_id(dataset) -> dict[int, VisitValidationStatus]: @@ -238,6 +240,7 @@ def _bulk_update_payments(opportunity: Opportunity, imported_data: Dataset) -> P payment = Payment.objects.create(opportunity_access=access, amount=amount) seen_users.add(username) payment_ids.append(payment.pk) + Event(event_type=Event.Type.PAYMENT_TRANSFERRED, user=access.user, opportunity=opportunity).track() missing_users = set(usernames) - seen_users send_payment_notification.delay(opportunity.id, payment_ids) return PaymentImportStatus(seen_users, missing_users) diff --git a/config/settings/base.py b/config/settings/base.py index 50175ddc..83c4db44 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -69,6 +69,7 @@ LOCAL_APPS = [ "commcare_connect.commcarehq_provider", + "commcare_connect.events", "commcare_connect.form_receiver", "commcare_connect.opportunity", "commcare_connect.organization", From f97b2c5ac2f4b034f0647fcac507be25543f1a7c Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Sun, 14 Jul 2024 18:14:40 +0530 Subject: [PATCH 02/15] Add mobile endpoint, tests --- .../events/migrations/0001_initial.py | 53 ++++++++++++++++ .../events/migrations/__init__.py | 0 commcare_connect/events/models.py | 43 ++++--------- commcare_connect/events/tasks.py | 37 ++++++++---- commcare_connect/events/tests.py | 60 +++++++++++++++++++ commcare_connect/events/types.py | 28 +++++++++ commcare_connect/events/views.py | 34 +++++++++++ commcare_connect/opportunity/tasks.py | 1 + config/api_router.py | 2 + 9 files changed, 215 insertions(+), 43 deletions(-) create mode 100644 commcare_connect/events/migrations/0001_initial.py create mode 100644 commcare_connect/events/migrations/__init__.py create mode 100644 commcare_connect/events/tests.py create mode 100644 commcare_connect/events/types.py create mode 100644 commcare_connect/events/views.py diff --git a/commcare_connect/events/migrations/0001_initial.py b/commcare_connect/events/migrations/0001_initial.py new file mode 100644 index 00000000..93bc7ab0 --- /dev/null +++ b/commcare_connect/events/migrations/0001_initial.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.5 on 2024-07-13 14:45 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("opportunity", "0044_opportunityverificationflags"), + ] + + operations = [ + migrations.CreateModel( + name="Event", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("date_created", models.DateTimeField(auto_now_add=True, db_index=True)), + ( + "event_type", + models.CharField( + choices=[ + ("invite_sent", "Invite Sent"), + ("records_approved", "Records Approved"), + ("records_flagged", "Records Flagged"), + ("records_rejected", "Records Rejected"), + ("payment_approved", "Payment Approved"), + ("payment_accrued", "Payment Accrued"), + ("payment_transferred", "Payment Transferred"), + ("notifications_sent", "Notifications Sent"), + ("additional_budget_added", "Additional Budget Added"), + ], + max_length=40, + ), + ), + ( + "opportunity", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.PROTECT, to="opportunity.opportunity" + ), + ), + ( + "user", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ], + ), + ] diff --git a/commcare_connect/events/migrations/__init__.py b/commcare_connect/events/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index dcc7f5c6..0a675220 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -1,5 +1,5 @@ from abc import ABCMeta, abstractproperty -from dataclasses import dataclass, field, fields +from dataclasses import dataclass from datetime import datetime from django.db import models @@ -7,40 +7,23 @@ from commcare_connect.users.models import User -EVENT_TYPES = {"invite_sent": {}} - - -@dataclass(frozen=True) -class EventTypeChoice: - INVITE_SENT: tuple[str, str] = field(default=("invite_sent", _("Invite Sent"))) - RECORDS_APPROVED: tuple[str, str] = field(default=("records_approved", _("Records Approved"))) - RECORDS_FLAGGED: tuple[str, str] = field(default=("records_flagged", _("Records Flagged"))) - RECORDS_REJECTED: tuple[str, str] = field(default=("records_rejected", _("Records Rejected"))) - PAYMENT_APPROVED: tuple[str, str] = field(default=("payment_approved", _("Payment Approved"))) - PAYMENT_ACCRUED: tuple[str, str] = field(default=("payment_accrued", _("Payment Accrued"))) - PAYMENT_TRANSFERRED: tuple[str, str] = field(default=("payment_transferred", _("Payment Transferred"))) - NOTIFICATIONS_SENT: tuple[str, str] = field(default=("notifications_sent", _("Notifications Sent"))) - ADDITIONAL_BUDGET_ADDED: tuple[str, str] = field(default=("additional_budget_added", _("Additional Budget Added"))) - - @classmethod - @property - def choices(cls): - return [(field.default[0], field.default[1]) for field in fields(cls)] +from . import types class Event(models.Model): - # this allows referring to Event.Type without importing EventTypeChoice separately - Type = EventTypeChoice + from commcare_connect.opportunity.models import Opportunity + + # this allows referring to event types in this style: Event.Type.INVITE_SENT + Type = types date_created = models.DateTimeField(auto_now_add=True, db_index=True) - event_type = models.CharField(max_length=40, choices=Type.choices) + event_type = models.CharField(max_length=40, choices=types.EVENT_TYPE_CHOICES) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) - opportunity = models.ForeignKey("Opportunity", on_delete=models.PROTECT, null=True) + opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) - @classmethod - def track(cls, use_async=True): + def track(self, use_async=True): """ - To track an event instantiate the object and call this method, + To track an event, instantiate the object and call this method, instead of calling save directly. If use_async is True, the event is queued in Redis and saved @@ -48,7 +31,7 @@ def track(cls, use_async=True): """ from commcare_connect.events.tasks import track_event - track_event(cls, use_async=use_async) + track_event(self, use_async=use_async) @dataclass @@ -64,7 +47,7 @@ class InferredEvent: class InferredEventSpec(metaclass=ABCMeta): """ Use this to define an Event that can be inferred - based on other models. + based on other models. See RecordsFlagged for example """ @abstractproperty @@ -136,7 +119,7 @@ def get_events(self, user=None, from_date=None, to_date=None): class RecordsFlagged(InferredEventSpec): - event_type = ("RECORDS_FLAGGED", _("Records Flagged")) + event_type = (types.RECORDS_FLAGGED, _("Records Flagged")) model_cls = "UserVisit" event_filters = {"flagged": True} user = "user" diff --git a/commcare_connect/events/tasks.py b/commcare_connect/events/tasks.py index 585629fb..c97a4253 100644 --- a/commcare_connect/events/tasks.py +++ b/commcare_connect/events/tasks.py @@ -10,27 +10,38 @@ REDIS_EVENTS_QUEUE = "events_queue" +class EventQueue: + def __init__(self): + self.redis_conn = get_redis_connection("default") + + def push(self, event_obj): + serialized_event = pickle.dumps(event_obj) + self.redis_conn.rpush(REDIS_EVENTS_QUEUE, serialized_event) + + def pop(self): + events = [pickle.loads(event) for event in self.redis_conn.lrange(REDIS_EVENTS_QUEUE, 0, -1)] + self.redis_conn.ltrim(REDIS_EVENTS_QUEUE, len(events), -1) + return events + + @celery_app.task def process_events_batch(): - redis_conn = get_redis_connection("default") - events = redis_conn.lrange(REDIS_EVENTS_QUEUE, 0, -1) + event_queue = EventQueue() + events = event_queue.pop() if not events: return - - with transaction.atomic(): - event_objs = [] + try: + with transaction.atomic(): + Event.objects.bulk_create(events) + except Exception as e: for event in events: - event_objs.append(pickle.loads(event)) - Event.objects.bulk_create(event_objs) - - redis_conn.ltrim(REDIS_EVENTS_QUEUE, len(events), -1) + event_queue.push(event) + raise e def track_event(event_obj, use_async=True): - event_obj.date_created = datetime.now() + event_obj.date_created = datetime.utcnow() if use_async: - redis_conn = get_redis_connection("default") - serialized_event = pickle.dumps(event_obj) - redis_conn.rpush(REDIS_EVENTS_QUEUE, serialized_event) + EventQueue().push(event_obj) else: event_obj.save() diff --git a/commcare_connect/events/tests.py b/commcare_connect/events/tests.py new file mode 100644 index 00000000..98e68931 --- /dev/null +++ b/commcare_connect/events/tests.py @@ -0,0 +1,60 @@ +from rest_framework.test import APIClient + +from commcare_connect.opportunity.models import Opportunity +from commcare_connect.users.models import User + +from .models import Event +from .tasks import EventQueue, process_events_batch + + +def test_post_events(mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity): + api_client.force_authenticate(mobile_user_with_connect_link) + response = api_client.post( + "/api/events/", + data=[ + { + "event_type": "invalid_event_name", + "user": mobile_user_with_connect_link.pk, + "opportunity": opportunity.pk, + } + ], + format="json", + ) + assert response.status_code == 400 + assert Event.objects.count() == 0 + response = api_client.post( + "/api/events/", + data=[ + { + "event_type": Event.Type.INVITE_SENT, + "user": mobile_user_with_connect_link.pk, + "opportunity": opportunity.pk, + }, + { + "event_type": Event.Type.RECORDS_APPROVED, + "user": mobile_user_with_connect_link.pk, + "opportunity": opportunity.pk, + }, + ], + format="json", + ) + assert response.status_code == 201 + assert Event.objects.count() == 2 + + +def test_event_queue(mobile_user_with_connect_link: User, opportunity: Opportunity): + event_queue = EventQueue() + assert event_queue.pop() == [] + + # queue the event + event = Event(event_type=Event.Type.INVITE_SENT, user=mobile_user_with_connect_link, opportunity=opportunity) + event.track() + queued_events = event_queue.pop() + process_events_batch() + assert len(queued_events) == 1 + assert Event.objects.count() == 0 + # process the batch + event.track() + process_events_batch() + assert Event.objects.count() == 1 + assert Event.objects.first().user == event.user diff --git a/commcare_connect/events/types.py b/commcare_connect/events/types.py new file mode 100644 index 00000000..8162e97a --- /dev/null +++ b/commcare_connect/events/types.py @@ -0,0 +1,28 @@ +from django.utils.translation import gettext as _ + +# Server/Web events +INVITE_SENT = "invite_sent" +RECORDS_APPROVED = "records_approved" +RECORDS_REJECTED = "records_rejected" +PAYMENT_APPROVED = "payment_approved" +PAYMENT_ACCRUED = "payment_accrued" +PAYMENT_TRANSFERRED = "payment_transferred" +NOTIFICATIONS_SENT = "notifications_sent" +ADDITIONAL_BUDGET_ADDED = "additional_budget_added" + +EVENT_TYPES = { + INVITE_SENT: _("Invite Sent"), + RECORDS_APPROVED: _("Records Approved"), + RECORDS_REJECTED: _("Records Rejected"), + PAYMENT_APPROVED: _("Payment Approved"), + PAYMENT_ACCRUED: _("Payment Accrued"), + PAYMENT_TRANSFERRED: _("Payment Transferred"), + NOTIFICATIONS_SENT: _("Notifications Sent"), + ADDITIONAL_BUDGET_ADDED: _("Additional Budget Added"), +} + +EVENT_TYPE_CHOICES = list(EVENT_TYPES.items()) + + +# Inferred Events +RECORDS_FLAGGED = "records_flagged" diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py new file mode 100644 index 00000000..25d010c7 --- /dev/null +++ b/commcare_connect/events/views.py @@ -0,0 +1,34 @@ +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from rest_framework import serializers, status +from rest_framework.generics import ListCreateAPIView +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response + +from .models import Event + + +class EventSerializer(serializers.ModelSerializer): + class Meta: + model = Event + fields = ["date_created", "event_type", "user", "opportunity"] + + +@method_decorator(csrf_exempt, name="dispatch") +class EventListCreateView(ListCreateAPIView): + queryset = Event.objects.all() + serializer_class = EventSerializer + permission_classes = [IsAuthenticated] + + def create(self, request, *args, **kwargs): + if not isinstance(request.data, list): + return Response({"error": "Expected a list of items"}, status=status.HTTP_400_BAD_REQUEST) + + serializer = self.get_serializer(data=request.data, many=True) + serializer.is_valid(raise_exception=True) + + event_objects = [Event(**item) for item in serializer.validated_data] + Event.objects.bulk_create(event_objects) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index 4e3632cc..fd6004f2 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -104,6 +104,7 @@ def invite_user(user_id, opportunity_access_id): from commcare_connect.events.models import Event Event(event_type=Event.Type.INVITE_SENT, user=user, opportunity=opportunity_access.opportunity).track( + # this is already in async worker, so user_async is False use_async=False ) diff --git a/config/api_router.py b/config/api_router.py index e3fd05cf..42cd6eb7 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -2,6 +2,7 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter, SimpleRouter +from commcare_connect.events.views import EventListCreateView from commcare_connect.form_receiver.views import FormReceiver from commcare_connect.opportunity.api.views import ( ClaimOpportunityView, @@ -25,6 +26,7 @@ app_name = "api" urlpatterns = [ path("", include(router.urls)), + path("events/", EventListCreateView.as_view(), name="create_events"), path("receiver/", FormReceiver.as_view(), name="receiver"), path("opportunity//learn_progress", UserLearnProgressView.as_view(), name="learn_progress"), path("opportunity//claim", ClaimOpportunityView.as_view()), From 4a2467c956f4769321bc5757de49e4781ffc330f Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Mon, 29 Jul 2024 17:08:53 +0530 Subject: [PATCH 03/15] Implement Events report --- commcare_connect/events/models.py | 8 +- commcare_connect/events/types.py | 1 + commcare_connect/events/urls.py | 9 +++ commcare_connect/events/views.py | 53 +++++++++++++ .../templates/events/event_table_htmx.html | 20 +++++ .../templates/events/event_table_partial.html | 3 + .../templates/events/htmx_table.html | 79 +++++++++++++++++++ config/settings/base.py | 4 + config/urls.py | 1 + requirements/base.in | 1 + 10 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 commcare_connect/events/urls.py create mode 100644 commcare_connect/templates/events/event_table_htmx.html create mode 100644 commcare_connect/templates/events/event_table_partial.html create mode 100644 commcare_connect/templates/events/htmx_table.html diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index 0a675220..e7d0946b 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -10,6 +10,12 @@ from . import types +def get_event_type_choices(): + # A callable avoids migration getting created + # each time when EVENT_TYPE_CHOICES is edited + return types.EVENT_TYPE_CHOICES + + class Event(models.Model): from commcare_connect.opportunity.models import Opportunity @@ -17,7 +23,7 @@ class Event(models.Model): Type = types date_created = models.DateTimeField(auto_now_add=True, db_index=True) - event_type = models.CharField(max_length=40, choices=types.EVENT_TYPE_CHOICES) + event_type = models.CharField(max_length=40, choices=get_event_type_choices()) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) diff --git a/commcare_connect/events/types.py b/commcare_connect/events/types.py index 8162e97a..67253b23 100644 --- a/commcare_connect/events/types.py +++ b/commcare_connect/events/types.py @@ -21,6 +21,7 @@ ADDITIONAL_BUDGET_ADDED: _("Additional Budget Added"), } + EVENT_TYPE_CHOICES = list(EVENT_TYPES.items()) diff --git a/commcare_connect/events/urls.py b/commcare_connect/events/urls.py new file mode 100644 index 00000000..f9b9db62 --- /dev/null +++ b/commcare_connect/events/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from .views import EventListView + +app_name = "events" + +urlpatterns = [ + path("", view=EventListView.as_view(), name="event_htmx"), +] diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index 25d010c7..31fe159b 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -1,10 +1,16 @@ +import django_tables2 as tables from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt +from django_filters import ChoiceFilter, FilterSet +from django_filters.views import FilterView from rest_framework import serializers, status from rest_framework.generics import ListCreateAPIView from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from commcare_connect.opportunity.forms import DateRanges +from commcare_connect.opportunity.models import Opportunity + from .models import Event @@ -32,3 +38,50 @@ def create(self, request, *args, **kwargs): headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + +class EventTable(tables.Table): + date_created = tables.Column(verbose_name="Time") + + class Meta: + model = Event + template_name = "events/htmx_table.html" + fields = ("user", "opportunity", "event_type", "date_created") + + +class EventFilter(FilterSet): + date_range = ChoiceFilter(choices=DateRanges.choices, method="filter_by_date_range", label="Date Range") + + class Meta: + model = Event + fields = ["opportunity", "user", "event_type"] + + def filter_by_date_range(self, queryset, name, value): + if not value: + return queryset + + try: + date_range = DateRanges(value) + return queryset.filter( + date_created__gte=date_range.get_cutoff_date(), + ) + except ValueError: + return queryset + + +class EventListView(tables.SingleTableMixin, FilterView): + table_class = EventTable + queryset = Event.objects.all() + filterset_class = EventFilter + paginate_by = 20 + + def get_template_names(self): + if self.request.htmx: + template_name = "events/event_table_partial.html" + else: + template_name = "events/event_table_htmx.html" + + return template_name + + def get_queryset(self): + return Event.objects.filter(opportunity__in=Opportunity.objects.filter(organization=self.request.org)) diff --git a/commcare_connect/templates/events/event_table_htmx.html b/commcare_connect/templates/events/event_table_htmx.html new file mode 100644 index 00000000..79a8d97c --- /dev/null +++ b/commcare_connect/templates/events/event_table_htmx.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} + +{% load render_table from django_tables2 %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block content %} +

Events

+ +
+ {% crispy filter.form %} +
+ + {% render_table table %} +{% endblock %} diff --git a/commcare_connect/templates/events/event_table_partial.html b/commcare_connect/templates/events/event_table_partial.html new file mode 100644 index 00000000..29f49998 --- /dev/null +++ b/commcare_connect/templates/events/event_table_partial.html @@ -0,0 +1,3 @@ +{% load render_table from django_tables2 %} + +{% render_table table %} diff --git a/commcare_connect/templates/events/htmx_table.html b/commcare_connect/templates/events/htmx_table.html new file mode 100644 index 00000000..768d7c5f --- /dev/null +++ b/commcare_connect/templates/events/htmx_table.html @@ -0,0 +1,79 @@ +{% extends "django_tables2/bootstrap5.html" %} + +{% load django_tables2 %} +{% load i18n %} + +{% block extra_head %} + +{% endblock %} + +{% block table.thead %} + {% if table.show_header %} + + + {% for column in table.columns %} + + {{ column.header }} + + {% endfor %} + + + {% endif %} +{% endblock table.thead %} + + +{% block pagination.previous %} + +{% endblock pagination.previous %} +{% block pagination.range %} + {% for p in table.page|table_page_range:table.paginator %} +
  • +
    + {{ p }} +
    +
    • + {% endfor %} +{% endblock pagination.range %} +{% block pagination.next %} + +{% endblock pagination.next %} diff --git a/config/settings/base.py b/config/settings/base.py index 83c4db44..355ce829 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -65,6 +65,7 @@ "drf_spectacular", "oauth2_provider", "django_tables2", + "django_filters", ] LOCAL_APPS = [ @@ -116,6 +117,7 @@ "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.locale.LocaleMiddleware", "django.middleware.common.CommonMiddleware", + "django_htmx.middleware.HtmxMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "commcare_connect.users.middleware.OrganizationMiddleware", @@ -303,6 +305,8 @@ } } +FILTERS_EMPTY_CHOICE_LABEL = "All" + DJANGO_TABLES2_TEMPLATE = "tables/tabbed_table.html" DJANGO_TABLES2_TABLE_ATTRS = { "class": "table table-bordered mb-0", diff --git a/config/urls.py b/config/urls.py index 1dd00bbe..76340136 100644 --- a/config/urls.py +++ b/config/urls.py @@ -19,6 +19,7 @@ # Your stuff: custom urls includes go here path("a//", include("commcare_connect.organization.urls")), path("a//opportunity/", include("commcare_connect.opportunity.urls", namespace="opportunity")), + path("a//events/", include("commcare_connect.events.urls", namespace="events")), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) # API URLS diff --git a/requirements/base.in b/requirements/base.in index ebbd1547..50e45baa 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -30,6 +30,7 @@ django-cors-headers # DRF-spectacular for api documentation drf-spectacular django-tables2 +django-filter # Temporary # ------------------------------------------------------------------------------- From dadbeaef5b6c531af3ae74f03a67fc6f31d65b15 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Wed, 31 Jul 2024 20:14:46 +0530 Subject: [PATCH 04/15] Fix sort reset, add icons --- .../templates/events/htmx_table.html | 102 ++++++++++-------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/commcare_connect/templates/events/htmx_table.html b/commcare_connect/templates/events/htmx_table.html index 768d7c5f..157e01a6 100644 --- a/commcare_connect/templates/events/htmx_table.html +++ b/commcare_connect/templates/events/htmx_table.html @@ -23,12 +23,19 @@ {% for column in table.columns %} {{ column.header }} + {% if column.order_by_alias == column.order_by_alias.next %} + + {% elif column.order_by_alias|slice:":1" == "-" %} + + {% else %} + + {% endif %} {% endfor %} @@ -36,44 +43,55 @@ {% endif %} {% endblock table.thead %} - -{% block pagination.previous %} - -{% endblock pagination.previous %} -{% block pagination.range %} - {% for p in table.page|table_page_range:table.paginator %} -
  • -
    - {{ p }} -
    -
    • - {% endfor %} -{% endblock pagination.range %} -{% block pagination.next %} - -{% endblock pagination.next %} +{% block pagination %} + {% if table.page and table.paginator.num_pages > 1 %} + + {% endif %} +{% endblock pagination %} From 99467b8b22de5f7f9cbb1dd4e0255fc8712072f0 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Wed, 31 Jul 2024 20:29:58 +0530 Subject: [PATCH 05/15] remove auto_now_add --- commcare_connect/events/migrations/0001_initial.py | 2 +- commcare_connect/events/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commcare_connect/events/migrations/0001_initial.py b/commcare_connect/events/migrations/0001_initial.py index 93bc7ab0..de5ce67d 100644 --- a/commcare_connect/events/migrations/0001_initial.py +++ b/commcare_connect/events/migrations/0001_initial.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): name="Event", fields=[ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("date_created", models.DateTimeField(auto_now_add=True, db_index=True)), + ("date_created", models.DateTimeField(db_index=True)), ( "event_type", models.CharField( diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index e7d0946b..36acbceb 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -22,7 +22,7 @@ class Event(models.Model): # this allows referring to event types in this style: Event.Type.INVITE_SENT Type = types - date_created = models.DateTimeField(auto_now_add=True, db_index=True) + date_created = models.DateTimeField(db_index=True) event_type = models.CharField(max_length=40, choices=get_event_type_choices()) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) From 6be106ab308018970f4a011dc0c127099e2379d0 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Thu, 1 Aug 2024 19:05:37 +0530 Subject: [PATCH 06/15] Make user field a select2 widget --- .../events/migrations/0001_initial.py | 6 ++++++ commcare_connect/events/models.py | 7 +++++++ commcare_connect/events/views.py | 16 +++++++++++++--- .../templates/events/event_table_htmx.html | 15 +++++++++++++++ .../templates/events/htmx_table.html | 14 -------------- commcare_connect/users/urls.py | 2 ++ commcare_connect/users/views.py | 11 +++++++++++ config/settings/base.py | 2 ++ requirements/base.in | 1 + 9 files changed, 57 insertions(+), 17 deletions(-) diff --git a/commcare_connect/events/migrations/0001_initial.py b/commcare_connect/events/migrations/0001_initial.py index de5ce67d..836359ff 100644 --- a/commcare_connect/events/migrations/0001_initial.py +++ b/commcare_connect/events/migrations/0001_initial.py @@ -48,6 +48,12 @@ class Migration(migrations.Migration): null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL ), ), + ( + "organization", + models.ForeignKey( + on_delete=models.CASCADE, related_name="events", related_query_name="event", to="organization.organization" + ), + ) ], ), ] diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index 36acbceb..249d79ee 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -5,6 +5,7 @@ from django.db import models from django.utils.translation import gettext as _ +from commcare_connect.organization.models import Organization from commcare_connect.users.models import User from . import types @@ -26,6 +27,12 @@ class Event(models.Model): event_type = models.CharField(max_length=40, choices=get_event_type_choices()) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + related_name="events", + related_query_name="event", + ) def track(self, use_async=True): """ diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index 31fe159b..1b11cd0f 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -1,7 +1,8 @@ import django_tables2 as tables +from dal.autocomplete import ModelSelect2 from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt -from django_filters import ChoiceFilter, FilterSet +from django_filters import ChoiceFilter, FilterSet, ModelChoiceFilter from django_filters.views import FilterView from rest_framework import serializers, status from rest_framework.generics import ListCreateAPIView @@ -9,7 +10,7 @@ from rest_framework.response import Response from commcare_connect.opportunity.forms import DateRanges -from commcare_connect.opportunity.models import Opportunity +from commcare_connect.users.models import User from .models import Event @@ -51,6 +52,15 @@ class Meta: class EventFilter(FilterSet): date_range = ChoiceFilter(choices=DateRanges.choices, method="filter_by_date_range", label="Date Range") + user = ModelChoiceFilter( + queryset=User.objects.all(), + widget=ModelSelect2( + url="users:search", + attrs={ + "data-placeholder": "All", + }, + ), + ) class Meta: model = Event @@ -84,4 +94,4 @@ def get_template_names(self): return template_name def get_queryset(self): - return Event.objects.filter(opportunity__in=Opportunity.objects.filter(organization=self.request.org)) + return Event.objects.filter(organization=self.request.org) diff --git a/commcare_connect/templates/events/event_table_htmx.html b/commcare_connect/templates/events/event_table_htmx.html index 79a8d97c..ec919d1a 100644 --- a/commcare_connect/templates/events/event_table_htmx.html +++ b/commcare_connect/templates/events/event_table_htmx.html @@ -18,3 +18,18 @@

    Events

    {% render_table table %} {% endblock %} + +{% block javascript %} +{{ block.super }} + + +{% endblock %} + +{% block css %} +{{ block.super }} + +{% endblock %} diff --git a/commcare_connect/templates/events/htmx_table.html b/commcare_connect/templates/events/htmx_table.html index 157e01a6..65a47bff 100644 --- a/commcare_connect/templates/events/htmx_table.html +++ b/commcare_connect/templates/events/htmx_table.html @@ -3,20 +3,6 @@ {% load django_tables2 %} {% load i18n %} -{% block extra_head %} - -{% endblock %} - {% block table.thead %} {% if table.show_header %} diff --git a/commcare_connect/users/urls.py b/commcare_connect/users/urls.py index 94205db1..9d1417dc 100644 --- a/commcare_connect/users/urls.py +++ b/commcare_connect/users/urls.py @@ -2,6 +2,7 @@ from commcare_connect.users.views import ( SMSStatusCallbackView, + UserSearchView, accept_invite, create_user_link_view, demo_user_tokens, @@ -21,4 +22,5 @@ path("accept_invite//", view=accept_invite, name="accept_invite"), path("demo_users/", view=demo_user_tokens, name="demo_users"), path("sms_status_callback/", SMSStatusCallbackView.as_view(), name="sms_status_callback"), + path("user_search/", UserSearchView.as_view(), name="search"), ] diff --git a/commcare_connect/users/views.py b/commcare_connect/users/views.py index 221bd33c..0653db83 100644 --- a/commcare_connect/users/views.py +++ b/commcare_connect/users/views.py @@ -1,4 +1,5 @@ from allauth.account.models import transaction +from dal.autocomplete import Select2QuerySetView from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required, user_passes_test from django.contrib.auth.mixins import LoginRequiredMixin @@ -161,3 +162,13 @@ def post(self, *args, **kwargs): user_invite.status = UserInviteStatus.sms_not_delivered user_invite.save() return Response(status=200) + + +class UserSearchView(LoginRequiredMixin, Select2QuerySetView): + def get_queryset(self): + if not self.request.user.is_authenticated: + return User.objects.none() + queryset = User.objects.filter(opportunityaccess__opportunity__organization=self.request.org).distinct() + if self.q: + queryset = queryset.filter(name__istartswith=self.q) + return queryset diff --git a/config/settings/base.py b/config/settings/base.py index 7b7d28fe..0e2f90d1 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -48,6 +48,8 @@ "django.contrib.messages", "whitenoise.runserver_nostatic", "django.contrib.staticfiles", + "dal", + "dal_select2", # "django.contrib.humanize", # Handy template tags "django.contrib.admin", "django.forms", diff --git a/requirements/base.in b/requirements/base.in index 50e45baa..fda2bbf0 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -31,6 +31,7 @@ django-cors-headers drf-spectacular django-tables2 django-filter +django-autocomplete-light # Temporary # ------------------------------------------------------------------------------- From a7b1db55397940e5ff6bfd166a730b560d2a0967 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 2 Aug 2024 18:43:51 +0530 Subject: [PATCH 07/15] don't constraint on event type.choices --- .../events/migrations/0001_initial.py | 12 +---- commcare_connect/events/models.py | 14 +++--- commcare_connect/events/types.py | 44 +++++++++---------- commcare_connect/events/views.py | 1 + 4 files changed, 30 insertions(+), 41 deletions(-) diff --git a/commcare_connect/events/migrations/0001_initial.py b/commcare_connect/events/migrations/0001_initial.py index 836359ff..699d4211 100644 --- a/commcare_connect/events/migrations/0001_initial.py +++ b/commcare_connect/events/migrations/0001_initial.py @@ -22,18 +22,8 @@ class Migration(migrations.Migration): ( "event_type", models.CharField( - choices=[ - ("invite_sent", "Invite Sent"), - ("records_approved", "Records Approved"), - ("records_flagged", "Records Flagged"), - ("records_rejected", "Records Rejected"), - ("payment_approved", "Payment Approved"), - ("payment_accrued", "Payment Accrued"), - ("payment_transferred", "Payment Transferred"), - ("notifications_sent", "Notifications Sent"), - ("additional_budget_added", "Additional Budget Added"), - ], max_length=40, + db_index=True ), ), ( diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index 249d79ee..8d3c3710 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -5,18 +5,13 @@ from django.db import models from django.utils.translation import gettext as _ +from commcare_connect.cache import quickcache from commcare_connect.organization.models import Organization from commcare_connect.users.models import User from . import types -def get_event_type_choices(): - # A callable avoids migration getting created - # each time when EVENT_TYPE_CHOICES is edited - return types.EVENT_TYPE_CHOICES - - class Event(models.Model): from commcare_connect.opportunity.models import Opportunity @@ -24,7 +19,7 @@ class Event(models.Model): Type = types date_created = models.DateTimeField(db_index=True) - event_type = models.CharField(max_length=40, choices=get_event_type_choices()) + event_type = models.CharField(max_length=40, db_index=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) organization = models.ForeignKey( @@ -34,6 +29,11 @@ class Event(models.Model): related_query_name="event", ) + @classmethod + @quickcache([], timeout=60 * 60) + def get_all_event_types(cls): + return set(cls.objects.values_list("event_type", flat=True).distinct()) | set(types.EVENT_TYPES) + def track(self, use_async=True): """ To track an event, instantiate the object and call this method, diff --git a/commcare_connect/events/types.py b/commcare_connect/events/types.py index 67253b23..5a0b6cb2 100644 --- a/commcare_connect/events/types.py +++ b/commcare_connect/events/types.py @@ -1,29 +1,27 @@ from django.utils.translation import gettext as _ # Server/Web events -INVITE_SENT = "invite_sent" -RECORDS_APPROVED = "records_approved" -RECORDS_REJECTED = "records_rejected" -PAYMENT_APPROVED = "payment_approved" -PAYMENT_ACCRUED = "payment_accrued" -PAYMENT_TRANSFERRED = "payment_transferred" -NOTIFICATIONS_SENT = "notifications_sent" -ADDITIONAL_BUDGET_ADDED = "additional_budget_added" - -EVENT_TYPES = { - INVITE_SENT: _("Invite Sent"), - RECORDS_APPROVED: _("Records Approved"), - RECORDS_REJECTED: _("Records Rejected"), - PAYMENT_APPROVED: _("Payment Approved"), - PAYMENT_ACCRUED: _("Payment Accrued"), - PAYMENT_TRANSFERRED: _("Payment Transferred"), - NOTIFICATIONS_SENT: _("Notifications Sent"), - ADDITIONAL_BUDGET_ADDED: _("Additional Budget Added"), -} - - -EVENT_TYPE_CHOICES = list(EVENT_TYPES.items()) +INVITE_SENT = _("Invite Sent") +RECORDS_APPROVED = _("Records Approved") +RECORDS_REJECTED = _("Records Rejected") +PAYMENT_APPROVED = _("Payment Approved") +PAYMENT_ACCRUED = _("Payment Accrued") +PAYMENT_TRANSFERRED = _("Payment Transferred") +NOTIFICATIONS_SENT = _("Notifications Sent") +ADDITIONAL_BUDGET_ADDED = _("Additional Budget Added") # Inferred Events -RECORDS_FLAGGED = "records_flagged" +RECORDS_FLAGGED = _("Records Flagged") + +EVENT_TYPES = [ + INVITE_SENT, + RECORDS_APPROVED, + RECORDS_REJECTED, + PAYMENT_APPROVED, + PAYMENT_ACCRUED, + PAYMENT_TRANSFERRED, + NOTIFICATIONS_SENT, + ADDITIONAL_BUDGET_ADDED, + RECORDS_FLAGGED, +] diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index 1b11cd0f..74ec270b 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -61,6 +61,7 @@ class EventFilter(FilterSet): }, ), ) + event_type = ChoiceFilter(choices=[(_type, _type) for _type in Event.get_all_event_types()]) class Meta: model = Event From e85f60560aa1e35638197feb28bb339189eb7fbf Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 2 Aug 2024 18:51:47 +0530 Subject: [PATCH 08/15] Remove async event saving --- commcare_connect/events/models.py | 15 ++----- commcare_connect/events/tasks.py | 47 -------------------- commcare_connect/events/tests.py | 19 -------- commcare_connect/opportunity/tasks.py | 5 +-- commcare_connect/opportunity/views.py | 6 +-- commcare_connect/opportunity/visit_import.py | 4 +- 6 files changed, 10 insertions(+), 86 deletions(-) delete mode 100644 commcare_connect/events/tasks.py diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index 8d3c3710..c6f39201 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -34,17 +34,10 @@ class Event(models.Model): def get_all_event_types(cls): return set(cls.objects.values_list("event_type", flat=True).distinct()) | set(types.EVENT_TYPES) - def track(self, use_async=True): - """ - To track an event, instantiate the object and call this method, - instead of calling save directly. - - If use_async is True, the event is queued in Redis and saved - via celery, otherwise it's saved directly. - """ - from commcare_connect.events.tasks import track_event - - track_event(self, use_async=use_async) + def save(self, *args, **kwargs): + if not self.date_created: + self.date_created = datetime.utcnow() + super().save(*args, **kwargs) @dataclass diff --git a/commcare_connect/events/tasks.py b/commcare_connect/events/tasks.py deleted file mode 100644 index c97a4253..00000000 --- a/commcare_connect/events/tasks.py +++ /dev/null @@ -1,47 +0,0 @@ -import pickle -from datetime import datetime - -from django.db import transaction -from django_redis import get_redis_connection - -from commcare_connect.events.models import Event -from config import celery_app - -REDIS_EVENTS_QUEUE = "events_queue" - - -class EventQueue: - def __init__(self): - self.redis_conn = get_redis_connection("default") - - def push(self, event_obj): - serialized_event = pickle.dumps(event_obj) - self.redis_conn.rpush(REDIS_EVENTS_QUEUE, serialized_event) - - def pop(self): - events = [pickle.loads(event) for event in self.redis_conn.lrange(REDIS_EVENTS_QUEUE, 0, -1)] - self.redis_conn.ltrim(REDIS_EVENTS_QUEUE, len(events), -1) - return events - - -@celery_app.task -def process_events_batch(): - event_queue = EventQueue() - events = event_queue.pop() - if not events: - return - try: - with transaction.atomic(): - Event.objects.bulk_create(events) - except Exception as e: - for event in events: - event_queue.push(event) - raise e - - -def track_event(event_obj, use_async=True): - event_obj.date_created = datetime.utcnow() - if use_async: - EventQueue().push(event_obj) - else: - event_obj.save() diff --git a/commcare_connect/events/tests.py b/commcare_connect/events/tests.py index 98e68931..0fa873ed 100644 --- a/commcare_connect/events/tests.py +++ b/commcare_connect/events/tests.py @@ -4,7 +4,6 @@ from commcare_connect.users.models import User from .models import Event -from .tasks import EventQueue, process_events_batch def test_post_events(mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity): @@ -40,21 +39,3 @@ def test_post_events(mobile_user_with_connect_link: User, api_client: APIClient, ) assert response.status_code == 201 assert Event.objects.count() == 2 - - -def test_event_queue(mobile_user_with_connect_link: User, opportunity: Opportunity): - event_queue = EventQueue() - assert event_queue.pop() == [] - - # queue the event - event = Event(event_type=Event.Type.INVITE_SENT, user=mobile_user_with_connect_link, opportunity=opportunity) - event.track() - queued_events = event_queue.pop() - process_events_batch() - assert len(queued_events) == 1 - assert Event.objects.count() == 0 - # process the batch - event.track() - process_events_batch() - assert Event.objects.count() == 1 - assert Event.objects.first().user == event.user diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index eecc5cc8..5c3d6622 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -133,10 +133,7 @@ def invite_user(user_id, opportunity_access_id): send_message(message) from commcare_connect.events.models import Event - Event(event_type=Event.Type.INVITE_SENT, user=user, opportunity=opportunity_access.opportunity).track( - # this is already in async worker, so user_async is False - use_async=False - ) + Event(event_type=Event.Type.INVITE_SENT, user=user, opportunity=opportunity_access.opportunity).save() @celery_app.task() diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py index c3b673bb..8635fc9b 100644 --- a/commcare_connect/opportunity/views.py +++ b/commcare_connect/opportunity/views.py @@ -401,7 +401,7 @@ def add_budget_existing_users(request, org_slug=None, pk=None): opportunity.total_budget += ocl.payment_unit.amount * additional_visits opportunity.save() return redirect("opportunity:detail", org_slug, pk) - Event(event_type=Event.Type.ADDITIONAL_BUDGET_ADDED, opportunity=opportunity).track() + Event(event_type=Event.Type.ADDITIONAL_BUDGET_ADDED, opportunity=opportunity).save() return render( request, @@ -810,7 +810,7 @@ def approve_visit(request, org_slug=None, pk=None): opp_id = user_visit.opportunity_id access = OpportunityAccess.objects.get(user_id=user_visit.user_id, opportunity_id=opp_id) update_payment_accrued(opportunity=access.opportunity, users=[access.user]) - Event(event_type=Event.Type.RECORDS_APPROVED, user=user_visit.user, opportunity=access.opportunity).track() + Event(event_type=Event.Type.RECORDS_APPROVED, user=user_visit.user, opportunity=access.opportunity).save() return redirect("opportunity:user_visits_list", org_slug=org_slug, opp_id=user_visit.opportunity.id, pk=access.id) @@ -824,7 +824,7 @@ def reject_visit(request, org_slug=None, pk=None): user_visit.save() access = OpportunityAccess.objects.get(user_id=user_visit.user_id, opportunity_id=user_visit.opportunity_id) update_payment_accrued(opportunity=access.opportunity, users=[access.user]) - Event(event_type=Event.Type.RECORDS_REJECTED, user=user_visit.user, opportunity=access.opportunity).track() + Event(event_type=Event.Type.RECORDS_REJECTED, user=user_visit.user, opportunity=access.opportunity).save() return redirect("opportunity:user_visits_list", org_slug=org_slug, opp_id=user_visit.opportunity_id, pk=access.id) diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py index 2c339307..423681a0 100644 --- a/commcare_connect/opportunity/visit_import.py +++ b/commcare_connect/opportunity/visit_import.py @@ -171,7 +171,7 @@ def update_payment_accrued(opportunity: Opportunity, users): access.payment_accrued += approved_count * completed_work.payment_unit.amount completed_work.save() access.save() - Event(event_type=Event.Type.PAYMENT_ACCRUED, user=access.user, opportunity=access.opportunity).track() + Event(event_type=Event.Type.PAYMENT_ACCRUED, user=access.user, opportunity=access.opportunity).save() def get_status_by_visit_id(dataset) -> dict[int, VisitValidationStatus]: @@ -261,7 +261,7 @@ def _bulk_update_payments(opportunity: Opportunity, imported_data: Dataset) -> P payment = Payment.objects.create(opportunity_access=access, amount=amount) seen_users.add(username) payment_ids.append(payment.pk) - Event(event_type=Event.Type.PAYMENT_TRANSFERRED, user=access.user, opportunity=opportunity).track() + Event(event_type=Event.Type.PAYMENT_TRANSFERRED, user=access.user, opportunity=opportunity).save() missing_users = set(usernames) - seen_users send_payment_notification.delay(opportunity.id, payment_ids) return PaymentImportStatus(seen_users, missing_users) From e1c03fd4a90fb00f52f92c880e9586d00a5e73d5 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 2 Aug 2024 19:03:44 +0530 Subject: [PATCH 09/15] Remove org column, fix user select --- commcare_connect/events/migrations/0001_initial.py | 6 ------ commcare_connect/events/models.py | 7 ------- commcare_connect/events/views.py | 3 ++- commcare_connect/templates/events/event_table_htmx.html | 9 +++++++++ 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/commcare_connect/events/migrations/0001_initial.py b/commcare_connect/events/migrations/0001_initial.py index 699d4211..ebc2c7b6 100644 --- a/commcare_connect/events/migrations/0001_initial.py +++ b/commcare_connect/events/migrations/0001_initial.py @@ -37,12 +37,6 @@ class Migration(migrations.Migration): models.ForeignKey( null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL ), - ), - ( - "organization", - models.ForeignKey( - on_delete=models.CASCADE, related_name="events", related_query_name="event", to="organization.organization" - ), ) ], ), diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index c6f39201..a91f38fa 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -6,7 +6,6 @@ from django.utils.translation import gettext as _ from commcare_connect.cache import quickcache -from commcare_connect.organization.models import Organization from commcare_connect.users.models import User from . import types @@ -22,12 +21,6 @@ class Event(models.Model): event_type = models.CharField(max_length=40, db_index=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) - organization = models.ForeignKey( - Organization, - on_delete=models.CASCADE, - related_name="events", - related_query_name="event", - ) @classmethod @quickcache([], timeout=60 * 60) diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index 74ec270b..eaa752f3 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -10,6 +10,7 @@ from rest_framework.response import Response from commcare_connect.opportunity.forms import DateRanges +from commcare_connect.opportunity.models import Opportunity from commcare_connect.users.models import User from .models import Event @@ -95,4 +96,4 @@ def get_template_names(self): return template_name def get_queryset(self): - return Event.objects.filter(organization=self.request.org) + return Event.objects.filter(opportunity__in=Opportunity.objects.filter(organization=self.request.org)) diff --git a/commcare_connect/templates/events/event_table_htmx.html b/commcare_connect/templates/events/event_table_htmx.html index ec919d1a..3e3f6aaa 100644 --- a/commcare_connect/templates/events/event_table_htmx.html +++ b/commcare_connect/templates/events/event_table_htmx.html @@ -21,8 +21,17 @@

    Events

    {% block javascript %} {{ block.super }} + + + {% endblock %} {% block css %} From c59a387a330d5e446036d13928f71e90539e7e30 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 2 Aug 2024 19:36:59 +0530 Subject: [PATCH 10/15] Handle partial failure --- commcare_connect/events/tests.py | 24 ++++++++++++++++++------ commcare_connect/events/views.py | 22 +++++++++++++++++++--- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/commcare_connect/events/tests.py b/commcare_connect/events/tests.py index 0fa873ed..15cb8e14 100644 --- a/commcare_connect/events/tests.py +++ b/commcare_connect/events/tests.py @@ -1,3 +1,5 @@ +from datetime import datetime + from rest_framework.test import APIClient from commcare_connect.opportunity.models import Opportunity @@ -8,34 +10,44 @@ def test_post_events(mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity): api_client.force_authenticate(mobile_user_with_connect_link) + assert Event.objects.count() == 0 response = api_client.post( "/api/events/", data=[ { - "event_type": "invalid_event_name", + "event_type": Event.Type.INVITE_SENT, + "user": mobile_user_with_connect_link.pk, + "opportunity": opportunity.pk, + "date_created": datetime.utcnow(), + }, + { + "event_type": Event.Type.RECORDS_APPROVED, "user": mobile_user_with_connect_link.pk, "opportunity": opportunity.pk, - } + "date_created": datetime.utcnow(), + }, ], format="json", ) - assert response.status_code == 400 - assert Event.objects.count() == 0 + assert response.status_code == 201 + assert Event.objects.count() == 2 response = api_client.post( "/api/events/", data=[ { "event_type": Event.Type.INVITE_SENT, - "user": mobile_user_with_connect_link.pk, + "user": -1, "opportunity": opportunity.pk, + "date_created": datetime.utcnow(), }, { "event_type": Event.Type.RECORDS_APPROVED, "user": mobile_user_with_connect_link.pk, "opportunity": opportunity.pk, + "date_created": datetime.utcnow(), }, ], format="json", ) - assert response.status_code == 201 + assert response.status_code == 400 assert Event.objects.count() == 2 diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index eaa752f3..5d285c28 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -1,5 +1,6 @@ import django_tables2 as tables from dal.autocomplete import ModelSelect2 +from django.db import transaction from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt from django_filters import ChoiceFilter, FilterSet, ModelChoiceFilter @@ -34,9 +35,24 @@ def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data, many=True) serializer.is_valid(raise_exception=True) - - event_objects = [Event(**item) for item in serializer.validated_data] - Event.objects.bulk_create(event_objects) + try: + event_objects = [Event(**item) for item in serializer.validated_data] + Event.objects.bulk_create(event_objects) + except Exception as e: + # Bulk create failed, try saving each item individually + failed_items = [] + + for item in serializer.validated_data: + try: + with transaction.atomic(): + Event.objects.save(**item) + except Exception: + failed_items.append((item, e)) + + if failed_items: + partial_error_response = {"error": "Some items could not be saved", "failed_items": failed_items} + headers = self.get_success_headers(serializer.data) + return Response(partial_error_response, status=status.HTTP_206_PARTIAL_CONTENT, headers=headers) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) From 51b7719cc439e84ba6564030cab84da900bc8cc2 Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Thu, 15 Aug 2024 18:15:00 +0530 Subject: [PATCH 11/15] Add more events --- .../events/migrations/0001_initial.py | 6 ++++- commcare_connect/events/models.py | 1 + commcare_connect/events/types.py | 10 ++++++++ commcare_connect/form_receiver/processor.py | 25 ++++++++++++++++++- commcare_connect/opportunity/api/views.py | 14 +++++++++++ commcare_connect/users/views.py | 8 ++++++ requirements/base.txt | 6 +++++ 7 files changed, 68 insertions(+), 2 deletions(-) diff --git a/commcare_connect/events/migrations/0001_initial.py b/commcare_connect/events/migrations/0001_initial.py index ebc2c7b6..3ce4b49d 100644 --- a/commcare_connect/events/migrations/0001_initial.py +++ b/commcare_connect/events/migrations/0001_initial.py @@ -37,7 +37,11 @@ class Migration(migrations.Migration): models.ForeignKey( null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL ), - ) + ), + ( + "metadata", + models.JSONField(default=dict), + ), ], ), ] diff --git a/commcare_connect/events/models.py b/commcare_connect/events/models.py index a91f38fa..e4637614 100644 --- a/commcare_connect/events/models.py +++ b/commcare_connect/events/models.py @@ -21,6 +21,7 @@ class Event(models.Model): event_type = models.CharField(max_length=40, db_index=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True) + metadata = models.JSONField() @classmethod @quickcache([], timeout=60 * 60) diff --git a/commcare_connect/events/types.py b/commcare_connect/events/types.py index 5a0b6cb2..ffa23820 100644 --- a/commcare_connect/events/types.py +++ b/commcare_connect/events/types.py @@ -2,6 +2,7 @@ # Server/Web events INVITE_SENT = _("Invite Sent") +INVITE_ACCEPTED = _("Invite Accepted") RECORDS_APPROVED = _("Records Approved") RECORDS_REJECTED = _("Records Rejected") PAYMENT_APPROVED = _("Payment Approved") @@ -10,6 +11,15 @@ NOTIFICATIONS_SENT = _("Notifications Sent") ADDITIONAL_BUDGET_ADDED = _("Additional Budget Added") +MODULE_COMPLETED = _("Module Completed") +ALL_MODULES_COMPLETED = _("All Modules Completed") +ASSESSMENT_PASSED = _("Assessment Passed") +ASSESSMENT_FAILED = _("Assessment Failed") + +JOB_CLAIMED = _("Job Claimed Successfully") +DELIVERY_FORM_SUBMITTED = _("Delivery Form Submitted") +PAYMENT_ACKNOWLEDGED = _("Payment Acknowledged") + # Inferred Events RECORDS_FLAGGED = _("Records Flagged") diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py index e6dac1ef..17161388 100644 --- a/commcare_connect/form_receiver/processor.py +++ b/commcare_connect/form_receiver/processor.py @@ -6,6 +6,7 @@ from jsonpath_ng import JSONPathError from jsonpath_ng.ext import parse +from commcare_connect.events.models import Event from commcare_connect.form_receiver.const import CCC_LEARN_XMLNS from commcare_connect.form_receiver.exceptions import ProcessingError from commcare_connect.form_receiver.serializers import XForm @@ -103,6 +104,15 @@ def process_learn_modules(user: User, xform: XForm, app: CommCareApp, opportunit if not created: raise ProcessingError("Learn Module is already completed") + else: + Event( + event_type=Event.Type.MODULE_COMPLETED, + user=user, + opportunity=opportunity, + metadata={"module_name": module.name}, + ).save() + if access.learn_progress == 100: + Event(event_type=Event.Type.ALL_MODULES_COMPLETED, user=user, opportunity=opportunity).save() def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Opportunity, blocks: list[dict]): @@ -121,6 +131,7 @@ def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Oppor # TODO: should this move to the opportunity to allow better re-use of the app? passing_score = app.passing_score access = OpportunityAccess.objects.get(user=user, opportunity=opportunity) + passed = score >= passing_score assessment, created = Assessment.objects.get_or_create( user=user, app=app, @@ -131,7 +142,7 @@ def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Oppor "date": xform.received_on, "score": score, "passing_score": passing_score, - "passed": score >= passing_score, + "passed": passed, "app_build_id": xform.build_id, "app_build_version": xform.metadata.app_build_version, }, @@ -139,6 +150,13 @@ def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Oppor if not created: return ProcessingError("Learn Assessment is already completed") + else: + Event( + event_type=Event.Type.ASSESSMENT_PASSED if passed else Event.Type.ASSESSMENT_FAILED, + user=user, + opportunity=opportunity, + metadata={"score": score, "passing_score": passing_score}, + ).save() def process_deliver_form(user, xform: XForm, app: CommCareApp, opportunity: Opportunity): @@ -229,6 +247,11 @@ def clean_form_submission(access: OpportunityAccess, user_visit: UserVisit, xfor def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Opportunity, deliver_unit_block: dict): + Event( + event_type=Event.Type.DELIVERY_FORM_SUBMITTED, + user=user, + opportunity=opportunity, + ).save() deliver_unit = get_or_create_deliver_unit(app, deliver_unit_block) access = OpportunityAccess.objects.get(opportunity=opportunity, user=user) counts = ( diff --git a/commcare_connect/opportunity/api/views.py b/commcare_connect/opportunity/api/views.py index fd4db244..523ff759 100644 --- a/commcare_connect/opportunity/api/views.py +++ b/commcare_connect/opportunity/api/views.py @@ -8,6 +8,7 @@ from rest_framework.response import Response from rest_framework.views import APIView +from commcare_connect.events.models import Event from commcare_connect.opportunity.api.serializers import ( CompletedWorkSerializer, DeliveryProgressSerializer, @@ -101,6 +102,11 @@ def post(self, *args, **kwargs): return Response("Failed to create user", status=400) cc_username = f"{self.request.user.username.lower()}@{domain}.commcarehq.org" ConnectIDUserLink.objects.create(commcare_username=cc_username, user=self.request.user, domain=domain) + Event( + event_type=Event.Type.JOB_CLAIMED, + user=opportunity_access.user, + opportunity=opportunity, + ).save() return Response(status=201) @@ -119,4 +125,12 @@ def post(self, *args, **kwargs): payment.confirmed = confirmed payment.confirmation_date = now() payment.save() + + Event( + event_type=Event.Type.PAYMENT_ACKNOWLEDGED, + user=payment.opportunity_access.user, + opportunity=payment.opportunity_access.opportunity, + metadata={"confirmed": confirmed}, + ).save() + return Response(status=200) diff --git a/commcare_connect/users/views.py b/commcare_connect/users/views.py index 0653db83..2340677d 100644 --- a/commcare_connect/users/views.py +++ b/commcare_connect/users/views.py @@ -21,6 +21,7 @@ from rest_framework.views import APIView from commcare_connect.connect_id_client.main import fetch_demo_user_tokens +from commcare_connect.events.models import Event from commcare_connect.opportunity.models import Opportunity, OpportunityAccess, UserInvite, UserInviteStatus from .helpers import create_hq_user @@ -134,6 +135,13 @@ def accept_invite(request, invite_id): user_invite = UserInvite.objects.get(opportunity_access=o) user_invite.status = UserInviteStatus.accepted user_invite.save() + + Event( + event_type=Event.Type.INVITE_ACCEPTED, + user=o.user, + opportunity=o.opportunity, + ).save() + return HttpResponse( "Thank you for accepting the invitation. Open your CommCare Connect App to " "see more information about the opportunity and begin learning" diff --git a/requirements/base.txt b/requirements/base.txt index cf653287..8a48fb1a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -79,9 +79,11 @@ django==4.2.5 # -r requirements/base.in # crispy-bootstrap5 # django-allauth + # django-autocomplete-light # django-celery-beat # django-cors-headers # django-crispy-forms + # django-filter # django-model-utils # django-oauth-toolkit # django-redis @@ -91,6 +93,8 @@ django==4.2.5 # drf-spectacular django-allauth==0.54.0 # via -r requirements/base.in +django-autocomplete-light==3.11.0 + # via -r requirements/base.in django-celery-beat==2.5.0 # via -r requirements/base.in django-cors-headers==4.2.0 @@ -101,6 +105,8 @@ django-crispy-forms==2.0 # crispy-bootstrap5 django-environ==0.10.0 # via -r requirements/base.in +django-filter==24.3 + # via -r requirements/base.in django-model-utils==4.3.1 # via -r requirements/base.in django-oauth-toolkit==2.3.0 From fdd182ad93d7aa0bcb9c6a46f14a6d155a11dd4c Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 16 Aug 2024 15:51:53 +0530 Subject: [PATCH 12/15] fix order --- commcare_connect/events/types.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/commcare_connect/events/types.py b/commcare_connect/events/types.py index ffa23820..3b945263 100644 --- a/commcare_connect/events/types.py +++ b/commcare_connect/events/types.py @@ -21,17 +21,21 @@ PAYMENT_ACKNOWLEDGED = _("Payment Acknowledged") -# Inferred Events -RECORDS_FLAGGED = _("Records Flagged") - EVENT_TYPES = [ INVITE_SENT, + INVITE_ACCEPTED, + JOB_CLAIMED, + MODULE_COMPLETED, + ALL_MODULES_COMPLETED, + ASSESSMENT_PASSED, + ASSESSMENT_FAILED, + DELIVERY_FORM_SUBMITTED, RECORDS_APPROVED, RECORDS_REJECTED, PAYMENT_APPROVED, PAYMENT_ACCRUED, PAYMENT_TRANSFERRED, + PAYMENT_ACKNOWLEDGED, NOTIFICATIONS_SENT, ADDITIONAL_BUDGET_ADDED, - RECORDS_FLAGGED, ] From badcef513ceac8f954979321e1f7363e1f978a7a Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 16 Aug 2024 16:37:41 +0530 Subject: [PATCH 13/15] lazy evaluating --- commcare_connect/events/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index 67d70156..b32ad742 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -79,7 +79,7 @@ class EventFilter(FilterSet): }, ), ) - event_type = ChoiceFilter(choices=[(_type, _type) for _type in Event.get_all_event_types()]) + event_type = ChoiceFilter(choices=lambda: [(_type, _type) for _type in Event.get_all_event_types()]) class Meta: model = Event From a7533116ccc2f0cf66d8936cafb9c40baea7ef1f Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Thu, 22 Aug 2024 19:43:03 +0530 Subject: [PATCH 14/15] accept uid to track errors --- commcare_connect/events/tests.py | 3 +++ commcare_connect/events/views.py | 32 ++++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/commcare_connect/events/tests.py b/commcare_connect/events/tests.py index 15cb8e14..63384fd9 100644 --- a/commcare_connect/events/tests.py +++ b/commcare_connect/events/tests.py @@ -19,12 +19,15 @@ def test_post_events(mobile_user_with_connect_link: User, api_client: APIClient, "user": mobile_user_with_connect_link.pk, "opportunity": opportunity.pk, "date_created": datetime.utcnow(), + "uid": "1", }, { "event_type": Event.Type.RECORDS_APPROVED, "user": mobile_user_with_connect_link.pk, "opportunity": opportunity.pk, "date_created": datetime.utcnow(), + "metadata": {"extra": "test"}, + "uid": "2", }, ], format="json", diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index b32ad742..17017982 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -1,4 +1,5 @@ import django_tables2 as tables +import sentry_sdk from dal.autocomplete import ModelSelect2 from django.db import transaction from django.utils.decorators import method_decorator @@ -12,15 +13,27 @@ from commcare_connect.opportunity.forms import DateRanges from commcare_connect.opportunity.models import Opportunity +from commcare_connect.opportunity.views import OrganizationUserMixin from commcare_connect.users.models import User from .models import Event class EventSerializer(serializers.ModelSerializer): + uid = serializers.JSONField(write_only=True, required=False) + class Meta: model = Event - fields = ["date_created", "event_type", "user", "opportunity"] + fields = ["date_created", "event_type", "user", "opportunity", "metadata", "uid"] + + def to_internal_value(self, data): + # Extract the 'meta' field if present and remove it from the data + uid = data.pop("uid", None) + + internal_value = super().to_internal_value(data) + internal_value["uid"] = uid + + return internal_value @method_decorator(csrf_exempt, name="dispatch") @@ -39,23 +52,26 @@ def create(self, request, *args, **kwargs): event_objects = [Event(**item) for item in serializer.validated_data] Event.objects.bulk_create(event_objects) except Exception as e: + sentry_sdk.capture_exception(e) # Bulk create failed, try saving each item individually failed_items = [] for item in serializer.validated_data: + uid = item.pop("uid") try: with transaction.atomic(): - Event.objects.save(**item) - except Exception: - failed_items.append((item, e)) + Event(**item).save() + except Exception as e: + sentry_sdk.capture_exception(e) + failed_items.append(uid) if failed_items: - partial_error_response = {"error": "Some items could not be saved", "failed_items": failed_items} + partial_error_response = {"success": False, "failed_items": failed_items} headers = self.get_success_headers(serializer.data) - return Response(partial_error_response, status=status.HTTP_206_PARTIAL_CONTENT, headers=headers) + return Response(partial_error_response, status=status.HTTP_201_CREATED, headers=headers) headers = self.get_success_headers(serializer.data) - return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + return Response({"success": True}, status=status.HTTP_201_CREATED, headers=headers) class EventTable(tables.Table): @@ -98,7 +114,7 @@ def filter_by_date_range(self, queryset, name, value): return queryset -class EventListView(tables.SingleTableMixin, FilterView): +class EventListView(tables.SingleTableMixin, OrganizationUserMixin, FilterView): table_class = EventTable queryset = Event.objects.all() filterset_class = EventFilter From dac55574851719c039e668e24c7dc030d125fd4e Mon Sep 17 00:00:00 2001 From: Sravan Reddy Date: Fri, 13 Sep 2024 10:23:27 +0530 Subject: [PATCH 15/15] move prod only module import --- commcare_connect/events/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commcare_connect/events/views.py b/commcare_connect/events/views.py index 17017982..45e4e90b 100644 --- a/commcare_connect/events/views.py +++ b/commcare_connect/events/views.py @@ -1,5 +1,4 @@ import django_tables2 as tables -import sentry_sdk from dal.autocomplete import ModelSelect2 from django.db import transaction from django.utils.decorators import method_decorator @@ -52,6 +51,8 @@ def create(self, request, *args, **kwargs): event_objects = [Event(**item) for item in serializer.validated_data] Event.objects.bulk_create(event_objects) except Exception as e: + import sentry_sdk + sentry_sdk.capture_exception(e) # Bulk create failed, try saving each item individually failed_items = []