add event tracking

commcare_connect/events/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class EventsAppConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "commcare_connect.events"

commcare_connect/events/migrations/0001_initial.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.2.5 on 2024-07-13 14:45
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("opportunity", "0044_opportunityverificationflags"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("date_created", models.DateTimeField(db_index=True)),
+                (
+                    "event_type",
+                    models.CharField(
+                        max_length=40,
+                        db_index=True
+                    ),
+                ),
+                (
+                    "opportunity",
+                    models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.PROTECT, to="opportunity.opportunity"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                (
+                    "metadata",
+                    models.JSONField(default=dict),
+                ),
+            ],
+        ),
+    ]

commcare_connect/events/models.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+
+from django.db import models
+
+from commcare_connect.cache import quickcache
+from commcare_connect.users.models import User
+
+from . import types
+
+
+class Event(models.Model):
+    from commcare_connect.opportunity.models import Opportunity
+
+    # this allows referring to event types in this style: Event.Type.INVITE_SENT
+    Type = types
+
+    date_created = models.DateTimeField(db_index=True)
+    event_type = models.CharField(max_length=40, db_index=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
+    opportunity = models.ForeignKey(Opportunity, on_delete=models.PROTECT, null=True)
+    metadata = models.JSONField(default=dict)
+
+    @classmethod
+    @quickcache([], timeout=60 * 60)
+    def get_all_event_types(cls):
+        return set(cls.objects.values_list("event_type", flat=True).distinct()) | set(types.EVENT_TYPES)
+
+    def save(self, *args, **kwargs):
+        if not self.date_created:
+            self.date_created = datetime.utcnow()
+        super().save(*args, **kwargs)

commcare_connect/events/tests.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+
+from rest_framework.test import APIClient
+
+from commcare_connect.opportunity.models import Opportunity
+from commcare_connect.users.models import User
+
+from .models import Event
+
+
+def test_post_events(mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity):
+    api_client.force_authenticate(mobile_user_with_connect_link)
+    assert Event.objects.count() == 0
+    response = api_client.post(
+        "/api/events/",
+        data=[
+            {
+                "event_type": Event.Type.INVITE_SENT,
+                "user": mobile_user_with_connect_link.pk,
+                "opportunity": opportunity.pk,
+                "date_created": datetime.utcnow(),
+                "uid": "1",
+            },
+            {
+                "event_type": Event.Type.RECORDS_APPROVED,
+                "user": mobile_user_with_connect_link.pk,
+                "opportunity": opportunity.pk,
+                "date_created": datetime.utcnow(),
+                "metadata": {"extra": "test"},
+                "uid": "2",
+            },
+        ],
+        format="json",
+    )
+    assert response.status_code == 201
+    assert Event.objects.count() == 2
+    response = api_client.post(
+        "/api/events/",
+        data=[
+            {
+                "event_type": Event.Type.INVITE_SENT,
+                "user": -1,
+                "opportunity": opportunity.pk,
+                "date_created": datetime.utcnow(),
+            },
+            {
+                "event_type": Event.Type.RECORDS_APPROVED,
+                "user": mobile_user_with_connect_link.pk,
+                "opportunity": opportunity.pk,
+                "date_created": datetime.utcnow(),
+            },
+        ],
+        format="json",
+    )
+    assert response.status_code == 400
+    assert Event.objects.count() == 2

commcare_connect/events/types.py

@@ -0,0 +1,41 @@
+from django.utils.translation import gettext as _
+
+# Server/Web events
+INVITE_SENT = _("Invite Sent")
+INVITE_ACCEPTED = _("Invite Accepted")
+RECORDS_APPROVED = _("Records Approved")
+RECORDS_REJECTED = _("Records Rejected")
+PAYMENT_APPROVED = _("Payment Approved")
+PAYMENT_ACCRUED = _("Payment Accrued")
+PAYMENT_TRANSFERRED = _("Payment Transferred")
+NOTIFICATIONS_SENT = _("Notifications Sent")
+ADDITIONAL_BUDGET_ADDED = _("Additional Budget Added")
+
+MODULE_COMPLETED = _("Module Completed")
+ALL_MODULES_COMPLETED = _("All Modules Completed")
+ASSESSMENT_PASSED = _("Assessment Passed")
+ASSESSMENT_FAILED = _("Assessment Failed")
+
+JOB_CLAIMED = _("Job Claimed Successfully")
+DELIVERY_FORM_SUBMITTED = _("Delivery Form Submitted")
+PAYMENT_ACKNOWLEDGED = _("Payment Acknowledged")
+
+
+EVENT_TYPES = [
+    INVITE_SENT,
+    INVITE_ACCEPTED,
+    JOB_CLAIMED,
+    MODULE_COMPLETED,
+    ALL_MODULES_COMPLETED,
+    ASSESSMENT_PASSED,
+    ASSESSMENT_FAILED,
+    DELIVERY_FORM_SUBMITTED,
+    RECORDS_APPROVED,
+    RECORDS_REJECTED,
+    PAYMENT_APPROVED,
+    PAYMENT_ACCRUED,
+    PAYMENT_TRANSFERRED,
+    PAYMENT_ACKNOWLEDGED,
+    NOTIFICATIONS_SENT,
+    ADDITIONAL_BUDGET_ADDED,
+]

commcare_connect/events/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+
+from .views import EventListView
+
+app_name = "events"
+
+urlpatterns = [
+    path("", view=EventListView.as_view(), name="event_htmx"),
+]

commcare_connect/events/views.py

@@ -0,0 +1,133 @@
+import django_tables2 as tables
+from dal.autocomplete import ModelSelect2
+from django.db import transaction
+from django.utils.decorators import method_decorator
+from django.views.decorators.csrf import csrf_exempt
+from django_filters import ChoiceFilter, FilterSet, ModelChoiceFilter
+from django_filters.views import FilterView
+from rest_framework import serializers, status
+from rest_framework.generics import ListCreateAPIView
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+
+from commcare_connect.opportunity.forms import DateRanges
+from commcare_connect.opportunity.models import Opportunity
+from commcare_connect.opportunity.views import OrganizationUserMixin
+from commcare_connect.users.models import User
+
+from .models import Event
+
+
+class EventSerializer(serializers.ModelSerializer):
+    uid = serializers.JSONField(write_only=True, required=False)
+
+    class Meta:
+        model = Event
+        fields = ["date_created", "event_type", "user", "opportunity", "metadata", "uid"]
+
+    def to_internal_value(self, data):
+        # Extract the 'meta' field if present and remove it from the data
+        uid = data.pop("uid", None)
+
+        internal_value = super().to_internal_value(data)
+        internal_value["uid"] = uid
+
+        return internal_value
+
+
+@method_decorator(csrf_exempt, name="dispatch")
+class EventListCreateView(ListCreateAPIView):
+    queryset = Event.objects.all()
+    serializer_class = EventSerializer
+    permission_classes = [IsAuthenticated]
+
+    def create(self, request, *args, **kwargs):
+        if not isinstance(request.data, list):
+            return Response({"error": "Expected a list of items"}, status=status.HTTP_400_BAD_REQUEST)
+
+        serializer = self.get_serializer(data=request.data, many=True)
+        serializer.is_valid(raise_exception=True)
+        try:
+            event_objects = [Event(**item) for item in serializer.validated_data]
+            Event.objects.bulk_create(event_objects)
+        except Exception as e:
+            import sentry_sdk
+
+            sentry_sdk.capture_exception(e)
+            # Bulk create failed, try saving each item individually
+            failed_items = []
+
+            for item in serializer.validated_data:
+                uid = item.pop("uid")
+                try:
+                    with transaction.atomic():
+                        Event(**item).save()
+                except Exception as e:
+                    sentry_sdk.capture_exception(e)
+                    failed_items.append(uid)
+
+            if failed_items:
+                partial_error_response = {"success": False, "failed_items": failed_items}
+                headers = self.get_success_headers(serializer.data)
+                return Response(partial_error_response, status=status.HTTP_201_CREATED, headers=headers)
+
+        headers = self.get_success_headers(serializer.data)
+        return Response({"success": True}, status=status.HTTP_201_CREATED, headers=headers)
+
+
+class EventTable(tables.Table):
+    date_created = tables.Column(verbose_name="Time")
+    metadata = tables.Column(verbose_name="Metadata", orderable=False)
+
+    class Meta:
+        model = Event
+        template_name = "events/htmx_table.html"
+        fields = ("user", "opportunity", "event_type", "date_created", "metadata")
+
+
+class EventFilter(FilterSet):
+    date_range = ChoiceFilter(choices=DateRanges.choices, method="filter_by_date_range", label="Date Range")
+    user = ModelChoiceFilter(
+        queryset=User.objects.all(),
+        widget=ModelSelect2(
+            url="users:search",
+            attrs={
+                "data-placeholder": "All",
+            },
+        ),
+    )
+    event_type = ChoiceFilter(choices=lambda: [(_type, _type) for _type in Event.get_all_event_types()])
+
+    class Meta:
+        model = Event
+        fields = ["opportunity", "user", "event_type"]
+
+    def filter_by_date_range(self, queryset, name, value):
+        if not value:
+            return queryset
+
+        try:
+            date_range = DateRanges(value)
+            return queryset.filter(
+                date_created__gte=date_range.get_cutoff_date(),
+            )
+        except ValueError:
+            return queryset
+
+
+class EventListView(tables.SingleTableMixin, OrganizationUserMixin, FilterView):
+    table_class = EventTable
+    queryset = Event.objects.all()
+    filterset_class = EventFilter
+    paginate_by = 20
+
+    def get_template_names(self):
+        if self.request.htmx:
+            template_name = "events/event_table_partial.html"
+        else:
+            template_name = "events/event_table_htmx.html"
+
+        return template_name
+
+    def get_queryset(self):
+        return Event.objects.filter(opportunity__in=Opportunity.objects.filter(organization=self.request.org))

commcare_connect/form_receiver/processor.py

@@ -6,6 +6,7 @@
 from jsonpath_ng import JSONPathError
 from jsonpath_ng.ext import parse
 
+from commcare_connect.events.models import Event
 from commcare_connect.form_receiver.const import CCC_LEARN_XMLNS
 from commcare_connect.form_receiver.exceptions import ProcessingError
 from commcare_connect.form_receiver.serializers import XForm
@@ -103,6 +104,15 @@ def process_learn_modules(user: User, xform: XForm, app: CommCareApp, opportunit
 
         if not created:
             raise ProcessingError("Learn Module is already completed")
+        else:
+            Event(
+                event_type=Event.Type.MODULE_COMPLETED,
+                user=user,
+                opportunity=opportunity,
+                metadata={"module_name": module.name},
+            ).save()
+            if access.learn_progress == 100:
+                Event(event_type=Event.Type.ALL_MODULES_COMPLETED, user=user, opportunity=opportunity).save()
 
 
 def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Opportunity, blocks: list[dict]):
@@ -121,6 +131,7 @@ def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Oppor
         # TODO: should this move to the opportunity to allow better re-use of the app?
         passing_score = app.passing_score
         access = OpportunityAccess.objects.get(user=user, opportunity=opportunity)
+        passed = score >= passing_score
         assessment, created = Assessment.objects.get_or_create(
             user=user,
             app=app,
@@ -131,14 +142,21 @@ def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Oppor
                 "date": xform.received_on,
                 "score": score,
                 "passing_score": passing_score,
-                "passed": score >= passing_score,
+                "passed": passed,
                 "app_build_id": xform.build_id,
                 "app_build_version": xform.metadata.app_build_version,
             },
         )
 
         if not created:
             return ProcessingError("Learn Assessment is already completed")
+        else:
+            Event(
+                event_type=Event.Type.ASSESSMENT_PASSED if passed else Event.Type.ASSESSMENT_FAILED,
+                user=user,
+                opportunity=opportunity,
+                metadata={"score": score, "passing_score": passing_score},
+            ).save()
 
 
 def process_deliver_form(user, xform: XForm, app: CommCareApp, opportunity: Opportunity):
@@ -309,6 +327,12 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
         completed_work_needs_save = True
     if completed_work_needs_save:
         completed_work.save()
+    Event(
+        event_type=Event.Type.DELIVERY_FORM_SUBMITTED,
+        user=user,
+        opportunity=opportunity,
+        metadata={"flags": [f[1] for f in flags]} if flags else {},
+    ).save()
     download_user_visit_attachments.delay(user_visit.id)