From cc6777486c5775cd50202f9b52b8c733b64499e6 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Thu, 3 Oct 2024 09:23:40 +0530 Subject: [PATCH 1/6] added payment date --- .../0059_completedwork_payment_date.py | 29 +++++++++++++++++++ commcare_connect/opportunity/models.py | 1 + commcare_connect/opportunity/visit_import.py | 14 +++++++++ 3 files changed, 44 insertions(+) create mode 100644 commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py diff --git a/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py b/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py new file mode 100644 index 00000000..155df806 --- /dev/null +++ b/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.5 on 2024-10-03 02:33 + +from django.db import migrations, models +from django.db import transaction + +from commcare_connect.opportunity.models import Payment +from commcare_connect.opportunity.visit_import import update_work_payment_date + + +@transaction.atomic +def update_paid_date_from_payments(apps, schema_editor): + payments = Payment.objects.all() + for payment in payments: + update_work_payment_date(payment) + + +class Migration(migrations.Migration): + dependencies = [ + ("opportunity", "0058_paymentinvoice_payment_invoice"), + ] + + operations = [ + migrations.AddField( + model_name="completedwork", + name="payment_date", + field=models.DateTimeField(null=True), + ), + migrations.RunPython(update_paid_date_from_payments, migrations.RunPython.noop), + ] diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py index 87d970c3..db74f16e 100644 --- a/commcare_connect/opportunity/models.py +++ b/commcare_connect/opportunity/models.py @@ -434,6 +434,7 @@ class CompletedWork(models.Model): entity_name = models.CharField(max_length=255, null=True, blank=True) reason = models.CharField(max_length=300, null=True, blank=True) status_modified_date = models.DateTimeField(null=True) + payment_date = models.DateTimeField(null=True) def __init__(self, *args, **kwargs): self.status = CompletedWorkStatus.incomplete diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py index 51755848..6764bf84 100644 --- a/commcare_connect/opportunity/visit_import.py +++ b/commcare_connect/opportunity/visit_import.py @@ -263,11 +263,25 @@ def _bulk_update_payments(opportunity: Opportunity, imported_data: Dataset) -> P payment = Payment.objects.create(opportunity_access=access, amount=amount) seen_users.add(username) payment_ids.append(payment.pk) + update_work_payment_date(payment) missing_users = set(usernames) - seen_users send_payment_notification.delay(opportunity.id, payment_ids) return PaymentImportStatus(seen_users, missing_users) +def update_work_payment_date(payment: Payment): + completed_works = CompletedWork.objects.filter(opportunity_access=payment.opportunity_access) + paid = payment.amount + works_to_update = [] + + for current_work in completed_works: + if paid > current_work.payment_accrued: + works_to_update.append(current_work.id) + + if works_to_update: + CompletedWork.objects.filter(id__in=works_to_update).update(payment_date=payment.date_paid) + + def bulk_update_completed_work_status(opportunity: Opportunity, file: UploadedFile) -> CompletedWorkImportStatus: file_format = get_file_extension(file) if file_format not in ("csv", "xlsx"): From 48e7ad83253d1a94437af781150e50403307d1f0 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Thu, 3 Oct 2024 09:41:17 +0530 Subject: [PATCH 2/6] moved function in util file --- .../opportunity/utils/completed_work.py | 21 ++++++++++++++++++- commcare_connect/opportunity/visit_import.py | 15 +------------ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py index 202bbb8d..4a5f7f1f 100644 --- a/commcare_connect/opportunity/utils/completed_work.py +++ b/commcare_connect/opportunity/utils/completed_work.py @@ -1,4 +1,10 @@ -from commcare_connect.opportunity.models import CompletedWorkStatus, VisitReviewStatus, VisitValidationStatus +from commcare_connect.opportunity.models import ( + CompletedWork, + CompletedWorkStatus, + Payment, + VisitReviewStatus, + VisitValidationStatus, +) def update_status(completed_works, opportunity_access, compute_payment=True): @@ -35,3 +41,16 @@ def update_status(completed_works, opportunity_access, compute_payment=True): if compute_payment: opportunity_access.payment_accrued = payment_accrued opportunity_access.save() + + +def update_work_payment_date(payment: Payment): + completed_works = CompletedWork.objects.filter(opportunity_access=payment.opportunity_access) + paid = payment.amount + works_to_update = [] + + for current_work in completed_works: + if paid > current_work.payment_accrued: + works_to_update.append(current_work.id) + + if works_to_update: + CompletedWork.objects.filter(id__in=works_to_update).update(payment_date=payment.date_paid) diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py index 6764bf84..f1afc3b6 100644 --- a/commcare_connect/opportunity/visit_import.py +++ b/commcare_connect/opportunity/visit_import.py @@ -19,7 +19,7 @@ VisitValidationStatus, ) from commcare_connect.opportunity.tasks import send_payment_notification -from commcare_connect.opportunity.utils.completed_work import update_status +from commcare_connect.opportunity.utils.completed_work import update_status, update_work_payment_date from commcare_connect.utils.file import get_file_extension from commcare_connect.utils.itertools import batched @@ -269,19 +269,6 @@ def _bulk_update_payments(opportunity: Opportunity, imported_data: Dataset) -> P return PaymentImportStatus(seen_users, missing_users) -def update_work_payment_date(payment: Payment): - completed_works = CompletedWork.objects.filter(opportunity_access=payment.opportunity_access) - paid = payment.amount - works_to_update = [] - - for current_work in completed_works: - if paid > current_work.payment_accrued: - works_to_update.append(current_work.id) - - if works_to_update: - CompletedWork.objects.filter(id__in=works_to_update).update(payment_date=payment.date_paid) - - def bulk_update_completed_work_status(opportunity: Opportunity, file: UploadedFile) -> CompletedWorkImportStatus: file_format = get_file_extension(file) if file_format not in ("csv", "xlsx"): From 802e47bfa002fde581ce5067751a2f4e60cc059d Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Fri, 4 Oct 2024 16:31:27 +0530 Subject: [PATCH 3/6] Update the payment logic --- .../0059_completedwork_payment_date.py | 8 ++--- .../opportunity/utils/completed_work.py | 31 ++++++++++++++----- commcare_connect/opportunity/visit_import.py | 2 +- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py b/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py index 155df806..f1b9c862 100644 --- a/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py +++ b/commcare_connect/opportunity/migrations/0059_completedwork_payment_date.py @@ -3,15 +3,15 @@ from django.db import migrations, models from django.db import transaction -from commcare_connect.opportunity.models import Payment +from commcare_connect.opportunity.models import OpportunityAccess from commcare_connect.opportunity.visit_import import update_work_payment_date @transaction.atomic def update_paid_date_from_payments(apps, schema_editor): - payments = Payment.objects.all() - for payment in payments: - update_work_payment_date(payment) + accesses = OpportunityAccess.objects.all() + for access in accesses: + update_work_payment_date(access) class Migration(migrations.Migration): diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py index 4a5f7f1f..ee45a6d6 100644 --- a/commcare_connect/opportunity/utils/completed_work.py +++ b/commcare_connect/opportunity/utils/completed_work.py @@ -1,6 +1,7 @@ from commcare_connect.opportunity.models import ( CompletedWork, CompletedWorkStatus, + OpportunityAccess, Payment, VisitReviewStatus, VisitValidationStatus, @@ -43,14 +44,30 @@ def update_status(completed_works, opportunity_access, compute_payment=True): opportunity_access.save() -def update_work_payment_date(payment: Payment): - completed_works = CompletedWork.objects.filter(opportunity_access=payment.opportunity_access) - paid = payment.amount +def update_work_payment_date(access: OpportunityAccess): + payments = Payment.objects.filter(opportunity_access=access).order_by("date_paid") + completed_works = CompletedWork.objects.filter(opportunity_access=access).order_by("status_modified_date") + + paid = 0 works_to_update = [] + completed_works_iter = iter(completed_works) + + try: + current_work = next(completed_works_iter) + except StopIteration: + return + + for payment in payments: + paid += payment.amount + + while paid > current_work.payment_accrued: + current_work.payment_date = payment.date_paid + works_to_update.append(current_work) - for current_work in completed_works: - if paid > current_work.payment_accrued: - works_to_update.append(current_work.id) + try: + current_work = next(completed_works_iter) + except StopIteration: + break if works_to_update: - CompletedWork.objects.filter(id__in=works_to_update).update(payment_date=payment.date_paid) + CompletedWork.objects.bulk_update(works_to_update, ["payment_date"]) diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py index f1afc3b6..ec2258df 100644 --- a/commcare_connect/opportunity/visit_import.py +++ b/commcare_connect/opportunity/visit_import.py @@ -263,7 +263,7 @@ def _bulk_update_payments(opportunity: Opportunity, imported_data: Dataset) -> P payment = Payment.objects.create(opportunity_access=access, amount=amount) seen_users.add(username) payment_ids.append(payment.pk) - update_work_payment_date(payment) + update_work_payment_date(access) missing_users = set(usernames) - seen_users send_payment_notification.delay(opportunity.id, payment_ids) return PaymentImportStatus(seen_users, missing_users) From c86286c7c05f728dc0070b8b5d78acc733608b99 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Fri, 4 Oct 2024 19:12:34 +0530 Subject: [PATCH 4/6] added test --- .../opportunity/tests/factories.py | 9 ++ .../opportunity/tests/test_visit_import.py | 90 +++++++++++++++++++ .../opportunity/utils/completed_work.py | 23 +++-- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py index a1d2bd15..44d5b5a1 100644 --- a/commcare_connect/opportunity/tests/factories.py +++ b/commcare_connect/opportunity/tests/factories.py @@ -216,3 +216,12 @@ class DeliveryTypeFactory(DjangoModelFactory): class Meta: model = "opportunity.DeliveryType" + + +class PaymentFactory(DjangoModelFactory): + opportunity_access = SubFactory(OpportunityAccessFactory) + amount = Faker("pyint", min_value=1, max_value=10000) + date_paid = Faker("past_date") + + class Meta: + model = "opportunity.Payment" diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py index 85ac8ff8..8dfd5ab8 100644 --- a/commcare_connect/opportunity/tests/test_visit_import.py +++ b/commcare_connect/opportunity/tests/test_visit_import.py @@ -1,9 +1,11 @@ import random import re +from datetime import timedelta from decimal import Decimal from itertools import chain import pytest +from django.utils import timezone from django.utils.timezone import now from tablib import Dataset @@ -24,9 +26,11 @@ CompletedWorkFactory, DeliverUnitFactory, OpportunityAccessFactory, + PaymentFactory, PaymentUnitFactory, UserVisitFactory, ) +from commcare_connect.opportunity.utils.completed_work import update_work_payment_date from commcare_connect.opportunity.visit_import import ( ImportException, VisitData, @@ -384,3 +388,89 @@ def test_bulk_update_catchments(opportunity, dataset, new_catchments, old_catchm updated_catchment.longitude == catchment.longitude + longitude_change ), f"Longitude not updated correctly for catchment {catchment.id}" assert updated_catchment.active, f"Active status not updated correctly for catchment {catchment.id}" + + +def prepare_opportunity_payment_test_data(opportunity): + user = MobileUserFactory() + access = OpportunityAccessFactory(opportunity=opportunity, user=user, accepted=True) + + payment_units = [ + PaymentUnitFactory(opportunity=opportunity, amount=100), + PaymentUnitFactory(opportunity=opportunity, amount=150), + PaymentUnitFactory(opportunity=opportunity, amount=200), + ] + + for payment_unit in payment_units: + DeliverUnitFactory.create_batch(2, payment_unit=payment_unit, app=opportunity.deliver_app, optional=False) + + completed_works = [] + for payment_unit in payment_units: + completed_work = CompletedWorkFactory( + opportunity_access=access, + payment_unit=payment_unit, + status=CompletedWorkStatus.approved.value, + payment_date=None, + ) + completed_works.append(completed_work) + for deliver_unit in payment_unit.deliver_units.all(): + UserVisitFactory( + opportunity=opportunity, + user=user, + deliver_unit=deliver_unit, + status=VisitValidationStatus.approved.value, + opportunity_access=access, + completed_work=completed_work, + ) + return user, access, payment_units, completed_works + + +@pytest.mark.django_db +def test_update_work_payment_date_partially(opportunity): + user, access, payment_units, completed_works = prepare_opportunity_payment_test_data(opportunity) + + payment_dates = [ + timezone.now() - timedelta(5), + timezone.now() - timedelta(3), + timezone.now() - timedelta(1), + ] + for date in payment_dates: + PaymentFactory(opportunity_access=access, amount=100, date_paid=date) + + update_work_payment_date(access) + + assert ( + get_assignable_completed_work_count(access) + == CompletedWork.objects.filter(opportunity_access=access, payment_date__isnull=False).count() + ) + + +@pytest.mark.django_db +def test_update_work_payment_date_fully(opportunity): + user, access, payment_units, completed_works = prepare_opportunity_payment_test_data(opportunity) + + payment_dates = [ + timezone.now() - timedelta(days=5), + timezone.now() - timedelta(days=3), + timezone.now() - timedelta(days=1), + ] + amounts = [100, 150, 200] + for date, amount in zip(payment_dates, amounts): + PaymentFactory(opportunity_access=access, amount=amount, date_paid=date) + + update_work_payment_date(access) + + assert CompletedWork.objects.filter( + opportunity_access=access, payment_date__isnull=False + ).count() == get_assignable_completed_work_count(access) + + +def get_assignable_completed_work_count(access: OpportunityAccess) -> int: + total_available_amount = sum(payment.amount for payment in Payment.objects.filter(opportunity_access=access)) + total_assigned_count = 0 + completed_works = CompletedWork.objects.filter(opportunity_access=access) + for completed_work in completed_works: + if total_available_amount >= completed_work.payment_accrued: + total_available_amount -= completed_work.payment_accrued + total_assigned_count += 1 + + return total_assigned_count diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py index ee45a6d6..c8be3acb 100644 --- a/commcare_connect/opportunity/utils/completed_work.py +++ b/commcare_connect/opportunity/utils/completed_work.py @@ -48,26 +48,35 @@ def update_work_payment_date(access: OpportunityAccess): payments = Payment.objects.filter(opportunity_access=access).order_by("date_paid") completed_works = CompletedWork.objects.filter(opportunity_access=access).order_by("status_modified_date") - paid = 0 + if not payments or not completed_works: + return + works_to_update = [] completed_works_iter = iter(completed_works) + current_work = next(completed_works_iter) - try: - current_work = next(completed_works_iter) - except StopIteration: - return + remaining_amount = 0 for payment in payments: - paid += payment.amount + remaining_amount += payment.amount - while paid > current_work.payment_accrued: + while remaining_amount >= current_work.payment_accrued: current_work.payment_date = payment.date_paid works_to_update.append(current_work) + remaining_amount -= current_work.payment_accrued try: current_work = next(completed_works_iter) except StopIteration: break + else: + continue + + # we've broken out of the inner while loop so all completed_works are processed. + break + + for cw in works_to_update: + print("cw before saving", cw.payment_date) if works_to_update: CompletedWork.objects.bulk_update(works_to_update, ["payment_date"]) From 6d2c08c33a8757e91145f53e12e6667b1a703dc2 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Fri, 4 Oct 2024 19:16:03 +0530 Subject: [PATCH 5/6] removed print statement --- commcare_connect/opportunity/utils/completed_work.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py index c8be3acb..df2d9906 100644 --- a/commcare_connect/opportunity/utils/completed_work.py +++ b/commcare_connect/opportunity/utils/completed_work.py @@ -75,8 +75,5 @@ def update_work_payment_date(access: OpportunityAccess): # we've broken out of the inner while loop so all completed_works are processed. break - for cw in works_to_update: - print("cw before saving", cw.payment_date) - if works_to_update: CompletedWork.objects.bulk_update(works_to_update, ["payment_date"]) From dc24d5a6b5db1e2c9c18b0a0030e54865a23eb46 Mon Sep 17 00:00:00 2001 From: hemant10yadav Date: Fri, 4 Oct 2024 22:37:34 +0530 Subject: [PATCH 6/6] added test for date check --- .../opportunity/tests/test_visit_import.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py index 8dfd5ab8..4c5f10ee 100644 --- a/commcare_connect/opportunity/tests/test_visit_import.py +++ b/commcare_connect/opportunity/tests/test_visit_import.py @@ -464,6 +464,71 @@ def test_update_work_payment_date_fully(opportunity): ).count() == get_assignable_completed_work_count(access) +@pytest.mark.django_db +def test_update_work_payment_date_with_precise_dates(opportunity): + user = MobileUserFactory() + access = OpportunityAccessFactory(opportunity=opportunity, user=user, accepted=True) + + payment_units = [ + PaymentUnitFactory(opportunity=opportunity, amount=5), + PaymentUnitFactory(opportunity=opportunity, amount=5), + ] + + for payment_unit in payment_units: + DeliverUnitFactory.create_batch(2, payment_unit=payment_unit, app=opportunity.deliver_app, optional=False) + + completed_work_1 = CompletedWorkFactory( + opportunity_access=access, + payment_unit=payment_units[0], + status=CompletedWorkStatus.approved.value, + payment_date=None, + ) + + completed_work_2 = CompletedWorkFactory( + opportunity_access=access, + payment_unit=payment_units[1], + status=CompletedWorkStatus.approved.value, + payment_date=None, + ) + + create_user_visits_for_completed_work(opportunity, user, access, payment_units[0], completed_work_1) + create_user_visits_for_completed_work(opportunity, user, access, payment_units[1], completed_work_2) + + now = timezone.now() + + payment_1 = PaymentFactory(opportunity_access=access, amount=7) + payment_2 = PaymentFactory(opportunity_access=access, amount=3) + + payment_1.date_paid = now - timedelta(3) + payment_2.date_paid = now - timedelta(1) + payment_1.save() + payment_2.save() + + payment_1.refresh_from_db() + payment_2.refresh_from_db() + + update_work_payment_date(access) + + completed_work_1.refresh_from_db() + completed_work_2.refresh_from_db() + + assert completed_work_1.payment_date == payment_1.date_paid + + assert completed_work_2.payment_date == payment_2.date_paid + + +def create_user_visits_for_completed_work(opportunity, user, access, payment_unit, completed_work): + for deliver_unit in payment_unit.deliver_units.all(): + UserVisitFactory( + opportunity=opportunity, + user=user, + deliver_unit=deliver_unit, + status=VisitValidationStatus.approved.value, + opportunity_access=access, + completed_work=completed_work, + ) + + def get_assignable_completed_work_count(access: OpportunityAccess) -> int: total_available_amount = sum(payment.amount for payment in Payment.objects.filter(opportunity_access=access)) total_assigned_count = 0