Merge branch 'main' into remove_old_bcast_code

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     env:
-      python-version: "3.10.x"
+      python-version: "3.11.x"
       node-version: "20"
       mailroom-version: "9.1.52"
       DJANGO_SETTINGS_MODULE: temba.settings_ci

CHANGELOG.md

@@ -1,3 +1,25 @@
+v9.1.172 (2024-07-04)
+-------------------------
+ * Add templates to broadcasts (hidden for now)
+ * Remove deprecated broadcast.template_state field on mailroom queue payload
+
+v9.1.171 (2024-07-03)
+-------------------------
+ * Update payload for queueing a bradocast
+
+v9.1.170 (2024-07-03)
+-------------------------
+ * Remove no longer needed task to sync stale Android relayers
+ * Don't allow template localization
+ * Update dependencies
+
+v9.1.169 (2024-07-02)
+-------------------------
+ * Use python 3.11.x
+ * Add Broadcast.template_variables
+ * Add new template list and read pages and remove old channel specific ones
+ * Fix globals list template
+
 v9.1.168 (2024-06-28)
 -------------------------
  * Don't sync classifiers in suspended orgs

package.json

@@ -9,8 +9,8 @@
     ]
   },
   "dependencies": {
-    "@nyaruka/flow-editor": "1.33.0",
-    "@nyaruka/temba-components": "0.94.5",
+    "@nyaruka/temba-components": "0.95.0",
+    "@nyaruka/flow-editor": "1.34.0",
     "codemirror": "5.18.2",
     "colorette": "1.2.2",
     "fa-icons": "0.2.0",

pyproject.toml

@@ -1,14 +1,14 @@
 [tool.poetry]
 name = "temba"
-version = "9.1.168"
+version = "9.1.172"
 description = "Hosted service for visually building interactive messaging applications"
 authors = ["Nyaruka <[email protected]>"]
 
 [tool.poetry.urls]
 repository = "http://github.com/rapidpro/rapidpro"
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "~3.11"
 Django = "^5.0"
 django-compressor = "^4.3.1"
 django-countries = "^7.0"
@@ -19,9 +19,9 @@ django-timezone-field = "^6.1.0"
 djangorestframework = "^3.15.1"
 dj-database-url = "^0.5.0"
 smartmin = "^5.0.7"
-celery = "^5.1.0"
-redis = "^4.5.4"
-boto3 = "^1.28.3"
+celery = "^5.4.0"
+redis = "^5.0.7"
+boto3 = "^1.34.137"
 cryptography = "^42.0.4"
 vonage = "2.5.2"
 pyotp = "2.4.1"
@@ -36,13 +36,13 @@ pyexcel-xls = "^0.7.0"
 pyexcel-xlsx = "^0.6.0"
 python-magic = "^0.4.22"
 xlsxlite = "^0.2.0"
-colorama = "^0.4.4"
+colorama = "^0.4.6"
 gunicorn = "^22.0.0"
 iptools = "^0.7.0"
 iso8601 = "^0.1.14"
-phonenumbers = "^8.13.16"
+phonenumbers = "^8.13.40"
 pycountry = "^22.3.5"
-python-dateutil = "^2.8.2"
+python-dateutil = "^2.9.0"
 packaging = "^22.0"
 requests-toolbelt = "^1.0.0"
 chardet = "^4.0.0"
@@ -57,11 +57,11 @@ iso639-lang = "^2.2.3"
 google-auth = "^2.30.0"
 
 [tool.poetry.group.dev.dependencies]
-black = "^24.3.0"
-coverage = {extras = ["toml"], version = "^7.2.7"}
+black = "^24.4.2"
+coverage = {extras = ["toml"], version = "^7.5.4"}
 isort = "^5.13.2"
-responses = "^0.12.1"
-ruff = "^0.3.3"
+responses = "^0.25.3"
+ruff = "^0.5.0"
 djlint = "^1.34.1"
 
 [tool.poetry_bumpversion.file."temba/__init__.py"]

temba/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "9.1.168"
+__version__ = "9.1.172"
 
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.

temba/api/internal/serializers.py

@@ -2,6 +2,7 @@
 
 from rest_framework import serializers
 
+from temba.locations.models import AdminBoundary
 from temba.templates.models import Template, TemplateTranslation
 
 
@@ -10,6 +11,12 @@ def to_representation(self, instance):
         return instance.as_json()
 
 
+class LocationReadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = AdminBoundary
+        fields = ("osm_id", "name", "path")
+
+
 class TemplateReadSerializer(serializers.ModelSerializer):
     STATUSES = {
         TemplateTranslation.STATUS_APPROVED: "approved",

temba/api/internal/tests.py

@@ -12,6 +12,64 @@
 
 
 class EndpointsTest(APITestMixin, TembaTest):
+    def test_locations(self):
+        endpoint_url = reverse("api.internal.locations") + ".json"
+
+        self.assertGetNotPermitted(endpoint_url, [None])
+        self.assertPostNotAllowed(endpoint_url)
+        self.assertDeleteNotAllowed(endpoint_url)
+
+        # no country, no results
+        self.assertGet(endpoint_url + "?level=state", [self.agent], results=[])
+
+        self.setUpLocations()
+
+        self.assertGet(
+            endpoint_url + "?level=state",
+            [self.agent],
+            results=[
+                {"osm_id": "171591", "name": "Eastern Province", "path": "Rwanda > Eastern Province"},
+                {"osm_id": "1708283", "name": "Kigali City", "path": "Rwanda > Kigali City"},
+            ],
+            num_queries=NUM_BASE_QUERIES + 2,
+        )
+        self.assertGet(
+            endpoint_url + "?level=district",
+            [self.user],
+            results=[
+                {"osm_id": "R1711131", "name": "Gatsibo", "path": "Rwanda > Eastern Province > Gatsibo"},
+                {"osm_id": "1711163", "name": "Kayônza", "path": "Rwanda > Eastern Province > Kayônza"},
+                {"osm_id": "3963734", "name": "Nyarugenge", "path": "Rwanda > Kigali City > Nyarugenge"},
+                {"osm_id": "1711142", "name": "Rwamagana", "path": "Rwanda > Eastern Province > Rwamagana"},
+            ],
+        )
+
+        # can query on name
+        self.assertGet(
+            endpoint_url + "?level=district&query=ga",
+            [self.editor],
+            results=[
+                {"osm_id": "R1711131", "name": "Gatsibo", "path": "Rwanda > Eastern Province > Gatsibo"},
+                {"osm_id": "1711142", "name": "Rwamagana", "path": "Rwanda > Eastern Province > Rwamagana"},
+            ],
+        )
+
+        # or alias
+        self.assertGet(
+            endpoint_url + "?level=state&query=kigari",
+            [self.admin],
+            results=[
+                {"osm_id": "1708283", "name": "Kigali City", "path": "Rwanda > Kigali City"},
+            ],
+        )
+
+        # but not aliases in other orgs
+        self.assertGet(endpoint_url + "?level=state&query=Chigali", [self.agent], results=[])
+
+        # missing or invalid level, no results
+        self.assertGet(endpoint_url + "?level=hood", [self.agent], results=[])
+        self.assertGet(endpoint_url, [self.agent], results=[])
+
     def test_notifications(self):
         endpoint_url = reverse("api.internal.notifications") + ".json"
 

temba/api/internal/urls.py

@@ -2,10 +2,11 @@
 
 from django.urls import re_path
 
-from .views import NotificationsEndpoint, TemplatesEndpoint
+from .views import LocationsEndpoint, NotificationsEndpoint, TemplatesEndpoint
 
 urlpatterns = [
     # ========== endpoints A-Z ===========
+    re_path(r"^locations$", LocationsEndpoint.as_view(), name="api.internal.locations"),
     re_path(r"^notifications$", NotificationsEndpoint.as_view(), name="api.internal.notifications"),
     re_path(r"^templates$", TemplatesEndpoint.as_view(), name="api.internal.templates"),
 ]

temba/api/internal/views.py

@@ -1,16 +1,18 @@
 from rest_framework import status
+from rest_framework.pagination import CursorPagination
 from rest_framework.response import Response
 
-from django.db.models import Prefetch
+from django.db.models import Prefetch, Q
 
 from temba.channels.models import Channel
+from temba.locations.models import AdminBoundary
 from temba.notifications.models import Notification
 from temba.templates.models import Template, TemplateTranslation
 
 from ..models import APIPermission, SSLPermission
 from ..support import APISessionAuthentication, CreatedOnCursorPagination, ModifiedOnCursorPagination
 from ..views import BaseAPIView, ListAPIMixin
-from .serializers import ModelAsJsonSerializer, TemplateReadSerializer
+from . import serializers
 
 
 class BaseEndpoint(BaseAPIView):
@@ -27,10 +29,47 @@ class BaseEndpoint(BaseAPIView):
 # ============================================================
 
 
+class LocationsEndpoint(ListAPIMixin, BaseEndpoint):
+    """
+    Admin boundaries searchable by name at a specified level.
+    """
+
+    LEVELS = {
+        "state": AdminBoundary.LEVEL_STATE,
+        "district": AdminBoundary.LEVEL_DISTRICT,
+        "ward": AdminBoundary.LEVEL_WARD,
+    }
+
+    class Pagination(CursorPagination):
+        ordering = ("name", "id")
+        offset_cutoff = 100000
+
+    model = AdminBoundary
+    serializer_class = serializers.LocationReadSerializer
+    pagination_class = Pagination
+
+    def derive_queryset(self):
+        org = self.request.org
+        level = self.LEVELS.get(self.request.query_params.get("level"))
+        query = self.request.query_params.get("query")
+
+        if not org.country or not level:
+            return AdminBoundary.objects.none()
+
+        qs = AdminBoundary.objects.filter(
+            path__startswith=f"{org.country.name} {AdminBoundary.PATH_SEPARATOR}", level=level
+        )
+
+        if query:
+            qs = qs.filter(Q(name__icontains=query) | Q(aliases__org=org, aliases__name__icontains=query))
+
+        return qs.only("osm_id", "name", "path")
+
+
 class NotificationsEndpoint(ListAPIMixin, BaseEndpoint):
     model = Notification
     pagination_class = CreatedOnCursorPagination
-    serializer_class = ModelAsJsonSerializer
+    serializer_class = serializers.ModelAsJsonSerializer
 
     def get_queryset(self):
         return (
@@ -52,7 +91,7 @@ class TemplatesEndpoint(ListAPIMixin, BaseEndpoint):
     """
 
     model = Template
-    serializer_class = TemplateReadSerializer
+    serializer_class = serializers.TemplateReadSerializer
     pagination_class = ModifiedOnCursorPagination
 
     def filter_queryset(self, queryset):

temba/api/v2/tests.py

@@ -810,7 +810,7 @@ def test_archives(self):
     def test_boundaries(self):
         endpoint_url = reverse("api.v2.boundaries") + ".json"
 
-        self.assertGetNotPermitted(endpoint_url, [None, self.agent])
+        self.assertGetNotPermitted(endpoint_url, [None])
         self.assertPostNotAllowed(endpoint_url)
         self.assertDeleteNotAllowed(endpoint_url)
 

temba/channels/tasks.py

@@ -53,20 +53,6 @@ def check_android_channels():
         ChannelDisconnectedIncidentType.get_or_create(channel)
 
 
-@cron_task()
-def sync_old_seen_channels():
-    from temba.channels.types.android import AndroidType
-
-    now = timezone.now()
-    window_end = now - timedelta(minutes=15)
-    window_start = now - timedelta(days=7)
-    old_seen_channels = Channel.objects.filter(
-        is_active=True, channel_type=AndroidType.code, last_seen__lte=window_end, last_seen__gt=window_start
-    )
-    for channel in old_seen_channels:
-        channel.trigger_sync()
-
-
 @shared_task
 def interrupt_channel_task(channel_id):
     channel = Channel.objects.get(pk=channel_id)

temba/channels/tests.py

@@ -36,7 +36,6 @@
 from .tasks import (
     check_android_channels,
     squash_channel_counts,
-    sync_old_seen_channels,
     track_org_channel_counts,
     trim_channel_events,
     trim_channel_logs,
@@ -1086,28 +1085,6 @@ def test_sync_event_model(self, mr_mocks):
         self.assertEqual("RW", self.tel_channel.country)
 
 
-class ChannelSyncTest(TembaTest):
-    @patch("temba.channels.models.Channel.trigger_sync")
-    def test_sync_old_seen_chaanels(self, mock_trigger_sync):
-        self.channel.last_seen = timezone.now() - timedelta(days=40)
-        self.channel.save()
-
-        sync_old_seen_channels()
-        self.assertFalse(mock_trigger_sync.called)
-
-        self.channel.last_seen = timezone.now() - timedelta(minutes=5)
-        self.channel.save()
-
-        sync_old_seen_channels()
-        self.assertFalse(mock_trigger_sync.called)
-
-        self.channel.last_seen = timezone.now() - timedelta(hours=3)
-        self.channel.save()
-
-        sync_old_seen_channels()
-        self.assertTrue(mock_trigger_sync.called)
-
-
 class ChannelIncidentsTest(TembaTest):
     def test_disconnected(self):
         # set our last seen to a while ago

temba/channels/types/dialog360/type.py

@@ -35,8 +35,6 @@ class Dialog360Type(ChannelType):
 
     config_ui = ConfigUI()  # has own template
 
-    menu_items = [dict(label=_("Message Templates"), view_name="templates.templatetranslation_channel")]
-
     def get_headers(self, channel):
         return {"D360-API-KEY": channel.config[Channel.CONFIG_AUTH_TOKEN], "Content-Type": "application/json"}
 

temba/channels/types/dialog360_legacy/type.py

@@ -34,8 +34,6 @@ class Dialog360LegacyType(ChannelType):
 
     config_ui = ConfigUI()  # has own template
 
-    menu_items = [dict(label=_("Message Templates"), view_name="templates.templatetranslation_channel")]
-
     def get_headers(self, channel):
         return {"D360-API-KEY": channel.config[Channel.CONFIG_AUTH_TOKEN], "Content-Type": "application/json"}
 

temba/channels/types/justcall/tests.py

@@ -32,7 +32,7 @@ def test_claim(self):
         # try once with an error
         with patch("requests.post") as mock_post:
             mock_post.return_value = MockResponse(400, '{ "message": "Failed try again later." }')
-            with self.assertRaisesRegexp(ValidationError, "Unable to add webhook to JustCall: Failed try again later."):
+            with self.assertRaisesRegex(ValidationError, "Unable to add webhook to JustCall: Failed try again later."):
                 response = self.client.post(url, post_data)
             self.assertFalse(Channel.objects.all())
 
@@ -41,7 +41,7 @@ def test_claim(self):
                 MockResponse(200, '{""}'),
                 MockResponse(400, '{ "message": "Failed try again later." }'),
             ]
-            with self.assertRaisesRegexp(ValidationError, "Unable to add webhook to JustCall: Failed try again later."):
+            with self.assertRaisesRegex(ValidationError, "Unable to add webhook to JustCall: Failed try again later."):
                 response = self.client.post(url, post_data)
             self.assertFalse(Channel.objects.all())