fix: move celery logging to Django (#3809)

* fix: move celery logging to Django

* fix: use json formatter for k8s console logs

* fix: bump celery to 5.4.0

backend/mlarchive/celeryapp.py

@@ -1,17 +1,22 @@
 import os
+import celery
 
-from celery import Celery
 from django.conf import settings
 
+
+# Disable celery's internal logging configuration, we set it up via Django
+@celery.signals.setup_logging.connect
+def on_setup_logging(**kwargs):
+    pass
+
+
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mlarchive.settings.settings')
 
-app = Celery('mlarchive')
+app = celery.Celery('mlarchive')
 app.config_from_object('django.conf:settings', namespace='CELERY')
-
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 
+
 @app.task(bind=True)
 def debug_task(self):
     print('Request: {0!r}'.format(self.request))
-
-

backend/mlarchive/settings/base.py

@@ -418,6 +418,11 @@
     'formatters': {
         'simple': {
             'format': "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        },
+        'json': {
+            "class": "mlarchive.utils.jsonlogger.MailArchiveJsonFormatter",
+            "style": "{",
+            "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}",
         }
     },
     'handlers': {
@@ -446,6 +451,10 @@
             'handlers': ['mlarchive'],
             'level': 'DEBUG',
             'propagate': False,
-        }
+        },
+        'celery': {
+            'handlers': ['console'],
+            'level': 'INFO',
+        },
     }
 }

backend/mlarchive/settings/settings.py

@@ -1,2 +1,5 @@
 # settings/settings.py
 from .base import *
+
+# Console logs as JSON instead of plain when running in k8s
+LOGGING["handlers"]["console"]["formatter"] = "json"

backend/mlarchive/utils/jsonlogger.py

@@ -0,0 +1,8 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+from pythonjsonlogger import jsonlogger
+import time
+
+
+class MailArchiveJsonFormatter(jsonlogger.JsonFormatter):
+    converter = time.gmtime  # use UTC
+    default_msec_format = "%s.%03d"  # '.' instead of ','

build/app/celery-start.sh

@@ -1,8 +1,34 @@
-#!/bin/bash
+#!/bin/bash -e
 #
 # Run a celery worker
 #
-# echo "Running Mailarchive checks..."
-# ./backend/manage.py check
 
-celery "$@"
+# echo "Running Datatracker checks..."
+# ./ietf/manage.py check
+
+# if ! ietf/manage.py migrate --check ; then
+#     echo "Unapplied migrations found, waiting to start..."
+#     sleep 5
+#     while ! ietf/manage.py migrate --check ; do
+#         echo "... still waiting for migrations..."
+#         sleep 5
+#     done
+# fi
+
+echo "Starting Celery..."
+
+cleanup () {
+  # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
+  if [[ -n "${celery_pid}" ]]; then
+    echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..."
+    kill -TERM "${celery_pid}"
+    wait "${celery_pid}"
+  fi
+}
+
+trap 'trap "" TERM; cleanup' TERM
+
+# start celery in the background so we can trap the TERM signal
+celery "$@" &
+celery_pid=$!
+wait "${celery_pid}"

requirements.txt

@@ -7,7 +7,7 @@
 #setuptools==53.0.0         # Require this first, to prevent later errors
 #bs4			# 4.1.3 was installed
 beautifulsoup4
-celery==5.2.7
+celery==5.4.0
 cloudflare==2.12.4
 cryptography
 Django>=4.1,<5.0
@@ -34,6 +34,7 @@ mozilla-django-oidc==2.0.0
 passlib
 psycopg==3.1.12
 pymemcache==3.5.2
+python-json-logger>=2.0.7
 pyopenssl
 pyquery
 pytest