Redis backend update (#4376)

* Redis backend update * Small redis tweaks --------- Co-authored-by: Katherine Fleming <[email protected]>
SEED-platform · Feb 13, 2024 · a5a2a9d · a5a2a9d
1 parent a6a68e0
commit a5a2a9d
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 118 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -116,6 +116,7 @@
     "edgecase",
     "eeej",
     "ejscreen",
+    "ELASTICACHE",
     "energystar",
     "enums",
     "env",
@@ -393,6 +394,7 @@
     "unmerge",
     "unmerges",
     "unpair",
+    "Unpickles",
     "unresolvable",
     "untracked",
     "uom",

diff --git a/config/settings/docker.py b/config/settings/docker.py
@@ -24,7 +24,7 @@
 # The optional vars will set the SERVER_EMAIL information as needed
 OPTIONAL_ENV_VARS = ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SES_REGION_NAME',
                      'AWS_SES_REGION_ENDPOINT', 'SERVER_EMAIL', 'SENTRY_JS_DSN', 'SENTRY_RAVEN_DSN',
-                     'REDIS_PASSWORD', 'REDIS_HOST', 'REDIS_AWS_ELASTICACHE', 'REDIS_URL', 'DJANGO_EMAIL_BACKEND',
+                     'REDIS_PASSWORD', 'REDIS_HOST', 'REDIS_AWS_ELASTICACHE', 'DJANGO_EMAIL_BACKEND',
                      'POSTGRES_HOST'] + SMTP_ENV_VARS
 
 for loc in ENV_VARS + OPTIONAL_ENV_VARS:
@@ -77,57 +77,33 @@
 
 # Redis / Celery config
 if 'REDIS_AWS_ELASTICACHE' in os.environ:
+    CELERY_BROKER_URL = f"rediss://:{os.environ.get('REDIS_PASSWORD')}@{os.environ.get('REDIS_HOST')}:6379/1?ssl_cert_reqs=required"
+
     CACHES = {
         'default': {
             'BACKEND': 'django_redis.cache.RedisCache',
-            'LOCATION': os.environ.get('REDIS_URL'),
-            'OPTIONS': {
-                'DB': 1,
-                "CLIENT_CLASS": "django_redis.client.DefaultClient",
-                'PASSWORD': os.environ.get('REDIS_PASSWORD'),
-                "SSL": True  # If your AWS Elasticache is set up for SSL, set this to True
-            },
-            'TIMEOUT': 300
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'rediss://:%s@%s:6379/%s?ssl_cert_reqs=required' % (
-        CACHES['default']['OPTIONS']['PASSWORD'],
-        os.environ.get('REDIS_HOST'),
-        CACHES['default']['OPTIONS']['DB']
-    )
 elif 'REDIS_PASSWORD' in os.environ:
+    CELERY_BROKER_URL = f"redis://:{os.environ.get('REDIS_PASSWORD')}@{os.environ.get('REDIS_HOST', 'db-redis')}:6379/1"
+
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': f"{os.environ.get('REDIS_HOST', 'db-redis')}:6379",
-            'OPTIONS': {
-                'DB': 1,
-                'PASSWORD': REDIS_PASSWORD,  # noqa F405
-            },
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://:%s@%s/%s' % (
-        CACHES['default']['OPTIONS']['PASSWORD'],
-        CACHES['default']['LOCATION'],
-        CACHES['default']['OPTIONS']['DB']
-    )
 else:
+    CELERY_BROKER_URL = f"redis://{os.environ.get('REDIS_HOST', 'db-redis')}:6379/1"
+
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': os.environ.get('REDIS_HOST', 'db-redis:6379'),
-            'OPTIONS': {
-                'DB': 1
-            },
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://%s/%s' % (
-        CACHES['default']['LOCATION'], CACHES['default']['OPTIONS']['DB']
-    )
 
-CELERY_BROKER_TRANSPORT = 'redis'
 CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 CELERY_TASK_DEFAULT_QUEUE = 'seed-docker'
 CELERY_TASK_QUEUES = (

diff --git a/config/settings/docker_dev.py b/config/settings/docker_dev.py
@@ -72,23 +72,18 @@
     TESTING_MAPQUEST_API_KEY = os.environ.get('TESTING_MAPQUEST_API_KEY', '<your_key_here>')
 else:
     # Redis / Celery config
+    if 'REDIS_PASSWORD' in os.environ:
+        CELERY_BROKER_URL = f"redis://:{os.environ.get('REDIS_PASSWORD')}@{os.environ.get('REDIS_HOST', 'db-redis')}:6379/1"
+    else:
+        CELERY_BROKER_URL = f"redis://{os.environ.get('REDIS_HOST', 'db-redis')}:6379/1"
+
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': f"{os.environ.get('REDIS_HOST', 'db-redis')}:6379",
-            'OPTIONS': {
-                'DB': 1
-            },
-            'TIMEOUT': 300,
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    if 'REDIS_PASSWORD' in os.environ:
-        CACHES['OPTIONS']['PASSWORD'] = os.environ.get('REDIS_PASSWORD')
-        CELERY_BROKER_URL = f"redis://:{CACHES['default']['OPTIONS']['PASSWORD']}@{CACHES['default']['LOCATION']}/{CACHES['default']['OPTIONS']['DB']}"
-    else:
-        CELERY_BROKER_URL = f"redis://{CACHES['default']['LOCATION']}/{CACHES['default']['OPTIONS']['DB']}"
 
-    CELERY_BROKER_TRANSPORT = 'redis'
     CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 
 CELERY_TASK_DEFAULT_QUEUE = 'seed-docker'

diff --git a/config/settings/local_untracked.py.dist b/config/settings/local_untracked.py.dist
@@ -95,18 +95,15 @@ DATABASES = {
 }
 
 # =============================== Celery/Redis Cache Settings (No Password) =========
+CELERY_BROKER_URL = 'redis://redis-hostname:6379/1'
+
 CACHES = {
     'default': {
-        'BACKEND': 'redis_cache.cache.RedisCache',
-        'LOCATION': 'your-cache-url:your-cache-port',
-        'OPTIONS': {'DB': 1},
-        'TIMEOUT': 300
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': CELERY_BROKER_URL,
     }
 }
 
-CELERY_BROKER_URL = 'redis://%s/%s' % (
-    CACHES['default']['LOCATION'], CACHES['default']['OPTIONS']['DB']
-)
 CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 CELERY_TASK_DEFAULT_QUEUE = 'seed-local'
 CELERY_TASK_QUEUES = (
@@ -118,32 +115,24 @@ CELERY_TASK_QUEUES = (
 )
 
 # =============================== Celery/Redis Cache Settings (w/Password) =========
-#CACHES = {
+# CELERY_BROKER_URL = 'redis://:your-redis-password@your-cache-url:6379/1'
+#
+# CACHES = {
 #   'default': {
-#        'BACKEND': 'redis_cache.cache.RedisCache',
-#        'LOCATION': 'your-cache-url:your-cache-port',
-#        'OPTIONS': {
-#            'DB': 1,
-#             'PASSWORD': 'your-redis-password',
-#        },
-#        'TIMEOUT': 300
+#        'BACKEND': 'django_redis.cache.RedisCache',
+#        'LOCATION': CELERY_BROKER_URL,
 #    }
-#}
+# }
 #
-#CELERY_BROKER_URL = 'redis://:%s@%s/%s' % (
-#    CACHES['default']['OPTIONS']['PASSWORD'],
-#    CACHES['default']['LOCATION'],
-#    CACHES['default']['OPTIONS']['DB']
-#)
-#CELERY_RESULT_BACKEND = CELERY_BROKER_URL
-#CELERY_TASK_DEFAULT_QUEUE = 'seed-local'
-#CELERY_TASK_QUEUES = (
+# CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+# CELERY_TASK_DEFAULT_QUEUE = 'seed-local'
+# CELERY_TASK_QUEUES = (
 #    Queue(
 #        CELERY_TASK_DEFAULT_QUEUE,
 #        Exchange(CELERY_TASK_DEFAULT_QUEUE),
 #        routing_key=CELERY_TASK_DEFAULT_QUEUE
 #    ),
-#)
+# )
 
 
 # =================================== Logging =======================================

diff --git a/config/settings/test_local_untracked.py b/config/settings/test_local_untracked.py
@@ -32,27 +32,23 @@
 }
 
 # redis cache config
-# with AWS ElastiCache redis, the LOCATION setting looks something like:
-# 'xx-yy-zzrr0aax9a.ntmprk.0001.usw2.cache.amazonaws.com:6379'
+# with AWS ElastiCache redis, the CELERY_BROKER_URL setting looks something like:
+# 'rediss://:password@xx-yy-zzrr0aax9a.ntmprk.0001.usw2.cache.amazonaws.com:6379/1?ssl_cert_reqs=required'
 
 EAGER = os.environ.get('CELERY_ALWAYS_EAGER', 'True') == 'True'
 if EAGER:
     CELERY_BROKER_BACKEND = 'memory'
     CELERY_TASK_ALWAYS_EAGER = True
     CELERY_TASK_EAGER_PROPAGATES = True
 else:
-    print("Using redis database")
+    print('Using redis database')
+    CELERY_BROKER_URL = 'redis://localhost:6379/1'
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': "localhost:6379",
-            'OPTIONS': {'DB': 1},
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://%s/%s' % (
-        CACHES['default']['LOCATION'], CACHES['default']['OPTIONS']['DB']
-    )
     CELERY_RESULT_BACKEND = CELERY_BROKER_URL
     CELERY_TASK_DEFAULT_QUEUE = 'seed-local'
     CELERY_TASK_QUEUES = (

diff --git a/docs/source/aws.rst b/docs/source/aws.rst
@@ -161,15 +161,13 @@ settings.
 
 .. code-block:: python
 
+    CELERY_BROKER_URL = 'redis://seed-core-cache.ntmprk.0001.usw2.cache.amazonaws.com:6379/1'
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': "seed-core-cache.ntmprk.0001.usw2.cache.amazonaws.com:6379",
-            'OPTIONS': { 'DB': 1 },
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://seed-core-cache.ntmprk.0001.usw2.cache.amazonaws.com:6379/1'
 
 Running Celery the Background Task Worker
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

diff --git a/docs/source/linux.rst b/docs/source/linux.rst
@@ -143,15 +143,13 @@ settings.
 
 .. code-block:: python
 
+    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/1'
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': '127.0.0.1:6379',
-            'OPTIONS': {'DB': 1},
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/1'
 
 
 Creating the initial user
@@ -320,15 +318,13 @@ local_untracked.py
     # config for local storage backend
     DOMAIN_URLCONFS = {'default': 'config.urls'}
 
+    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/1'
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': '127.0.0.1:6379',
-            'OPTIONS': {'DB': 1},
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/1'
 
     # SMTP config
     EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'

diff --git a/docs/source/migrations.rst b/docs/source/migrations.rst
@@ -15,28 +15,19 @@ Docker then you will not need to do this.
     CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 
 If you are using a password, then in your local_untracked.py configuration, add the password to
-the CACHES configuration option. Your final configuration should look like the following in your
+the CELERY_BROKER_URL. Your final configuration should look like the following in your
 local_untracked.py file
 
 .. code-block:: python
 
+    CELERY_BROKER_URL = 'redis://:[email protected]:6379/1'
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': "127.0.0.1:6379",
-            'OPTIONS': {
-                'DB': 1,
-                'PASSWORD': 'password',
-            },
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
 
-    CELERY_BROKER_URL = 'redis://:%s@%s/%s' % (
-        CACHES['default']['OPTIONS']['PASSWORD'],
-        CACHES['default']['LOCATION'],
-        CACHES['default']['OPTIONS']['DB']
-    )
     CELERY_RESULT_BACKEND = CELERY_BROKER_URL
     CELERY_TASK_DEFAULT_QUEUE = 'seed-local'
     CELERY_TASK_QUEUES = (

diff --git a/docs/source/setup_osx.rst b/docs/source/setup_osx.rst
@@ -270,15 +270,13 @@ For Redis, edit the `CACHES` and `CELERY_BROKER_URL` values to look like this:
 
 .. code-block:: python
 
+    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/1'
     CACHES = {
         'default': {
-            'BACKEND': 'redis_cache.cache.RedisCache',
-            'LOCATION': "127.0.0.1:6379",
-            'OPTIONS': {'DB': 1},
-            'TIMEOUT': 300
+            'BACKEND': 'django_redis.cache.RedisCache',
+            'LOCATION': CELERY_BROKER_URL,
         }
     }
-    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/1'
 
 MapQuest API Key
 ----------------

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -11,11 +11,13 @@ psycopg2-binary==2.9.6
 
 # background process management
 celery==5.2.2
-django-redis-cache==3.0.0
+django-celery-beat==2.2.1
+django-redis==5.4.0
+hiredis==2.3.2
+
 django-compressor==4.4
 django-extensions==3.1.3
 django-model-utils==4.3.1
-django-celery-beat==2.2.1
 
 # Time zones support - Do not update these without doing significant testing!
 pytz==2018.7