feat: switching to rabbitmq, adding flower and pgadmin

.env.ci

@@ -1,5 +1,34 @@
 # Docker Compose environment file used in CI.
 
+# -- Stack Configuration -----------------------------------------------------
+
+# Backend
+BACKEND_CORS_ORIGINS=
+SECRET_KEY=ONLY-CI-secret-key
+FIRST_SUPERUSER=ONLY-CI-admin
+FIRST_SUPERUSER_PASSWORD=ONLY-CI-superuser-password
+# SMTP_TLS=True
+# SMTP_PORT=
+# SMTP_HOST=
+# SMTP_USER=
+# SMTP_PASSWORD=
+# EMAILS_FROM_EMAIL=
+
+# Postgres
+POSTGRES_SERVER=postgres
+POSTGRES_USER=reev
+POSTGRES_PASSWORD_FILE=/run/secrets/db-password
+POSTGRES_DB=reev
+
+# pgAdmin
+PGADMIN_LISTEN_PORT=80
+PGADMIN_DEFAULT_EMAIL=[email protected]
+PGADMIN_DEFAULT_PASSWORD_FILE=/run/secrets/pgadmin-password
+PGADMIN_DISABLE_POSTFIX=1
+
+# Flower
+FLOWER_BASIC_AUTH=admin:flower-password
+
 # -- Docker Images -----------------------------------------------------------
 
 # Name of the registry server and org to use for our images.

.github/workflows/ci.yml

@@ -28,18 +28,35 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
+      - name: Prepare secrets
+        run: |
+          mkdir -p .ci/secrets
+          echo db-password > .ci/secrets/db-password
+          echo pgadmin-password > .ci/secrets/pgadmin-password
+
       - name: Prepare empty volumes
         run: |
+          mkdir -p .ci/volumes/minio/data
+          mkdir -p .ci/volumes/pgadmin/data
           mkdir -p .ci/volumes/postgres/data
+          mkdir -p .ci/volumes/rabbitmq/data
           mkdir -p .ci/volumes/redis/data
-          mkdir -p .ci/volumes/minio/data
-          mkdir -p .ci/volumes/varfish-static/data
+          mkdir -p .ci/volumes/reev-static/data
+
+      - name: Prepare configuration
+        run: |
+          cp .env.ci .env
+
+          mkdir -p .ci/config/nginx
+          cp utils/nginx/nginx.conf .ci/config/nginx
+          mkdir -p .ci/config/pgadmin
+          cp utils/pgadmin/servers.json .ci/config/pgadmin
 
       - name: Bring up and shut down services
         uses: isbang/[email protected]
         with:
           compose-file: docker-compose.yml
           compose-flags: |
-            --env-file .env.ci
+            --env-file .env
         env:
           DB_PASSWORD: "password"

README.md

@@ -40,16 +40,18 @@ In a production deployment, these directories should live outside of the checkou
 Now, we create the directories for data storage.
 
 ```bash session
-mkdir -p .dev/volumes/reev-static/data
+mkdir -p .dev/volumes/pgadmin/data
 mkdir -p .dev/volumes/postgres/data
-mkdir -p .dev/volumes/redis/data
+mkdir -p .dev/volumes/rabbitmq/data
+mkdir -p .dev/volumes/reev-static/data
 ```
 
 Next, we setup some "secrets" for the passwords.
 
 ```bash session
 mkdir -p .dev/secrets
 echo db-password >.dev/secrets/db-password
+echo pgadmin-password >.dev/secrets/pgadmin-password
 ```
 
 We now copy the `env.tpl` file to the default location for the environment `.env`.
@@ -60,7 +62,7 @@ cp env.tpl .env
 
 Next, create a `docker-compose.override.yml` with the contents of the file `docker-compose.override.yml-dev`.
 This will disable everything that we assume is running on your host when you are developing.
-This includes the REEV backend, redis, celery workers, postgres.
+This includes the REEV backend, rabbitmq, celery workers, postgres.
 
 ```bash session
 cp docker-compose.override.yml-dev docker-compose.override.yml
@@ -295,6 +297,9 @@ The next step step is to create the configuration files in `.dev/config`.
 ```bash session
 mkdir -p .dev/config/nginx
 cp utils/nginx/nginx.conf .dev/config/nginx
+
+mkdir -p .dev/config/pgadmin
+cp utils/pgadmin/servers.json .dev/config/pgadmin
 ```
 
 ### Startup and Check
@@ -347,9 +352,9 @@ Annonars (by the REEV authors) provides variant annotation from public databases
 
 We use postgres for the database backend of REEV.
 
-### Redis
+### Rabbitmq
 
-The Redis database is used for key-value store, e.g., for caching and the queues in the REEV server.
+We use rabbitmq for message queues.
 
 ## Developer Info
 

docker-compose.override.yml-dev

@@ -13,6 +13,7 @@
 #   - `3004` -- nginx
 #   - `3020` -- postgres
 #   - `3030` -- redis
+#   - `3040` -- flower
 
 services:
   # map annonars to port 3001
@@ -45,7 +46,12 @@ services:
     ports:
       - "3020:5432"
 
-  # map redis port to 6379
-  redis:
+  # map rabbitmq port to 3030
+  rabbitmq:
     ports:
       - "3030:6379"
+
+  # map flower port to 3040
+  flower:
+    ports:
+      - "3040:5555"

docker-compose.yml

@@ -6,6 +6,8 @@ version: "3.9"
 
 # Default service definition for all (incl. postgres/redis/...)
 x-service-default: &service_default
+  env_file:
+    - .env
   networks:
     - reev
   restart: unless-stopped
@@ -79,6 +81,7 @@ services:
   # -- REEV ------------------------------------------------------------------
   #
   # REEV web server
+
   reev:
     <<: *service_reev_default
     container_name: reev
@@ -89,7 +92,7 @@ services:
       - viguno
       - nginx
       - postgres
-      - redis
+      - rabbitmq
     image: "${image_base:-ghcr.io/bihealth}/${image_reev_name:-reev}:\
       ${image_reev_version:-main}"
     labels:
@@ -146,38 +149,76 @@ services:
     container_name: postgres
     hostname: postgres
     image: ${image_postgres_name:-postgres}:${image_postgres_version:-12}
-    environment:
-      POSTGRES_USER: reev
-      POSTGRES_PASSWORD_FILE: /run/secrets/db-password
-      POSTGRES_DB: reev
     secrets:
       - db-password
     volumes:
       - type: bind
         source: ${volumes_basedir:-./.dev/volumes}/postgres/data
         target: /var/lib/postgresql/data
 
-  # -- Redis -----------------------------------------------------------------
+  # -- rabbitmq --------------------------------------------------------------
   #
   # We use the default configuration, but mount a volume for the data for
   # persistent storage.
 
-  redis:
+  rabbitmq:
     <<: *service_default
-    container_name: redis
-    hostname: redis
-    image: ${image_redis_name:-redis}:${image_redis_version:-6}
+    container_name: rabbitmq
+    hostname: rabbitmq
+    image: ${image_rabbitmq_name:-rabbitmq}:${image_rabbitmq_version:-3}
     volumes:
       - type: bind
-        source: ${volumes_basedir:-./.dev/volumes}/redis/data
-        target: /data
+        source: ${volumes_basedir:-./.dev/volumes}/rabbitmq/data
+        target: /var/lib/rabbitmq
+
+  # -- pgAdmin ----------------------------------------------------------------
+  #
+  # Useful for interactive database administration.
+
+  pgadmin:
+    <<: *service_default
+    container_name: pgadmin
+    hostname: pgadmin
+    image: "${image_pgadmin_name:-dpage/pgadmin4}:\
+      ${image_pgadmin_version:-latest}"
+    secrets:
+      - db-password
+      - pgadmin-password
+    depends_on:
+      - postgres
+    volumes:
+      - type: bind
+        source: ${volumes_basedir:-./.dev/volumes}/pgadmin/data
+        target: /private/var/lib/pgadmin
+      - type: bind
+        source: ${config_basedir:-./.dev/config}/pgadmin/servers.json
+        target: /pgadmin4/servers.json
+
+  # -- flower -----------------------------------------------------------------
+  #
+  # Real-time celery monitoring.
+
+  flower:
+    <<: *service_default
+    container_name: flower
+    hostname: flower
+    image: ${image_flower_name:-mher/flower}:${image_flower_version:-latest}
+    depends_on:
+      - rabbitmq
+    command:
+      - "celery"
+      - "--broker=amqp://guest@rabbitmq:5672//"
+      - "flower"
 
 # == Secrets ================================================================
 
 secrets:
   # The PostgreSQL database password.
   db-password:
     file: ${secrets_basedir:-./.dev/secrets}/db-password
+  # The pgAdmin admin password.
+  pgadmin-password:
+    file: ${secrets_basedir:-./.dev/secrets}/pgadmin-password
 
 # == Networks ================================================================
 

env.tpl

@@ -1,5 +1,34 @@
 # Template for an .env file.
 
+# -- Stack Configuration -----------------------------------------------------
+
+# Backend
+BACKEND_CORS_ORIGINS=
+SECRET_KEY=SECURITY-ALERT-REPLACE-THIS-KEY
+FIRST_SUPERUSER=admin
+FIRST_SUPERUSER_PASSWORD=SECURITY-ALERT-REPLACE-THIS
+# SMTP_TLS=True
+# SMTP_PORT=
+# SMTP_HOST=
+# SMTP_USER=
+# SMTP_PASSWORD=
+# EMAILS_FROM_EMAIL=
+
+# Postgres
+POSTGRES_SERVER=postgres
+POSTGRES_USER=reev
+POSTGRES_PASSWORD_FILE=/run/secrets/db-password
+POSTGRES_DB=reev
+
+# pgAdmin
+PGADMIN_LISTEN_PORT=80
+[email protected]
+PGADMIN_DEFAULT_PASSWORD_FILE=/run/secrets/pgadmin-password
+PGADMIN_DISABLE_POSTFIX=1
+
+# Flower
+FLOWER_BASIC_AUTH=admin:flower-password
+
 # -- Docker Images -----------------------------------------------------------
 
 # Name of the registry server and org to use for our images.
@@ -35,11 +64,23 @@
 # Version of the postgres image to use.
 # image_postgres_version=12
 
-# Name of the redis image to use.
-# image_redis_name=redis
+# Name of the rabbitmq image to use.
+# image_rabbitmq_name=rabbitmq
+
+# Version of the rabbitmq image to use.
+# image_rabbitmq_version=3
+
+# Name of the flower image to use.
+# image_flower_name=mher/flower
+
+# Version of the flower image to use.
+# image_flower_version=latest
+
+# Name of the pgadmin image to use.
+# image_pgadmin_name=dpage/pgadmin4
 
-# Version of the redis image to use.
-# image_redis_version=6
+# Version of the pgadmin image to use
+# image_pgadmin_version=latest
 
 # Name of the reev image to use.
 # image_reev_name=reev

utils/pgadmin/servers.json

@@ -0,0 +1,14 @@
+{
+  "Servers": {
+      "1": {
+          "Name": "postgres",
+          "Group": "Servers",
+          "Port": 5432,
+          "Username": "reev",
+          "Host": "postgres",
+          "SSLMode": "prefer",
+          "MaintenanceDB": "postgres",
+          "PasswordExecCommand": "cat /run/secrets/db-password"
+      }
+  }
+}