From c823aafd5f26e0bcd7f88e54227a2ecebceb1a2d Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Wed, 27 Sep 2023 09:27:47 +0200 Subject: [PATCH 1/5] feat: switching to rabbitmq, adding flower and pgadmin --- .env.ci | 29 +++++++++++++++ README.md | 15 +++++--- docker-compose.override.yml-dev | 10 ++++-- docker-compose.yml | 64 ++++++++++++++++++++++++++------- env.tpl | 49 ++++++++++++++++++++++--- utils/pgadmin/servers.json | 14 ++++++++ 6 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 utils/pgadmin/servers.json diff --git a/.env.ci b/.env.ci index d17b3fa..a54eac0 100644 --- a/.env.ci +++ b/.env.ci @@ -1,5 +1,34 @@ # Docker Compose environment file used in CI. +# -- Stack Configuration ----------------------------------------------------- + +# Backend +BACKEND_CORS_ORIGINS= +SECRET_KEY=ONLY-CI-secret-key +FIRST_SUPERUSER=ONLY-CI-admin +FIRST_SUPERUSER_PASSWORD=ONLY-CI-superuser-password +# SMTP_TLS=True +# SMTP_PORT= +# SMTP_HOST= +# SMTP_USER= +# SMTP_PASSWORD= +# EMAILS_FROM_EMAIL= + +# Postgres +POSTGRES_SERVER=postgres +POSTGRES_USER=reev +POSTGRES_PASSWORD_FILE=/run/secrets/db-password +POSTGRES_DB=reev + +# pgAdmin +PGADMIN_LISTEN_PORT=80 +PGADMIN_DEFAULT_EMAIL=admin@example.com +PGADMIN_DEFAULT_PASSWORD_FILE=/run/secrets/pgadmin-password +PGADMIN_DISABLE_POSTFIX=1 + +# Flower +FLOWER_BASIC_AUTH=admin:flower-password + # -- Docker Images ----------------------------------------------------------- # Name of the registry server and org to use for our images. diff --git a/README.md b/README.md index 8b713ab..ef03816 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,10 @@ In a production deployment, these directories should live outside of the checkou Now, we create the directories for data storage. ```bash session -mkdir -p .dev/volumes/reev-static/data +mkdir -p .dev/volumes/pgadmin/data mkdir -p .dev/volumes/postgres/data -mkdir -p .dev/volumes/redis/data +mkdir -p .dev/volumes/rabbitmq/data +mkdir -p .dev/volumes/reev-static/data ``` Next, we setup some "secrets" for the passwords. @@ -50,6 +51,7 @@ Next, we setup some "secrets" for the passwords. ```bash session mkdir -p .dev/secrets echo db-password >.dev/secrets/db-password +echo pgadmin-password >.dev/secrets/pgadmin-password ``` We now copy the `env.tpl` file to the default location for the environment `.env`. @@ -60,7 +62,7 @@ cp env.tpl .env Next, create a `docker-compose.override.yml` with the contents of the file `docker-compose.override.yml-dev`. This will disable everything that we assume is running on your host when you are developing. -This includes the REEV backend, redis, celery workers, postgres. +This includes the REEV backend, rabbitmq, celery workers, postgres. ```bash session cp docker-compose.override.yml-dev docker-compose.override.yml @@ -295,6 +297,9 @@ The next step step is to create the configuration files in `.dev/config`. ```bash session mkdir -p .dev/config/nginx cp utils/nginx/nginx.conf .dev/config/nginx + +mkdir -p .dev/config/pgadmin +cp utils/pgadmin/servers.json .dev/config/pgadmin ``` ### Startup and Check @@ -347,9 +352,9 @@ Annonars (by the REEV authors) provides variant annotation from public databases We use postgres for the database backend of REEV. -### Redis +### Rabbitmq -The Redis database is used for key-value store, e.g., for caching and the queues in the REEV server. +We use rabbitmq for message queues. ## Developer Info diff --git a/docker-compose.override.yml-dev b/docker-compose.override.yml-dev index c9d1cc0..8fe71eb 100644 --- a/docker-compose.override.yml-dev +++ b/docker-compose.override.yml-dev @@ -13,6 +13,7 @@ # - `3004` -- nginx # - `3020` -- postgres # - `3030` -- redis +# - `3040` -- flower services: # map annonars to port 3001 @@ -45,7 +46,12 @@ services: ports: - "3020:5432" - # map redis port to 6379 - redis: + # map rabbitmq port to 3030 + rabbitmq: ports: - "3030:6379" + + # map flower port to 3040 + flower: + ports: + - "3040:5555" diff --git a/docker-compose.yml b/docker-compose.yml index 725fe90..da16cee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ version: "3.9" # Default service definition for all (incl. postgres/redis/...) x-service-default: &service_default + env_file: + - .env networks: - reev restart: unless-stopped @@ -79,6 +81,7 @@ services: # -- REEV ------------------------------------------------------------------ # # REEV web server + reev: <<: *service_reev_default container_name: reev @@ -89,7 +92,7 @@ services: - viguno - nginx - postgres - - redis + - rabbitmq image: "${image_base:-ghcr.io/bihealth}/${image_reev_name:-reev}:\ ${image_reev_version:-main}" labels: @@ -146,10 +149,6 @@ services: container_name: postgres hostname: postgres image: ${image_postgres_name:-postgres}:${image_postgres_version:-12} - environment: - POSTGRES_USER: reev - POSTGRES_PASSWORD_FILE: /run/secrets/db-password - POSTGRES_DB: reev secrets: - db-password volumes: @@ -157,20 +156,58 @@ services: source: ${volumes_basedir:-./.dev/volumes}/postgres/data target: /var/lib/postgresql/data - # -- Redis ----------------------------------------------------------------- + # -- rabbitmq -------------------------------------------------------------- # # We use the default configuration, but mount a volume for the data for # persistent storage. - redis: + rabbitmq: <<: *service_default - container_name: redis - hostname: redis - image: ${image_redis_name:-redis}:${image_redis_version:-6} + container_name: rabbitmq + hostname: rabbitmq + image: ${image_rabbitmq_name:-rabbitmq}:${image_rabbitmq_version:-3} volumes: - type: bind - source: ${volumes_basedir:-./.dev/volumes}/redis/data - target: /data + source: ${volumes_basedir:-./.dev/volumes}/rabbitmq/data + target: /var/lib/rabbitmq + + # -- pgAdmin ---------------------------------------------------------------- + # + # Useful for interactive database administration. + + pgadmin: + <<: *service_default + container_name: pgadmin + hostname: pgadmin + image: ${image_pgadmin_name:-dpage/pgadmin4}:${image_pgadmin_version:-latest} + secrets: + - db-password + - pgadmin-password + depends_on: + - postgres + volumes: + - type: bind + source: ${volumes_basedir:-./.dev/volumes}/pgadmin/data + target: /private/var/lib/pgadmin + - type: bind + source: ${config_basedir:-./.dev/config}/pgadmin/servers.json + target: /pgadmin4/servers.json + + # -- flower ----------------------------------------------------------------- + # + # Real-time celery monitoring. + + flower: + <<: *service_default + container_name: flower + hostname: flower + image: ${image_flower_name:-mher/flower}:${image_flower_version:-latest} + depends_on: + - rabbitmq + command: + - "celery" + - "--broker=amqp://guest@rabbitmq:5672//" + - "flower" # == Secrets ================================================================ @@ -178,6 +215,9 @@ secrets: # The PostgreSQL database password. db-password: file: ${secrets_basedir:-./.dev/secrets}/db-password + # The pgAdmin admin password. + pgadmin-password: + file: ${secrets_basedir:-./.dev/secrets}/pgadmin-password # == Networks ================================================================ diff --git a/env.tpl b/env.tpl index 3f0ecaa..2229f3d 100644 --- a/env.tpl +++ b/env.tpl @@ -1,5 +1,34 @@ # Template for an .env file. +# -- Stack Configuration ----------------------------------------------------- + +# Backend +BACKEND_CORS_ORIGINS= +SECRET_KEY=SECURITY-ALERT-REPLACE-THIS-KEY +FIRST_SUPERUSER=admin +FIRST_SUPERUSER_PASSWORD=SECURITY-ALERT-REPLACE-THIS +# SMTP_TLS=True +# SMTP_PORT= +# SMTP_HOST= +# SMTP_USER= +# SMTP_PASSWORD= +# EMAILS_FROM_EMAIL= + +# Postgres +POSTGRES_SERVER=postgres +POSTGRES_USER=reev +POSTGRES_PASSWORD_FILE=/run/secrets/db-password +POSTGRES_DB=reev + +# pgAdmin +PGADMIN_LISTEN_PORT=80 +PGADMIN_DEFAULT_EMAIL=admin@example.com +PGADMIN_DEFAULT_PASSWORD_FILE=/run/secrets/pgadmin-password +PGADMIN_DISABLE_POSTFIX=1 + +# Flower +FLOWER_BASIC_AUTH=admin:flower-password + # -- Docker Images ----------------------------------------------------------- # Name of the registry server and org to use for our images. @@ -35,11 +64,23 @@ # Version of the postgres image to use. # image_postgres_version=12 -# Name of the redis image to use. -# image_redis_name=redis +# Name of the rabbitmq image to use. +# image_rabbitmq_name=rabbitmq + +# Version of the rabbitmq image to use. +# image_rabbitmq_version=3 + +# Name of the flower image to use. +# image_flower_name=mher/flower + +# Version of the flower image to use. +# image_flower_version=latest + +# Name of the pgadmin image to use. +# image_pgadmin_name=dpage/pgadmin4 -# Version of the redis image to use. -# image_redis_version=6 +# Version of the pgadmin image to use +# image_pgadmin_version=latest # Name of the reev image to use. # image_reev_name=reev diff --git a/utils/pgadmin/servers.json b/utils/pgadmin/servers.json new file mode 100644 index 0000000..498de9b --- /dev/null +++ b/utils/pgadmin/servers.json @@ -0,0 +1,14 @@ +{ + "Servers": { + "1": { + "Name": "postgres", + "Group": "Servers", + "Port": 5432, + "Username": "reev", + "Host": "postgres", + "SSLMode": "prefer", + "MaintenanceDB": "postgres", + "PasswordExecCommand": "cat /run/secrets/db-password" + } + } +} From f288c8c34710fbb6036992cd9bd51ddcfa7d755b Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Wed, 27 Sep 2023 09:33:50 +0200 Subject: [PATCH 2/5] lint --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index da16cee..9929968 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,8 @@ services: <<: *service_default container_name: pgadmin hostname: pgadmin - image: ${image_pgadmin_name:-dpage/pgadmin4}:${image_pgadmin_version:-latest} + image: "${image_pgadmin_name:-dpage/pgadmin4}:\ + ${image_pgadmin_version:-latest}" secrets: - db-password - pgadmin-password From a94928db89dbe591cf7f6128ba10c0c8a628dad2 Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Wed, 27 Sep 2023 09:37:41 +0200 Subject: [PATCH 3/5] fix CI --- .ci/volumes/reev-static/data/.gitkeep | 0 .github/workflows/ci.yml | 13 +++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) delete mode 100644 .ci/volumes/reev-static/data/.gitkeep diff --git a/.ci/volumes/reev-static/data/.gitkeep b/.ci/volumes/reev-static/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 365fdcc..37efdcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,19 @@ jobs: - name: Prepare empty volumes run: | + mkdir -p .ci/volumes/minio/data + mkdir -p .ci/volumes/pgadmin/data mkdir -p .ci/volumes/postgres/data + mkdir -p .ci/volumes/rabbitmq/data mkdir -p .ci/volumes/redis/data - mkdir -p .ci/volumes/minio/data - mkdir -p .ci/volumes/varfish-static/data + mkdir -p .ci/volumes/reev-static/data + + - name: Prepare configuration + run: | + mkdir -p .ci/config/nginx + cp utils/nginx/nginx.conf .ci/config/nginx + mkdir -p .ci/config/pgadmin + cp utils/pgadmin/servers.json .ci/config/pgadmin - name: Bring up and shut down services uses: isbang/compose-action@v1.4.1 From 6266a7dc2f4377cbf546e5c01f2b404aeba3a257 Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Wed, 27 Sep 2023 09:39:01 +0200 Subject: [PATCH 4/5] wip --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37efdcb..01e3b85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: - name: Prepare configuration run: | + cp .env.ci .env + mkdir -p .ci/config/nginx cp utils/nginx/nginx.conf .ci/config/nginx mkdir -p .ci/config/pgadmin @@ -49,6 +51,6 @@ jobs: with: compose-file: docker-compose.yml compose-flags: | - --env-file .env.ci + --env-file .env env: DB_PASSWORD: "password" From dd2314fd7c259d14c65c40309fccde73c6b4ab79 Mon Sep 17 00:00:00 2001 From: Manuel Holtgrewe Date: Wed, 27 Sep 2023 09:44:45 +0200 Subject: [PATCH 5/5] wip --- .ci/config/nginx/nginx.conf | 35 ----------------------------------- .ci/secrets/db-password | 1 - .github/workflows/ci.yml | 6 ++++++ 3 files changed, 6 insertions(+), 36 deletions(-) delete mode 100644 .ci/config/nginx/nginx.conf delete mode 100644 .ci/secrets/db-password diff --git a/.ci/config/nginx/nginx.conf b/.ci/config/nginx/nginx.conf deleted file mode 100644 index 5c13ff0..0000000 --- a/.ci/config/nginx/nginx.conf +++ /dev/null @@ -1,35 +0,0 @@ -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - keepalive_timeout 65; - - gzip on; - - # The full static data directory is mounted into the container but we only - # serve the "nginx" sub directory (with indices in the default - # configuration). - server { - location / { - root /data/nginx; - autoindex on; - } - } -} diff --git a/.ci/secrets/db-password b/.ci/secrets/db-password deleted file mode 100644 index f3097ab..0000000 --- a/.ci/secrets/db-password +++ /dev/null @@ -1 +0,0 @@ -password diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01e3b85..6796ce3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + - name: Prepare secrets + run: | + mkdir -p .ci/secrets + echo db-password > .ci/secrets/db-password + echo pgadmin-password > .ci/secrets/pgadmin-password + - name: Prepare empty volumes run: | mkdir -p .ci/volumes/minio/data