Add prometheus and grafana

.github/workflows/deploy-production.yaml

@@ -33,8 +33,12 @@ jobs:
           echo "SWAGGER_ROOT_PATH=${PRODUCTION__SWAGGER_ROOT_PATH}" >> $GITHUB_ENV
           echo "KAFKA_UI_ROOT_PATH=${PRODUCTION__KAFKA_UI_ROOT_PATH}" >> $GITHUB_ENV
           echo "KAFKA_UI_PASSWORD=${PRODUCTION__KAFKA_UI_PASSWORD}" >> $GITHUB_ENV
+          echo "GRAFANA_ROOT_URL=${PRODUCTION__GRAFANA_ROOT_URL}" >> $GITHUB_ENV
+          echo "GRAFANA_ROOT_PATH=${PRODUCTION__GRAFANA_ROOT_PATH}" >> $GITHUB_ENV
           echo "BROKER_DIR=${PRODUCTION__BROKER_DIR}" >> $GITHUB_ENV
           echo "DATABASE_DIR=${PRODUCTION__DATABASE_DIR}" >> $GITHUB_ENV
+          echo "GRAFANA_DIR=${PRODUCTION__GRAFANA_DIR}" >> $GITHUB_ENV
+          echo "PROMETHEUS_DIR=${PRODUCTION__PROMETHEUS_DIR}" >> $GITHUB_ENV
           echo "REDIS_DIR=${PRODUCTION__REDIS_DIR}" >> $GITHUB_ENV
           echo "PROJECTS_MEDIA_DIR_PATH=${PRODUCTION__PROJECTS_MEDIA_DIR_PATH}" >> $GITHUB_ENV
           echo "USERS_MEDIA_DIR_PATH=${PRODUCTION__USERS_MEDIA_DIR_PATH}" >> $GITHUB_ENV

.github/workflows/deploy-stage.yaml

@@ -33,8 +33,12 @@ jobs:
           echo "SWAGGER_ROOT_PATH=${STAGE__SWAGGER_ROOT_PATH}" >> $GITHUB_ENV
           echo "KAFKA_UI_ROOT_PATH=${STAGE__KAFKA_UI_ROOT_PATH}" >> $GITHUB_ENV
           echo "KAFKA_UI_PASSWORD=${STAGE__KAFKA_UI_PASSWORD}" >> $GITHUB_ENV
+          echo "GRAFANA_ROOT_URL=${STAGE__GRAFANA_ROOT_URL}" >> $GITHUB_ENV
+          echo "GRAFANA_ROOT_PATH=${STAGE__GRAFANA_ROOT_PATH}" >> $GITHUB_ENV
           echo "BROKER_DIR=${STAGE__BROKER_DIR}" >> $GITHUB_ENV
           echo "DATABASE_DIR=${STAGE__DATABASE_DIR}" >> $GITHUB_ENV
+          echo "GRAFANA_DIR=${STAGE__GRAFANA_DIR}" >> $GITHUB_ENV
+          echo "PROMETHEUS_DIR=${STAGE__PROMETHEUS_DIR}" >> $GITHUB_ENV
           echo "REDIS_DIR=${STAGE__REDIS_DIR}" >> $GITHUB_ENV
           echo "PROJECTS_MEDIA_DIR_PATH=${STAGE__PROJECTS_MEDIA_DIR_PATH}" >> $GITHUB_ENV
           echo "USERS_MEDIA_DIR_PATH=${STAGE__USERS_MEDIA_DIR_PATH}" >> $GITHUB_ENV

.gitignore

@@ -9,7 +9,9 @@
 __pycache__
 broker_data
 database_data
+grafana_data
 redis_data
 projects_data
+prometheus_data
 pyrightconfig.json
 users_data

Makefile

@@ -23,9 +23,6 @@ build:
 down:
 	docker stack rm sapphire || true
 
-up:
-	docker stack deploy -c docker-compose.yaml sapphire
-
 clean:
 	docker rmi sapphire --force
 
@@ -36,4 +33,7 @@ else
 	sleep 15
 endif
 
-restart: down clean build sleep up
+restart:
+	docker stack deploy -c docker-compose.yaml sapphire
+
+up: clean build down sleep restart

README.md

@@ -118,6 +118,8 @@ mkdir -p database_data
 mkdir -p broker_data/kafka/data
 mkdir -p broker_data/zookeeper/data
 mkdir -p broker_data/zookeeper/log
+mkdir -p prometheus_data
+mkdir -p grafana_data
 mkdir -p projects_data/media
 mkdir -p users_data/media
 ```

docker-compose.yaml

@@ -1,4 +1,4 @@
-version: "3"
+version: "3.8"
 services:
   database:
     image: postgres:16.0-bookworm
@@ -68,43 +68,89 @@ services:
       labels:
         - "traefik.enable=true"
         - "traefik.http.routers.kafka.rule=PathPrefix(`${KAFKA_UI_ROOT_PATH:-/kafka}`)"
-        - "traefik.http.routers.kafka.entrypoints=web" 
+        - "traefik.http.routers.kafka.entrypoints=web"
         - "traefik.http.routers.kafka.service=kafka"
         - "traefik.http.services.kafka.loadbalancer.server.port=8080"
       replicas: 1
 
+  prometheus-node-exporter:
+    image: prom/node-exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.rootfs=/rootfs"
+      - "--path.sysfs=/host/sys"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    deploy:
+      mode: global
+
+  prometheus:
+    image: prom/prometheus
+    user: "0:0"
+    volumes:
+      - ${PROMETHEUS_DIR:-./prometheus_data/}:/prometheus
+    configs:
+      - source: prometheus
+        target: /etc/prometheus/prometheus.yaml
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yaml"
+      - "--storage.tsdb.path=/prometheus"
+    deploy:
+      replicas: 1
+
+  grafana:
+    image: grafana/grafana
+    user: "0:0"
+    volumes:
+      - ${GRAFANA_DIR:-./grafana_data}:/var/lib/grafana
+    environment:
+      GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000}${GRAFANA_ROOT_PATH:-/grafana}
+    deploy:
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.middlewares.grafana-stripprefix.stripprefix.prefixes=${GRAFANA_ROOT_PATH:-/grafana}"
+        - "traefik.http.routers.grafana.middlewares=grafana-stripprefix"
+        - "traefik.http.routers.grafana.rule=PathPrefix(`${GRAFANA_ROOT_PATH:-/grafana}`)"
+        - "traefik.http.routers.grafana.service=grafana"
+        - "traefik.http.routers.grafana.entrypoints=web"
+        - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+      replicas: 1
+
   sapphire:
     image: ${SAPPHIRE_IMAGE:-sapphire}
     environment:
       DATABASE__DSN: ${DATABASE__DSN:-postgresql+asyncpg://sapphire:P%40ssw0rd@database:5432/sapphire}
       MESSENGER__API__PORT: "8000"
-      MESSENGER__API__ROOT_URL: ${MESSENGER__API__ROOT_URL:-http://localhost:3000/messenger}
+      MESSENGER__API__ROOT_URL: ${MESSENGER__API__ROOT_URL:-http://localhost:3000}
       MESSENGER__API__ROOT_PATH: ${MESSENGER__API__ROOT_PATH:-/messenger}
       MESSENGER__API__ALLOWED_ORIGINS: ${MESSENGER__API__ALLOWED_ORIGINS:-["http://localhost:3000"]}
       MESSENGER__BROKER__SERVERS: '["kafka:9091"]'
       MESSENGER__BROKER__TOPICS: '["chats"]'
       MESSENGER__DATABASE__DSN: ${MESSENGER__DATABASE__DSN:-postgresql+asyncpg://sapphire:P%40ssw0rd@database:5432/sapphire}
       NOTIFICATIONS__API__PORT: "8010"
-      NOTIFICATIONS__API__ROOT_URL: ${NOTIFICATIONS__API__ROOT_URL:-http://localhost:3000/notifications}
+      NOTIFICATIONS__API__ROOT_URL: ${NOTIFICATIONS__API__ROOT_URL:-http://localhost:3000}
       NOTIFICATIONS__API__ROOT_PATH: ${NOTIFICATIONS__API__ROOT_PATH:-/notifications}
       NOTIFICATIONS__API__ALLOWED_ORIGINS: ${NOTIFICATIONS__API__ALLOWED_ORIGINS:-["http://localhost:3000"]}
       NOTIFICATIONS__BROKER__SERVERS: '["kafka:9091"]'
       NOTIFICATIONS__BROKER__TOPICS: '["notifications"]'
       NOTIFICATIONS__DATABASE__DSN: ${NOTIFICATIONS__DATABASE__DSN:-postgresql+asyncpg://sapphire:P%40ssw0rd@database:5432/sapphire}
       PROJECTS__API__PORT: "8020"
-      PROJECTS__API__ROOT_URL: ${PROJECTS__API__ROOT_URL:-http://localhost:3000/projects}
+      PROJECTS__API__ROOT_URL: ${PROJECTS__API__ROOT_URL:-http://localhost:3000}
       PROJECTS__API__ROOT_PATH: ${PROJECTS__API__ROOT_PATH:-/projects}
       PROJECTS__API__ALLOWED_ORIGINS: ${PROJECTS__API__ALLOWED_ORIGINS:-["http://localhost:3000"]}
       PROJECTS__API__MEDIA_DIR_PATH: "/projects/media"
       PROJECTS__BROKER__SERVERS: '["kafka:9091"]'
       PROJECTS__DATABASE__DSN: ${PROJECTS__DATABASE__DSN:-postgresql+asyncpg://sapphire:P%40ssw0rd@database:5432/sapphire}
       STORAGE__API__PORT: "8030"
-      STORAGE__API__ROOT_URL: ${STORAGE__API__ROOT_URL:-http://localhost:3000/storage}
+      STORAGE__API__ROOT_URL: ${STORAGE__API__ROOT_URL:-http://localhost:3000}
       STORAGE__API__ROOT_PATH: ${STORAGE__API__ROOT_PATH:-/storage}
       STORAGE__API__ALLOWED_ORIGINS: ${STORAGE__API__ALLOWED_ORIGINS:-["http://localhost:3000"]}
       STORAGE__DATABASE__DSN: ${STORAGE__DATABASE__DSN:-postgresql+asyncpg://sapphire:P%40ssw0rd@database:5432/sapphire}
       USERS__API__PORT: "8040"
-      USERS__API__ROOT_URL: ${USERS__API__ROOT_URL:-http://localhost:3000/users}
+      USERS__API__ROOT_URL: ${USERS__API__ROOT_URL:-http://localhost:3000}
       USERS__API__ROOT_PATH: ${USERS__API__ROOT_PATH:-/users}
       USERS__API__ALLOWED_ORIGINS: ${USERS__API__ALLOWED_ORIGINS:-["http://localhost:3000"]}
       USERS__API__OAUTH2_HABR_CALLBACK_URL: ${USERS__API__OAUTH2_HABR_CALLBACK_URL:-http://localhost:3000/users/api/rest/auth/oauth2/habr/callback}
@@ -215,23 +261,23 @@ services:
       URLS: >
         [
           {
-            "url": "${USERS__API__ROOT_URL:-http://localhost:3000/users}${USERS__API__ROOT_PATH}/openapi.json",
+            "url": "${USERS__API__ROOT_URL:-http://localhost:3000}${USERS__API__ROOT_PATH:-/users}/openapi.json",
             "name": "Users",
           },
           {
-            "url": "${STORAGE__API__ROOT_URL:-http://localhost:3000/storage}${STORAGE__API__ROOT_PATH}/openapi.json",
+            "url": "${STORAGE__API__ROOT_URL:-http://localhost:3000}${STORAGE__API__ROOT_PATH:-/storage}/openapi.json",
             "name": "Storage",
           },
           {
-            "url": "${PROJECTS__API__ROOT_URL:-http://localhost:3000/projects}${PROJECTS__API__ROOT_PATH}/openapi.json",
+            "url": "${PROJECTS__API__ROOT_URL:-http://localhost:3000}${PROJECTS__API__ROOT_PATH:-/projects}/openapi.json",
             "name": "Projects",
           },
           {
-            "url": "${NOTIFICATIONS__API__ROOT_URL:-http://localhost:3000/notifications}${NOTIFICATIONS__API__ROOT_PATH}/openapi.json",
+            "url": "${NOTIFICATIONS__API__ROOT_URL:-http://localhost:3000}${NOTIFICATIONS__API__ROOT_PATH:-/notifications}/openapi.json",
             "name": "Notifications",
           },
           {
-            "url": "${MESSENGER__API__ROOT_URL:-http://localhost:3000/messenger}${MESSENGER__API__ROOT_PATH}/openapi.json",
+            "url": "${MESSENGER__API__ROOT_URL:-http://localhost:3000}${MESSENGER__API__ROOT_PATH:-/messenger}/openapi.json",
             "name": "Messenger",
           },
         ]
@@ -267,6 +313,9 @@ services:
         order: start-first
       replicas: 1
 
+configs:
+  prometheus:
+    file: "prometheus.yaml"
 secrets:
   oauth2_habr_client_id:
     external: true

prometheus.yaml

@@ -0,0 +1,11 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 30s
+
+scrape_configs:
+- job_name: "node-exporter"
+  metrics_path: /metrics
+  static_configs:
+    - targets:
+      - prometheus-node-exporter:9100

sapphire/common/database/service.py

@@ -36,9 +36,13 @@ class BaseDatabaseService(ServiceMixin):
         FixtureFormatEnum.JSON: json.load,
     }
 
-    def __init__(self, dsn: str):
+    def __init__(self, dsn: str, pool_size: int = 5, pool_recycle: int = 60):
         self._dsn = dsn
-        self._engine = create_async_engine(self._dsn, pool_recycle=60)
+        engine_parameters = {"url": self._dsn, "pool_recycle": pool_recycle}
+        if not dsn.startswith("sqlite"):
+            engine_parameters["pool_size"] = pool_size
+
+        self._engine = create_async_engine(**engine_parameters)
         self._sessionmaker = async_sessionmaker(self._engine, expire_on_commit=False)
 
     def get_migrations_directory_path(self) -> pathlib.Path:

sapphire/common/database/settings.py

@@ -1,5 +1,7 @@
-from pydantic import AnyUrl, BaseModel
+from pydantic import AnyUrl, BaseModel, PositiveInt
 
 
 class BaseDatabaseSettings(BaseModel):
     dsn: AnyUrl
+    pool_size: PositiveInt = 5
+    pool_recycle: PositiveInt = 60  # in seconds: 1 minute

sapphire/database/service.py

@@ -19,4 +19,8 @@ def get_models(self) -> Iterable[Type[Base]]:
 
 
 def get_service(settings: Settings) -> Service:
-    return Service(dsn=str(settings.dsn))
+    return Service(
+        dsn=str(settings.dsn),
+        pool_size=settings.pool_size,
+        pool_recycle=settings.pool_recycle,
+    )

sapphire/database/settings.py

@@ -1,5 +1,7 @@
-from pydantic import AnyUrl, BaseModel
+from pydantic import AnyUrl
 
+from sapphire.common.database.settings import BaseDatabaseSettings
 
-class Settings(BaseModel):
+
+class Settings(BaseDatabaseSettings):
     dsn: AnyUrl = AnyUrl("sqlite+aiosqlite:///db.sqlite3")

sapphire/messenger/database/service.py

@@ -132,4 +132,8 @@ async def get_chat_message(
 
 
 def get_service(settings: Settings) -> Service:
-    return Service(dsn=str(settings.dsn))
+    return Service(
+        dsn=str(settings.dsn),
+        pool_size=settings.pool_size,
+        pool_recycle=settings.pool_recycle,
+    )

sapphire/notifications/database/service.py

@@ -91,4 +91,8 @@ async def update_notification(
 
 
 def get_service(settings: Settings) -> Service:
-    return Service(dsn=str(settings.dsn))
+    return Service(
+        dsn=str(settings.dsn),
+        pool_size=settings.pool_size,
+        pool_recycle=settings.pool_recycle,
+    )

sapphire/projects/database/service.py

@@ -767,4 +767,8 @@ async def get_review(
 
 
 def get_service(settings: Settings) -> Service:
-    return Service(dsn=str(settings.dsn))
+    return Service(
+        dsn=str(settings.dsn),
+        pool_size=settings.pool_size,
+        pool_recycle=settings.pool_recycle,
+    )

sapphire/storage/database/service.py

@@ -247,4 +247,8 @@ async def get_skill(self, session: AsyncSession, habr_id: int) -> Skill | None:
 
 
 def get_service(settings: Settings) -> Service:
-    return Service(dsn=str(settings.dsn))
+    return Service(
+        dsn=str(settings.dsn),
+        pool_size=settings.pool_size,
+        pool_recycle=settings.pool_recycle,
+    )

sapphire/users/database/service.py

@@ -124,4 +124,8 @@ async def update_user_skills(self,
 
 
 def get_service(settings: Settings) -> Service:
-    return Service(dsn=str(settings.dsn))
+    return Service(
+        dsn=str(settings.dsn),
+        pool_size=settings.pool_size,
+        pool_recycle=settings.pool_recycle,
+    )