Merge pull request #256 from bento-platform/feat/logs

feat: monitoring system
bento-platform · Aug 14, 2024 · 4f68763 · 4f68763
2 parents 4af80ff + d7eefb2
commit 4f68763
Show file tree

Hide file tree

Showing 15 changed files with 264 additions and 0 deletions.
diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml
@@ -65,6 +65,11 @@ services:
           - ${BENTOV2_DOMAIN}
           - ${BENTOV2_PORTAL_DOMAIN}
           - ${BENTOV2_AUTH_DOMAIN}
+      monitoring-net:
+          aliases:
+          - ${BENTOV2_DOMAIN}
+          - ${BENTOV2_PORTAL_DOMAIN}
+          - ${BENTOV2_AUTH_DOMAIN}
       public-net:
         aliases:
           - ${BENTOV2_DOMAIN}
@@ -232,3 +237,14 @@ services:
   cbioportal:
     ports:
       - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}"
+
+  grafana:
+    ports:
+      - "3000:3000"
+    environment:
+    # Workaround for self signed certificates in dev
+      - GF_AUTH_GENERIC_OAUTH_TLS_SKIP_VERIFY_INSECURE=true
+
+  loki:
+    ports:
+      - "3100:3100"
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -13,6 +13,7 @@ include:
       - lib/event-relay/docker-compose.event-relay.yaml
       - lib/gohan/docker-compose.gohan.yaml  # Optional feature; controlled by a compose profile
       - lib/katsu/docker-compose.katsu.yaml
+      - lib/logs/docker-compose.logs.yaml
       - lib/notification/docker-compose.notification.yaml
       - lib/public/docker-compose.public.yaml  # Optional feature; controlled by a compose profile
       - lib/redis/docker-compose.redis.yaml

diff --git a/etc/bento.env b/etc/bento.env
@@ -440,3 +440,19 @@ BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.15
 BENTO_CBIOPORTAL_SESSION_DATABASE_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session-db
 BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/session_db
 # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network
+
+# Monitoring
+BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net
+BENTO_PRIVATE_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/grafana
+BENTO_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana
+BENTO_LOKI_IMAGE=grafana/loki
+BENTO_LOKI_IMAGE_VERSION=3.1.1
+BENTO_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki
+BENTO_LOKI_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp
+BENTO_GRAFANA_IMAGE=grafana/grafana
+BENTO_GRAFANA_IMAGE_VERSION=11.1.3
+BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana
+BENTO_GRAFANA_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib
+BENTO_PROMTAIL_IMAGE=grafana/promtail
+BENTO_PROMTAIL_IMAGE_VERSION=2.9.10
+BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail
diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env
@@ -11,6 +11,7 @@ BENTO_GATEWAY_USE_TLS='true'
 BENTO_BEACON_ENABLED='false'  # Set to true if using Beacon!
 BENTO_BEACON_UI_ENABLED='false'
 BENTO_CBIOPORTAL_ENABLED='false'
+BENTO_MONITORING_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 
 #  - Switch to enable French translation in Bento Public
@@ -53,6 +54,10 @@ BENTO_AUTHZ_DB_PASSWORD=  # TODO: SET ME WHEN DEPLOYING!
 #  - WES Client ID/secret; client within BENTOV2_AUTH_REALM
 BENTO_WES_CLIENT_ID=wes
 BENTO_WES_CLIENT_SECRET=  # TODO: SET ME WHEN DEPLOYING!
+
+#  - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM
+BENTO_GRAFANA_CLIENT_ID=grafana
+BENTO_GRAFANA_CLIENT_SECRET=
 # ---------------------------------------------------------------------
 
 BENTO_WEB_CUSTOM_HEADER=

diff --git a/etc/bento_dev.env b/etc/bento_dev.env
@@ -11,6 +11,7 @@ BENTO_GATEWAY_USE_TLS='true'
 BENTO_BEACON_ENABLED='true'
 BENTO_BEACON_UI_ENABLED='true'
 BENTO_CBIOPORTAL_ENABLED='false'
+BENTO_MONITORING_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 
 #  - Switch to enable French translation in Bento Public
@@ -53,6 +54,10 @@ BENTOV2_AUTH_TEST_PASSWORD=
 #  - WES Client ID/secret; client within BENTOV2_AUTH_REALM
 BENTO_WES_CLIENT_ID=wes
 BENTO_WES_CLIENT_SECRET=
+
+#  - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM
+BENTO_GRAFANA_CLIENT_ID=grafana
+BENTO_GRAFANA_CLIENT_SECRET=
 # --------------------------------------------------------------------
 
 # Gohan

diff --git a/etc/default_config.env b/etc/default_config.env
@@ -81,6 +81,9 @@ BENTO_AUTHZ_DB_PASSWORD=
 #  - cBioPortal Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM
 BENTO_CBIOPORTAL_CLIENT_ID=cbioportal
 BENTO_CBIOPORTAL_CLIENT_SECRET=
+#  - Grafana Client ID/secret; secret to be filled by local.env client within BENTOV2_AUTH_REALM
+BENTO_GRAFANA_CLIENT_ID=grafana
+BENTO_GRAFANA_CLIENT_SECRET=
 #  - WES Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM
 BENTO_WES_CLIENT_ID=wes
 BENTO_WES_CLIENT_SECRET=

diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml
@@ -22,6 +22,7 @@ services:
       - BENTO_BEACON_ENABLED
       - BENTO_CBIOPORTAL_ENABLED
       - BENTO_GOHAN_ENABLED
+      - BENTO_MONITORING_ENABLED
 
       - BENTOV2_GATEWAY_CONTAINER_NAME
 
@@ -76,6 +77,7 @@ services:
       - BENTO_BEACON_INTERNAL_PORT
       - BENTO_CBIOPORTAL_CONTAINER_NAME
       - BENTO_CBIOPORTAL_INTERNAL_PORT
+      - BENTO_GRAFANA_CONTAINER_NAME
     networks:
       - aggregation-net
       - auth-net
@@ -87,6 +89,7 @@ services:
       - event-relay-net
       - gohan-api-net
       - katsu-net
+      - monitoring-net
       - notification-net
       - public-net
       - reference-net
@@ -166,6 +169,9 @@ networks:
   katsu-net:
     external: true
     name: ${BENTO_KATSU_NETWORK}
+  monitoring-net:
+    external: true
+    name: ${BENTO_MONITORING_NETWORK}
   notification-net:
     external: true
     name: ${BENTO_NOTIFICATION_NETWORK}

diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl
@@ -0,0 +1,13 @@
+# env: BENTO_MONITORING_ENABLED
+location /api/grafana { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/grafana/; }
+location /api/grafana/ {
+    # Reverse proxy settings
+    include /gateway/conf/proxy.conf;
+    include /gateway/conf/proxy_extra.conf;
+    
+    # Immediate set/re-use means we don't get resolve errors if not up (as opposed to passing as a literal)
+    set $upstream_grafana http://${BENTO_GRAFANA_CONTAINER_NAME}:3000;
+    
+    proxy_pass $upstream_grafana;
+    error_log /var/log/bentov2_grafana_errors.log;
+}
diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml
@@ -0,0 +1,93 @@
+services: 
+  grafana:
+    image: ${BENTO_GRAFANA_IMAGE}:${BENTO_GRAFANA_IMAGE_VERSION}
+    container_name: ${BENTO_GRAFANA_CONTAINER_NAME}
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+      - GF_SERVER_ROOT_URL=${BENTO_PRIVATE_GRAFANA_URL}
+      - GF_SERVER_SERVE_FROM_SUB_PATH=true
+      - GF_SECURITY_COOKIE_SAMESITE=none
+      - GF_SECURITY_ALLOW_EMBEDDING=true
+      - GF_LOG_LEVEL=debug
+      - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true
+      - GF_AUTH_GENERIC_OAUTH_ENABLED=true
+      - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth
+      - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true
+      - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID}
+      - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET}
+      - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile offline_access roles
+      - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=username
+      - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name
+      - GF_AUTH_GENERIC_OAUTH_USE_PKCE=true
+      - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth
+      - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token
+      - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo
+      - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='GrafanaAdmin'
+      - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true
+    entrypoint:
+      - sh
+      - -euc
+      - |
+        mkdir -p /etc/grafana/provisioning/datasources
+        cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
+        apiVersion: 1
+        datasources:
+        - name: Loki
+          type: loki
+          access: proxy
+          orgId: 1
+          url: http://loki:3100
+          basicAuth: false
+          isDefault: true
+          version: 1
+          editable: true
+        EOF
+        /run.sh
+    user:
+      ${BENTO_UID}
+    volumes:
+      - ${BENTO_GRAFANA_LIB_DIR}:/var/lib/grafana
+    expose:
+      - 3000
+    healthcheck:
+      test: [ "CMD", "curl", "-k", "https://localhost:3000"]
+      timeout: 5s
+      interval: 15s
+    profiles:
+      - monitoring
+    networks:
+      - monitoring-net
+
+  loki:
+    container_name: ${BENTO_LOKI_CONTAINER_NAME}
+    image: ${BENTO_LOKI_IMAGE}:${BENTO_LOKI_IMAGE_VERSION}
+    volumes:
+      - ${BENTO_LOKI_TEMP_DIR}:/tmp/loki
+      - ${PWD}/lib/logs/loki-config.yaml:/etc/loki/loki-config.yaml
+    expose:
+      - 3100
+    command: -config.file=/etc/loki/loki-config.yaml
+    user:
+      ${BENTO_UID}
+    networks:
+      - monitoring-net
+    profiles:
+      - monitoring
+
+  promtail:
+    container_name: ${BENTO_PROMTAIL_CONTAINER_NAME}
+    image: ${BENTO_PROMTAIL_IMAGE}:${BENTO_PROMTAIL_IMAGE_VERSION}
+    volumes:
+      - "/var/lib/docker/containers:/var/lib/docker/containers"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "${PWD}/lib/logs/promtail-config.yaml:/etc/promtail/config.yaml"
+    command: "-config.file=/etc/promtail/config.yaml"  
+    networks:
+      - monitoring-net
+    profiles:
+      - monitoring
+
+networks:
+   monitoring-net:
+    external: true
+    name: ${BENTO_MONITORING_NETWORK}
diff --git a/lib/logs/loki-config.yaml b/lib/logs/loki-config.yaml
@@ -0,0 +1,33 @@
+
+# This is a complete configuration to deploy Loki backed by the filesystem.
+# The index will be shipped to the storage via tsdb-shipper.
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /tmp/loki
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: tsdb
+    object_store: filesystem
+    schema: v13
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+  filesystem:
+    directory: /tmp/loki/chunks
+
+limits_config:
+  reject_old_samples: false
diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml
@@ -0,0 +1,24 @@
+server:
+    http_listen_port: 9080
+    grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    pipeline_stages:
+      - static_labels:
+          job: docker
+          host: docker
+          agent: promtail
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
diff --git a/lib/web/docker-compose.web.yaml b/lib/web/docker-compose.web.yaml
@@ -9,6 +9,7 @@ services:
     environment:
       - BENTO_UID
       - BENTO_CBIOPORTAL_ENABLED
+      - BENTO_MONITORING_ENABLED
       - BENTO_CBIOPORTAL_PUBLIC_URL
       - BENTO_DROP_BOX_FS_BASE_PATH
       - BENTO_URL=${BENTOV2_PORTAL_PUBLIC_URL}

diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py
@@ -41,6 +41,9 @@
 WES_CLIENT_ID = os.getenv("BENTO_WES_CLIENT_ID")
 WES_WORKFLOW_TIMEOUT = int(os.getenv("BENTOV2_WES_WORKFLOW_TIMEOUT"))
 
+GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID")
+GRAFANA_PRIVATE_URL = os.getenv("BENTO_PRIVATE_GRAFANA_URL")
+
 KC_CLIENTS_ENDPOINT = f"admin/realms/{AUTH_REALM}/clients"
 
 
@@ -226,6 +229,42 @@ def create_web_client_if_needed(token: str) -> None:
             use_refresh_tokens=True,
         )
 
+    def create_grafana_client_if_needed(token: str) -> None:
+        grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID)
+
+        if grafana_client_kc_id is None:
+            # Create the Bento WES client
+            create_keycloak_client_or_exit(
+                token,
+                GRAFANA_CLIENT_ID,
+                standard_flow_enabled=True,
+                service_accounts_enabled=False,
+                public_client=False,  # Use client secret for this one
+                redirect_uris=[
+                    f"{GRAFANA_PRIVATE_URL}/*"
+                ],
+                web_origins=[GRAFANA_PRIVATE_URL],
+                access_token_lifespan=900,  # default access token lifespan: 15 minutes
+                use_refresh_tokens=False,
+            )
+            grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID)
+
+        # Fetch and print secret
+
+        client_secret_res = get_keycloak_client_secret(grafana_client_kc_id, token)
+
+        client_secret_data = client_secret_res.json()
+        if not client_secret_res.ok:
+            err(f"    Failed to get client secret for {GRAFANA_CLIENT_ID}; {client_secret_res.status_code} "
+                f"{client_secret_data}")
+            exit(1)
+
+        client_secret = client_secret_data["value"]
+        cprint(
+            f"    Please set BENTO_GRAFANA_CLIENT_SECRET to {client_secret} in local.env and restart Grafana",
+            attrs=["bold"],
+        )
+
     # noinspection PyUnusedLocal
     def create_cbioportal_client_if_needed(token: str) -> None:
         cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID)
@@ -376,6 +415,11 @@ def success():
     create_wes_client_if_needed(access_token)
     success()
 
+    if c.BENTO_FEATURE_MONITORING.enabled:
+        info(f"  Creating Grafana client: {GRAFANA_CLIENT_ID}")
+        create_grafana_client_if_needed(access_token)
+        success()
+
     info(f"  Creating user: {AUTH_TEST_USER}")
     create_test_user_if_needed(access_token)
     success()

diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py
@@ -103,6 +103,9 @@ def __init__(self, enabled: bool, profile: str):
     enabled=_env_get_bool("BENTO_CBIOPORTAL_ENABLED", default=False), profile="cbioportal")
 BENTO_FEATURE_GOHAN = BentoOptionalFeature(
     enabled=_env_get_bool("BENTO_GOHAN_ENABLED", default=False), profile="gohan")
+BENTO_FEATURE_MONITORING = BentoOptionalFeature(
+    enabled=_env_get_bool("BENTO_MONITORING_ENABLED", default=False), profile="monitoring")
+
 BENTO_FEATURE_PUBLIC = BentoOptionalFeature(enabled=BENTOV2_USE_BENTO_PUBLIC, profile="public")
 BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=bool(BENTO_DOMAIN_REDIRECT), profile="redirect")
 

diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py
@@ -287,6 +287,7 @@ def init_docker(client: docker.DockerClient):
         ("BENTO_GOHAN_ES_NETWORK", dict(driver="bridge", internal=True)),  # Does not need to access the web
         ("BENTO_KATSU_NETWORK", dict(driver="bridge")),
         ("BENTO_KATSU_DB_NETWORK", dict(driver="bridge", internal=True)),  # Does not need to access the web
+        ("BENTO_MONITORING_NETWORK", dict(driver="bridge")),
         ("BENTO_NOTIFICATION_NETWORK", dict(driver="bridge")),
         ("BENTO_PUBLIC_NETWORK", dict(driver="bridge")),
         ("BENTO_REDIS_NETWORK", dict(driver="bridge", internal=True)),  # Does not need to access the web