diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 96dddf5f..8e23f5af 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -65,6 +65,11 @@ services: - ${BENTOV2_DOMAIN} - ${BENTOV2_PORTAL_DOMAIN} - ${BENTOV2_AUTH_DOMAIN} + monitoring-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + - ${BENTOV2_AUTH_DOMAIN} public-net: aliases: - ${BENTOV2_DOMAIN} @@ -232,3 +237,14 @@ services: cbioportal: ports: - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" + + grafana: + ports: + - "3000:3000" + environment: + # Workaround for self signed certificates in dev + - GF_AUTH_GENERIC_OAUTH_TLS_SKIP_VERIFY_INSECURE=true + + loki: + ports: + - "3100:3100" diff --git a/docker-compose.yaml b/docker-compose.yaml index 05c9e908..f91aaa52 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,6 +13,7 @@ include: - lib/event-relay/docker-compose.event-relay.yaml - lib/gohan/docker-compose.gohan.yaml # Optional feature; controlled by a compose profile - lib/katsu/docker-compose.katsu.yaml + - lib/logs/docker-compose.logs.yaml - lib/notification/docker-compose.notification.yaml - lib/public/docker-compose.public.yaml # Optional feature; controlled by a compose profile - lib/redis/docker-compose.redis.yaml diff --git a/etc/bento.env b/etc/bento.env index 9c0f84ad..2ff92af4 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -440,3 +440,19 @@ BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.15 BENTO_CBIOPORTAL_SESSION_DATABASE_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session-db BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/session_db # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network + +# Monitoring +BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net +BENTO_PRIVATE_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/grafana +BENTO_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana +BENTO_LOKI_IMAGE=grafana/loki +BENTO_LOKI_IMAGE_VERSION=3.1.1 +BENTO_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki +BENTO_LOKI_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp +BENTO_GRAFANA_IMAGE=grafana/grafana +BENTO_GRAFANA_IMAGE_VERSION=11.1.3 +BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana +BENTO_GRAFANA_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib +BENTO_PROMTAIL_IMAGE=grafana/promtail +BENTO_PROMTAIL_IMAGE_VERSION=2.9.10 +BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 86a5c98a..8d0be71d 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -11,6 +11,7 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='false' # Set to true if using Beacon! BENTO_BEACON_UI_ENABLED='false' BENTO_CBIOPORTAL_ENABLED='false' +BENTO_MONITORING_ENABLED='false' BENTO_GOHAN_ENABLED='true' # - Switch to enable French translation in Bento Public @@ -53,6 +54,10 @@ BENTO_AUTHZ_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! + +# - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_GRAFANA_CLIENT_ID=grafana +BENTO_GRAFANA_CLIENT_SECRET= # --------------------------------------------------------------------- BENTO_WEB_CUSTOM_HEADER= diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 884d1d65..9e471dc2 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -11,6 +11,7 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' +BENTO_MONITORING_ENABLED='false' BENTO_GOHAN_ENABLED='true' # - Switch to enable French translation in Bento Public @@ -53,6 +54,10 @@ BENTOV2_AUTH_TEST_PASSWORD= # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= + +# - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_GRAFANA_CLIENT_ID=grafana +BENTO_GRAFANA_CLIENT_SECRET= # -------------------------------------------------------------------- # Gohan diff --git a/etc/default_config.env b/etc/default_config.env index 949f7309..3ff8966f 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -81,6 +81,9 @@ BENTO_AUTHZ_DB_PASSWORD= # - cBioPortal Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM BENTO_CBIOPORTAL_CLIENT_ID=cbioportal BENTO_CBIOPORTAL_CLIENT_SECRET= +# - Grafana Client ID/secret; secret to be filled by local.env client within BENTOV2_AUTH_REALM +BENTO_GRAFANA_CLIENT_ID=grafana +BENTO_GRAFANA_CLIENT_SECRET= # - WES Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 83b9385e..5ccab6b7 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -22,6 +22,7 @@ services: - BENTO_BEACON_ENABLED - BENTO_CBIOPORTAL_ENABLED - BENTO_GOHAN_ENABLED + - BENTO_MONITORING_ENABLED - BENTOV2_GATEWAY_CONTAINER_NAME @@ -76,6 +77,7 @@ services: - BENTO_BEACON_INTERNAL_PORT - BENTO_CBIOPORTAL_CONTAINER_NAME - BENTO_CBIOPORTAL_INTERNAL_PORT + - BENTO_GRAFANA_CONTAINER_NAME networks: - aggregation-net - auth-net @@ -87,6 +89,7 @@ services: - event-relay-net - gohan-api-net - katsu-net + - monitoring-net - notification-net - public-net - reference-net @@ -166,6 +169,9 @@ networks: katsu-net: external: true name: ${BENTO_KATSU_NETWORK} + monitoring-net: + external: true + name: ${BENTO_MONITORING_NETWORK} notification-net: external: true name: ${BENTO_NOTIFICATION_NETWORK} diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl new file mode 100644 index 00000000..214968a3 --- /dev/null +++ b/lib/gateway/services/grafana.conf.tpl @@ -0,0 +1,13 @@ +# env: BENTO_MONITORING_ENABLED +location /api/grafana { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/grafana/; } +location /api/grafana/ { + # Reverse proxy settings + include /gateway/conf/proxy.conf; + include /gateway/conf/proxy_extra.conf; + + # Immediate set/re-use means we don't get resolve errors if not up (as opposed to passing as a literal) + set $upstream_grafana http://${BENTO_GRAFANA_CONTAINER_NAME}:3000; + + proxy_pass $upstream_grafana; + error_log /var/log/bentov2_grafana_errors.log; +} diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml new file mode 100644 index 00000000..f8b3c0bb --- /dev/null +++ b/lib/logs/docker-compose.logs.yaml @@ -0,0 +1,93 @@ +services: + grafana: + image: ${BENTO_GRAFANA_IMAGE}:${BENTO_GRAFANA_IMAGE_VERSION} + container_name: ${BENTO_GRAFANA_CONTAINER_NAME} + environment: + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_SERVER_ROOT_URL=${BENTO_PRIVATE_GRAFANA_URL} + - GF_SERVER_SERVE_FROM_SUB_PATH=true + - GF_SECURITY_COOKIE_SAMESITE=none + - GF_SECURITY_ALLOW_EMBEDDING=true + - GF_LOG_LEVEL=debug + - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true + - GF_AUTH_GENERIC_OAUTH_ENABLED=true + - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth + - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true + - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} + - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET} + - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile offline_access roles + - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=username + - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name + - GF_AUTH_GENERIC_OAUTH_USE_PKCE=true + - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth + - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token + - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='GrafanaAdmin' + - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true + entrypoint: + - sh + - -euc + - | + mkdir -p /etc/grafana/provisioning/datasources + cat < /etc/grafana/provisioning/datasources/ds.yaml + apiVersion: 1 + datasources: + - name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: true + EOF + /run.sh + user: + ${BENTO_UID} + volumes: + - ${BENTO_GRAFANA_LIB_DIR}:/var/lib/grafana + expose: + - 3000 + healthcheck: + test: [ "CMD", "curl", "-k", "https://localhost:3000"] + timeout: 5s + interval: 15s + profiles: + - monitoring + networks: + - monitoring-net + + loki: + container_name: ${BENTO_LOKI_CONTAINER_NAME} + image: ${BENTO_LOKI_IMAGE}:${BENTO_LOKI_IMAGE_VERSION} + volumes: + - ${BENTO_LOKI_TEMP_DIR}:/tmp/loki + - ${PWD}/lib/logs/loki-config.yaml:/etc/loki/loki-config.yaml + expose: + - 3100 + command: -config.file=/etc/loki/loki-config.yaml + user: + ${BENTO_UID} + networks: + - monitoring-net + profiles: + - monitoring + + promtail: + container_name: ${BENTO_PROMTAIL_CONTAINER_NAME} + image: ${BENTO_PROMTAIL_IMAGE}:${BENTO_PROMTAIL_IMAGE_VERSION} + volumes: + - "/var/lib/docker/containers:/var/lib/docker/containers" + - "/var/run/docker.sock:/var/run/docker.sock" + - "${PWD}/lib/logs/promtail-config.yaml:/etc/promtail/config.yaml" + command: "-config.file=/etc/promtail/config.yaml" + networks: + - monitoring-net + profiles: + - monitoring + +networks: + monitoring-net: + external: true + name: ${BENTO_MONITORING_NETWORK} diff --git a/lib/logs/loki-config.yaml b/lib/logs/loki-config.yaml new file mode 100644 index 00000000..8e47a677 --- /dev/null +++ b/lib/logs/loki-config.yaml @@ -0,0 +1,33 @@ + +# This is a complete configuration to deploy Loki backed by the filesystem. +# The index will be shipped to the storage via tsdb-shipper. + +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /tmp/loki + +schema_config: + configs: + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /tmp/loki/chunks + +limits_config: + reject_old_samples: false diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml new file mode 100644 index 00000000..fe6d644c --- /dev/null +++ b/lib/logs/promtail-config.yaml @@ -0,0 +1,24 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + pipeline_stages: + - static_labels: + job: docker + host: docker + agent: promtail + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' diff --git a/lib/web/docker-compose.web.yaml b/lib/web/docker-compose.web.yaml index 1de72f2f..0865f3be 100644 --- a/lib/web/docker-compose.web.yaml +++ b/lib/web/docker-compose.web.yaml @@ -9,6 +9,7 @@ services: environment: - BENTO_UID - BENTO_CBIOPORTAL_ENABLED + - BENTO_MONITORING_ENABLED - BENTO_CBIOPORTAL_PUBLIC_URL - BENTO_DROP_BOX_FS_BASE_PATH - BENTO_URL=${BENTOV2_PORTAL_PUBLIC_URL} diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 9d9b2437..90211a8e 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -41,6 +41,9 @@ WES_CLIENT_ID = os.getenv("BENTO_WES_CLIENT_ID") WES_WORKFLOW_TIMEOUT = int(os.getenv("BENTOV2_WES_WORKFLOW_TIMEOUT")) +GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID") +GRAFANA_PRIVATE_URL = os.getenv("BENTO_PRIVATE_GRAFANA_URL") + KC_CLIENTS_ENDPOINT = f"admin/realms/{AUTH_REALM}/clients" @@ -226,6 +229,42 @@ def create_web_client_if_needed(token: str) -> None: use_refresh_tokens=True, ) + def create_grafana_client_if_needed(token: str) -> None: + grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) + + if grafana_client_kc_id is None: + # Create the Bento WES client + create_keycloak_client_or_exit( + token, + GRAFANA_CLIENT_ID, + standard_flow_enabled=True, + service_accounts_enabled=False, + public_client=False, # Use client secret for this one + redirect_uris=[ + f"{GRAFANA_PRIVATE_URL}/*" + ], + web_origins=[GRAFANA_PRIVATE_URL], + access_token_lifespan=900, # default access token lifespan: 15 minutes + use_refresh_tokens=False, + ) + grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) + + # Fetch and print secret + + client_secret_res = get_keycloak_client_secret(grafana_client_kc_id, token) + + client_secret_data = client_secret_res.json() + if not client_secret_res.ok: + err(f" Failed to get client secret for {GRAFANA_CLIENT_ID}; {client_secret_res.status_code} " + f"{client_secret_data}") + exit(1) + + client_secret = client_secret_data["value"] + cprint( + f" Please set BENTO_GRAFANA_CLIENT_SECRET to {client_secret} in local.env and restart Grafana", + attrs=["bold"], + ) + # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) @@ -376,6 +415,11 @@ def success(): create_wes_client_if_needed(access_token) success() + if c.BENTO_FEATURE_MONITORING.enabled: + info(f" Creating Grafana client: {GRAFANA_CLIENT_ID}") + create_grafana_client_if_needed(access_token) + success() + info(f" Creating user: {AUTH_TEST_USER}") create_test_user_if_needed(access_token) success() diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index be1c06dc..dbf33762 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -103,6 +103,9 @@ def __init__(self, enabled: bool, profile: str): enabled=_env_get_bool("BENTO_CBIOPORTAL_ENABLED", default=False), profile="cbioportal") BENTO_FEATURE_GOHAN = BentoOptionalFeature( enabled=_env_get_bool("BENTO_GOHAN_ENABLED", default=False), profile="gohan") +BENTO_FEATURE_MONITORING = BentoOptionalFeature( + enabled=_env_get_bool("BENTO_MONITORING_ENABLED", default=False), profile="monitoring") + BENTO_FEATURE_PUBLIC = BentoOptionalFeature(enabled=BENTOV2_USE_BENTO_PUBLIC, profile="public") BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=bool(BENTO_DOMAIN_REDIRECT), profile="redirect") diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 40dbf6ac..9f0e3eeb 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -287,6 +287,7 @@ def init_docker(client: docker.DockerClient): ("BENTO_GOHAN_ES_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web ("BENTO_KATSU_NETWORK", dict(driver="bridge")), ("BENTO_KATSU_DB_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web + ("BENTO_MONITORING_NETWORK", dict(driver="bridge")), ("BENTO_NOTIFICATION_NETWORK", dict(driver="bridge")), ("BENTO_PUBLIC_NETWORK", dict(driver="bridge")), ("BENTO_REDIS_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web