From 7c935caa17eb0069690ec0c8d4d921db5a57343f Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Wed, 22 May 2024 16:11:54 -0400 Subject: [PATCH 01/75] add basic loki, promtail, grafana logging functionality --- docker-compose.yaml | 3 +- lib/logs/docker-compose.logs.yaml | 52 +++++++++++++++++++++++++++++++ lib/logs/promtail-config.yaml | 30 ++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 lib/logs/docker-compose.logs.yaml create mode 100644 lib/logs/promtail-config.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index 05c9e908..f00eae74 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,6 +13,7 @@ include: - lib/event-relay/docker-compose.event-relay.yaml - lib/gohan/docker-compose.gohan.yaml # Optional feature; controlled by a compose profile - lib/katsu/docker-compose.katsu.yaml + - lib/logs/docker-compose.logs.yaml - lib/notification/docker-compose.notification.yaml - lib/public/docker-compose.public.yaml # Optional feature; controlled by a compose profile - lib/redis/docker-compose.redis.yaml @@ -20,4 +21,4 @@ include: - lib/service-registry/docker-compose.service-registry.yaml - lib/web/docker-compose.web.yaml - lib/wes/docker-compose.wes.yaml - project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory + project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory \ No newline at end of file diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml new file mode 100644 index 00000000..b8dc1dcc --- /dev/null +++ b/lib/logs/docker-compose.logs.yaml @@ -0,0 +1,52 @@ +services: + grafana: + container_name: bentov2-grafana + environment: + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + entrypoint: + - sh + - -euc + - | + mkdir -p /etc/grafana/provisioning/datasources + cat < /etc/grafana/provisioning/datasources/ds.yaml + apiVersion: 1 + datasources: + - name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: true + EOF + /run.sh + image: grafana/grafana:latest + ports: + - "3000:3000" + networks: + - loki + loki: + container_name: bentov2-loki + image: grafana/loki:2.9.2 + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - loki + promtail: + container_name: bentov2-promtail + image: grafana/promtail:2.9.2 + volumes: + - "/var/log:/var/log" + - "/var/lib/docker/containers:/var/lib/docker/containers" + - "/var/run/docker.sock:/var/run/docker.sock" + - "./lib/logs/promtail-config.yaml:/etc/promtail/config.yaml" + command: "-config.file=/etc/promtail/config.yaml" + networks: + - loki +networks: + loki: \ No newline at end of file diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml new file mode 100644 index 00000000..b2e89c93 --- /dev/null +++ b/lib/logs/promtail-config.yaml @@ -0,0 +1,30 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker +# pipeline_stages: +# - docker: {} +# static_configs: +# - targets: +# - localhost +# labels: +# job: docker +# __path__: /var/lib/docker/containers/*/*-json.log + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + #- name: name + # values: [flog] + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' \ No newline at end of file From bbc909f0905ff2bb95614ada1df9a3e85484bba9 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Wed, 5 Jun 2024 09:48:15 -0400 Subject: [PATCH 02/75] Add official loki and grafana endpoints --- etc/bento.env | 3 ++ lib/gateway/docker-compose.gateway.yaml | 5 ++++ lib/gateway/public_services/grafana.conf.tpl | 12 ++++++++ lib/gateway/public_services/loki.conf.tpl | 15 ++++++++++ lib/logs/docker-compose.logs.yaml | 29 +++++++++++++++++--- lib/logs/promtail-config.yaml | 11 +++++++- py_bentoctl/other_helpers.py | 1 + 7 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 lib/gateway/public_services/grafana.conf.tpl create mode 100644 lib/gateway/public_services/loki.conf.tpl diff --git a/etc/bento.env b/etc/bento.env index fa7d0982..19dd6506 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -439,3 +439,6 @@ BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.7 BENTO_CBIOPORTAL_SESSION_DATABASE_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session-db BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/session_db # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network + +# Monitoring +BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net \ No newline at end of file diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 83b9385e..57582494 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -87,6 +87,7 @@ services: - event-relay-net - gohan-api-net - katsu-net + - monitoring-net - notification-net - public-net - reference-net @@ -104,6 +105,7 @@ services: - ${BENTOV2_GATEWAY_CERTS_DIR}:${BENTOV2_GATEWAY_INTERNAL_CERTS_DIR}:ro - ${PWD}/lib/gateway/services:/gateway/services:ro - ${PWD}/lib/gateway/public_services:/gateway/public_services:ro + - /var/log:/var/log mem_limit: ${BENTOV2_GATEWAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 cpus: ${BENTOV2_GATEWAY_CPUS} cpu_shares: 512 @@ -166,6 +168,9 @@ networks: katsu-net: external: true name: ${BENTO_KATSU_NETWORK} + monitoring-net: + external: true + name: ${BENTO_MONITORING_NETWORK} notification-net: external: true name: ${BENTO_NOTIFICATION_NETWORK} diff --git a/lib/gateway/public_services/grafana.conf.tpl b/lib/gateway/public_services/grafana.conf.tpl new file mode 100644 index 00000000..70e22873 --- /dev/null +++ b/lib/gateway/public_services/grafana.conf.tpl @@ -0,0 +1,12 @@ +location /api/grafana { return 302 https://${BENTOV2_DOMAIN}/api/grafana/; } +location /api/grafana/ { + # Reverse proxy settings + include /gateway/conf/proxy.conf; + + add_header Content-Security-Policy "frame-ancestors 'self' https://${BENTOV2_DOMAIN};"; + # Immediate set/re-use means we don't get resolve errors if not up (as opposed to passing as a literal) + set $upstream_cbio http://bentov2-grafana:3000; + + proxy_pass $upstream_cbio; + error_log /var/log/bentov2_grafana_errors.log; +} diff --git a/lib/gateway/public_services/loki.conf.tpl b/lib/gateway/public_services/loki.conf.tpl new file mode 100644 index 00000000..ec532dca --- /dev/null +++ b/lib/gateway/public_services/loki.conf.tpl @@ -0,0 +1,15 @@ +location /api/loki { return 302 https://${BENTOV2_DOMAIN}/api/loki/; } +location /api/loki/ { + # Reverse proxy settings + include /gateway/conf/proxy.conf; + include /gateway/conf/proxy_extra.conf; + + # Forward request to the aggregation + rewrite ^ $request_uri; + rewrite ^/api/loki/(.*) /$1 break; + return 400; + proxy_pass http://bentov2-loki:3100/loki/api/v1$uri; + + # Errors + error_log /var/log/bentov2_loki_errors.log; +} diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index b8dc1dcc..aa32971b 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -5,6 +5,9 @@ services: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_SERVER_ROOT_URL=https://bentov2.local:443/api/grafana + - GF_SERVER_SERVE_FROM_SUB_PATH=true + - GF_SECURITY_ALLOW_EMBEDDING=true entrypoint: - sh - -euc @@ -28,7 +31,8 @@ services: ports: - "3000:3000" networks: - - loki + - monitoring-net + loki: container_name: bentov2-loki image: grafana/loki:2.9.2 @@ -36,9 +40,11 @@ services: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml networks: - - loki + - monitoring-net + promtail: container_name: bentov2-promtail + restart: unless-stopped image: grafana/promtail:2.9.2 volumes: - "/var/log:/var/log" @@ -47,6 +53,21 @@ services: - "./lib/logs/promtail-config.yaml:/etc/promtail/config.yaml" command: "-config.file=/etc/promtail/config.yaml" networks: - - loki + - monitoring-net + + prometheus: + container_name: bentov2-prometheus + image: prom/prometheus:latest + ports: + - 9090:9090 + command: + - --config.file=/etc/prometheus/prometheus.yaml + volumes: + - ./lib/logs/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + networks: + - monitoring-net + networks: - loki: \ No newline at end of file + monitoring-net: + external: true + name: ${BENTO_MONITORING_NETWORK} \ No newline at end of file diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml index b2e89c93..19a26bde 100644 --- a/lib/logs/promtail-config.yaml +++ b/lib/logs/promtail-config.yaml @@ -27,4 +27,13 @@ scrape_configs: relabel_configs: - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' - target_label: 'container' \ No newline at end of file + target_label: 'container' + - job_name: nginx + static_configs: + - targets: + - localhost + labels: + job: nginx + host: nginx01 + agent: promtail + __path__: /var/log/access.log \ No newline at end of file diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index addb76e7..bea82f7a 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -286,6 +286,7 @@ def init_docker(client: docker.DockerClient): ("BENTO_GOHAN_ES_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web ("BENTO_KATSU_NETWORK", dict(driver="bridge")), ("BENTO_KATSU_DB_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web + ("BENTO_MONITORING_NETWORK", dict(driver="bridge")), ("BENTO_NOTIFICATION_NETWORK", dict(driver="bridge")), ("BENTO_PUBLIC_NETWORK", dict(driver="bridge")), ("BENTO_REDIS_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web From ed99c93d3dbc3d1718d32cb53d999c9b7a15a936 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Thu, 13 Jun 2024 12:47:56 -0400 Subject: [PATCH 03/75] pr verions --- etc/bento.env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index b0f256bd..8fd34b0c 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=0.11.0 +BENTOV2_GATEWAY_VERSION=pr-17 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=edge +BENTOV2_WEB_VERSION=pr-404 BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net @@ -269,7 +269,7 @@ BENTOV2_KATSU_DB_CPUS=4 # Katsu BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu -BENTOV2_KATSU_VERSION=edge +BENTOV2_KATSU_VERSION=pr-508 BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=edge +BENTO_PUBLIC_VERSION=pr-161 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 981eaaa4a4f39987c6ff259f85392cd1651030f4 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 17 Jun 2024 16:38:44 -0400 Subject: [PATCH 04/75] storing for later --- etc/bento.env | 16 +++++++++++- lib/logs/docker-compose.logs.yaml | 41 ++++++++++++++++++++++--------- lib/logs/loki-config.yaml | 30 ++++++++++++++++++++++ lib/logs/prometheus.yaml | 18 ++++++++++++++ lib/logs/promtail-config.yaml | 13 ++++------ 5 files changed, 97 insertions(+), 21 deletions(-) create mode 100644 lib/logs/loki-config.yaml create mode 100644 lib/logs/prometheus.yaml diff --git a/etc/bento.env b/etc/bento.env index 19dd6506..ae56ff8e 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -441,4 +441,18 @@ BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/ses # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network # Monitoring -BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net \ No newline at end of file +BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net +BENTOV2_LOKI_BASE_IMAGE=grafana/loki +BENTOV2_LOKI_BASE_IMAGE_VERSION=3.0.0 +BENTOV2_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki +BENTOV2_LOKI_PROD_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp +BENTOV2_GRAFANA_BASE_IMAGE=grafana/grafana +BENTOV2_GRAFANA_BASE_IMAGE_VERSION=11.0.0 +BENTOV2_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana +BENTOV2_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib +BENTOV2_PROMTAIL_BASE_IMAGE=grafana/promtail +BENTOV2_PROMTAIL_BASE_IMAGE_VERSION=2.9.2 +BENTOV2_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail +BENTOV2_PROMETHEUS_BASE_IMAGE=prom/prometheus +BENTOV2_PROMETHEUS_BASE_IMAGE_VERSION=v2.52.0 +BENTOV2_PROMETHEUS_CONTAINER_NAME=${BENTOV2_PREFIX}-prometheus diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index aa32971b..de81eeea 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -1,6 +1,7 @@ services: grafana: - container_name: bentov2-grafana + image: ${BENTOV2_GRAFANA_BASE_IMAGE}:${BENTOV2_GRAFANA_BASE_IMAGE_VERSION} + container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - GF_AUTH_ANONYMOUS_ENABLED=true @@ -25,45 +26,61 @@ services: isDefault: true version: 1 editable: true + - name: Promethus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: true EOF /run.sh - image: grafana/grafana:latest + user: + ${BENTO_UID} + volumes: + - ${BENTOV2_GRAFANA_PROD_LIB_DIR}:/var/lib/grafana ports: - "3000:3000" networks: - monitoring-net loki: - container_name: bentov2-loki - image: grafana/loki:2.9.2 + container_name: ${BENTOV2_LOKI_CONTAINER_NAME} + image: ${BENTOV2_LOKI_BASE_IMAGE}:${BENTOV2_LOKI_BASE_IMAGE_VERSION} + volumes: + - ${BENTOV2_LOKI_PROD_TEMP_DIR}:/tmp/loki + - ${PWD}/lib/logs/loki-config.yaml:/etc/loki/loki-config.yaml ports: - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml + command: -config.file=/etc/loki/loki-config.yaml + user: + ${BENTO_UID} networks: - monitoring-net promtail: - container_name: bentov2-promtail - restart: unless-stopped - image: grafana/promtail:2.9.2 + container_name: ${BENTOV2_PROMTAIL_CONTAINER_NAME} + image: ${BENTOV2_PROMTAIL_BASE_IMAGE}:${BENTOV2_PROMTAIL_BASE_IMAGE_VERSION} volumes: - "/var/log:/var/log" - "/var/lib/docker/containers:/var/lib/docker/containers" - "/var/run/docker.sock:/var/run/docker.sock" - - "./lib/logs/promtail-config.yaml:/etc/promtail/config.yaml" + - "${PWD}/lib/logs/promtail-config.yaml:/etc/promtail/config.yaml" command: "-config.file=/etc/promtail/config.yaml" networks: - monitoring-net prometheus: - container_name: bentov2-prometheus - image: prom/prometheus:latest + container_name: ${BENTOV2_PROMETHEUS_CONTAINER_NAME} + image: ${BENTOV2_PROMETHEUS_BASE_IMAGE}:${BENTOV2_PROMETHEUS_BASE_IMAGE_VERSION} ports: - 9090:9090 command: - --config.file=/etc/prometheus/prometheus.yaml volumes: - - ./lib/logs/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ${PWD}/lib/logs/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro networks: - monitoring-net diff --git a/lib/logs/loki-config.yaml b/lib/logs/loki-config.yaml new file mode 100644 index 00000000..81ebd34c --- /dev/null +++ b/lib/logs/loki-config.yaml @@ -0,0 +1,30 @@ + +# This is a complete configuration to deploy Loki backed by the filesystem. +# The index will be shipped to the storage via tsdb-shipper. + +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /tmp/loki + +schema_config: + configs: + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /tmp/loki/chunks \ No newline at end of file diff --git a/lib/logs/prometheus.yaml b/lib/logs/prometheus.yaml new file mode 100644 index 00000000..8d6f7016 --- /dev/null +++ b/lib/logs/prometheus.yaml @@ -0,0 +1,18 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + + external_labels: + monitor: 'my-project' + +rule_files: + + +scrape_configs: + - job_name: 'prometheus' + + scrape_interval: 5s + + + static_configs: + - targets: ['loki:3100'] \ No newline at end of file diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml index 19a26bde..d06df815 100644 --- a/lib/logs/promtail-config.yaml +++ b/lib/logs/promtail-config.yaml @@ -10,14 +10,11 @@ clients: scrape_configs: - job_name: docker -# pipeline_stages: -# - docker: {} -# static_configs: -# - targets: -# - localhost -# labels: -# job: docker -# __path__: /var/lib/docker/containers/*/*-json.log + pipeline_stages: + - static_labels: + job: docker + host: docker + agent: promtail docker_sd_configs: - host: unix:///var/run/docker.sock refresh_interval: 5s From fb977288f80ca45ee03c54146701d1e1195cc91e Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 17 Jun 2024 16:42:38 -0400 Subject: [PATCH 05/75] remove prometheus --- etc/bento.env | 5 +---- lib/logs/docker-compose.logs.yaml | 21 --------------------- lib/logs/prometheus.yaml | 18 ------------------ lib/logs/promtail-config.yaml | 11 +---------- 4 files changed, 2 insertions(+), 53 deletions(-) delete mode 100644 lib/logs/prometheus.yaml diff --git a/etc/bento.env b/etc/bento.env index ae56ff8e..e0dd92bc 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -452,7 +452,4 @@ BENTOV2_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana BENTOV2_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib BENTOV2_PROMTAIL_BASE_IMAGE=grafana/promtail BENTOV2_PROMTAIL_BASE_IMAGE_VERSION=2.9.2 -BENTOV2_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail -BENTOV2_PROMETHEUS_BASE_IMAGE=prom/prometheus -BENTOV2_PROMETHEUS_BASE_IMAGE_VERSION=v2.52.0 -BENTOV2_PROMETHEUS_CONTAINER_NAME=${BENTOV2_PREFIX}-prometheus +BENTOV2_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail \ No newline at end of file diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index de81eeea..fcb17ac6 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -26,15 +26,6 @@ services: isDefault: true version: 1 editable: true - - name: Promethus - type: prometheus - access: proxy - orgId: 1 - url: http://prometheus:9090 - basicAuth: false - isDefault: false - version: 1 - editable: true EOF /run.sh user: @@ -71,18 +62,6 @@ services: command: "-config.file=/etc/promtail/config.yaml" networks: - monitoring-net - - prometheus: - container_name: ${BENTOV2_PROMETHEUS_CONTAINER_NAME} - image: ${BENTOV2_PROMETHEUS_BASE_IMAGE}:${BENTOV2_PROMETHEUS_BASE_IMAGE_VERSION} - ports: - - 9090:9090 - command: - - --config.file=/etc/prometheus/prometheus.yaml - volumes: - - ${PWD}/lib/logs/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - networks: - - monitoring-net networks: monitoring-net: diff --git a/lib/logs/prometheus.yaml b/lib/logs/prometheus.yaml deleted file mode 100644 index 8d6f7016..00000000 --- a/lib/logs/prometheus.yaml +++ /dev/null @@ -1,18 +0,0 @@ -global: - scrape_interval: 5s - evaluation_interval: 5s - - external_labels: - monitor: 'my-project' - -rule_files: - - -scrape_configs: - - job_name: 'prometheus' - - scrape_interval: 5s - - - static_configs: - - targets: ['loki:3100'] \ No newline at end of file diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml index d06df815..fb594f44 100644 --- a/lib/logs/promtail-config.yaml +++ b/lib/logs/promtail-config.yaml @@ -24,13 +24,4 @@ scrape_configs: relabel_configs: - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' - target_label: 'container' - - job_name: nginx - static_configs: - - targets: - - localhost - labels: - job: nginx - host: nginx01 - agent: promtail - __path__: /var/log/access.log \ No newline at end of file + target_label: 'container' \ No newline at end of file From 4cc1b5d349572f150ec4356a83634e84944371cf Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 17 Jun 2024 17:16:46 -0400 Subject: [PATCH 06/75] add SERVICE_URL_BASE_PATH to katsu env --- lib/katsu/docker-compose.katsu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/katsu/docker-compose.katsu.yaml b/lib/katsu/docker-compose.katsu.yaml index e8da7e3f..181c965a 100644 --- a/lib/katsu/docker-compose.katsu.yaml +++ b/lib/katsu/docker-compose.katsu.yaml @@ -25,6 +25,7 @@ services: - DRS_URL=http://${BENTOV2_DRS_CONTAINER_NAME}:${BENTOV2_DRS_INTERNAL_PORT} - SERVICE_TEMP=/app/tmp - SERVICE_SECRET_KEY=${BENTOV2_KATSU_APP_SECRET} + - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/metadata - DJANGO_SETTINGS_MODULE=chord_metadata_service.metadata.settings - BENTOV2_PORTAL_DOMAIN # Allow access by container name or localhost for healthchecks: From af10974feca1ab8ca657ba67d99eae47e084183b Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Thu, 20 Jun 2024 16:54:31 -0400 Subject: [PATCH 07/75] temporary wip --- lib/gateway/public_services/grafana.conf.tpl | 12 ------------ lib/gateway/services/grafana.conf.tpl | 12 ++++++++++++ .../{public_services => services}/loki.conf.tpl | 3 ++- lib/logs/docker-compose.logs.yaml | 7 ++++--- lib/logs/loki-config.yaml | 5 ++++- 5 files changed, 22 insertions(+), 17 deletions(-) delete mode 100644 lib/gateway/public_services/grafana.conf.tpl create mode 100644 lib/gateway/services/grafana.conf.tpl rename lib/gateway/{public_services => services}/loki.conf.tpl (75%) diff --git a/lib/gateway/public_services/grafana.conf.tpl b/lib/gateway/public_services/grafana.conf.tpl deleted file mode 100644 index 70e22873..00000000 --- a/lib/gateway/public_services/grafana.conf.tpl +++ /dev/null @@ -1,12 +0,0 @@ -location /api/grafana { return 302 https://${BENTOV2_DOMAIN}/api/grafana/; } -location /api/grafana/ { - # Reverse proxy settings - include /gateway/conf/proxy.conf; - - add_header Content-Security-Policy "frame-ancestors 'self' https://${BENTOV2_DOMAIN};"; - # Immediate set/re-use means we don't get resolve errors if not up (as opposed to passing as a literal) - set $upstream_cbio http://bentov2-grafana:3000; - - proxy_pass $upstream_cbio; - error_log /var/log/bentov2_grafana_errors.log; -} diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl new file mode 100644 index 00000000..c16ab9f4 --- /dev/null +++ b/lib/gateway/services/grafana.conf.tpl @@ -0,0 +1,12 @@ +location /api/grafana { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/grafana/; } +location /api/grafana/ { + # Reverse proxy settings + include /gateway/conf/proxy.conf; + include /gateway/conf/proxy_extra.conf; + include /gateway/conf/proxy_private.conf; + + + proxy_pass http://bentov2-grafana:3000; + + error_log /var/log/bentov2_grafana_errors.log; +} diff --git a/lib/gateway/public_services/loki.conf.tpl b/lib/gateway/services/loki.conf.tpl similarity index 75% rename from lib/gateway/public_services/loki.conf.tpl rename to lib/gateway/services/loki.conf.tpl index ec532dca..3330c6b6 100644 --- a/lib/gateway/public_services/loki.conf.tpl +++ b/lib/gateway/services/loki.conf.tpl @@ -1,8 +1,9 @@ -location /api/loki { return 302 https://${BENTOV2_DOMAIN}/api/loki/; } +location /api/loki { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/loki/; } location /api/loki/ { # Reverse proxy settings include /gateway/conf/proxy.conf; include /gateway/conf/proxy_extra.conf; + include /gateway/conf/proxy_private.conf; # Forward request to the aggregation rewrite ^ $request_uri; diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index fcb17ac6..6e9492b3 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -4,11 +4,13 @@ services: container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - GF_SERVER_ROOT_URL=https://bentov2.local:443/api/grafana - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_ALLOW_EMBEDDING=true + #- GF_AUTH_JWT_ENABLED=true + #- GF_AUTH_JWT_HEADER_NAME=X-JWT-Assertion + #- GF_AUTH_JWT_USERNAME_CLAIM=sub + #- GF_AUTH_JWT_EMAIL_CLAIM=sub entrypoint: - sh - -euc @@ -55,7 +57,6 @@ services: container_name: ${BENTOV2_PROMTAIL_CONTAINER_NAME} image: ${BENTOV2_PROMTAIL_BASE_IMAGE}:${BENTOV2_PROMTAIL_BASE_IMAGE_VERSION} volumes: - - "/var/log:/var/log" - "/var/lib/docker/containers:/var/lib/docker/containers" - "/var/run/docker.sock:/var/run/docker.sock" - "${PWD}/lib/logs/promtail-config.yaml:/etc/promtail/config.yaml" diff --git a/lib/logs/loki-config.yaml b/lib/logs/loki-config.yaml index 81ebd34c..8e47a677 100644 --- a/lib/logs/loki-config.yaml +++ b/lib/logs/loki-config.yaml @@ -27,4 +27,7 @@ schema_config: storage_config: filesystem: - directory: /tmp/loki/chunks \ No newline at end of file + directory: /tmp/loki/chunks + +limits_config: + reject_old_samples: false From 9feec2d02d92411ddfd649c19e8a1dbb9645a7a2 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Tue, 16 Jul 2024 17:20:48 -0400 Subject: [PATCH 08/75] add authentication --- docker-compose.dev.yaml | 9 +++++++++ lib/gateway/services/grafana.conf.tpl | 4 ++-- lib/logs/docker-compose.logs.yaml | 18 +++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 96dddf5f..f73d872c 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -232,3 +232,12 @@ services: cbioportal: ports: - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" + + grafana: + environment: + # temporary workaround, cannot extract from url on self signed certificates + - GF_AUTH_JWT_JWK_SET_FILE=/etc/grafana/provisioning/jwks.json + depends_on: + - auth + - gateway + diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl index c16ab9f4..d1ac4693 100644 --- a/lib/gateway/services/grafana.conf.tpl +++ b/lib/gateway/services/grafana.conf.tpl @@ -3,9 +3,9 @@ location /api/grafana/ { # Reverse proxy settings include /gateway/conf/proxy.conf; include /gateway/conf/proxy_extra.conf; - include /gateway/conf/proxy_private.conf; - + proxy_pass_request_headers on; + proxy_pass http://bentov2-grafana:3000; error_log /var/log/bentov2_grafana_errors.log; diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 6e9492b3..27cc2fea 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -4,13 +4,16 @@ services: container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_SERVER_ROOT_URL=https://bentov2.local:443/api/grafana + - GF_SERVER_ROOT_URL=https://portal.bentov2.local:443/api/grafana - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_ALLOW_EMBEDDING=true - #- GF_AUTH_JWT_ENABLED=true - #- GF_AUTH_JWT_HEADER_NAME=X-JWT-Assertion - #- GF_AUTH_JWT_USERNAME_CLAIM=sub - #- GF_AUTH_JWT_EMAIL_CLAIM=sub + - GF_AUTH_JWT_ENABLED=true + - GF_AUTH_JWT_ENABLE_LOGIN_TOKEN=true + - GF_AUTH_JWT_HEADER_NAME=X-Forwarded-Access-Token + - GF_AUTH_JWT_USERNAME_CLAIM=sub + - GF_AUTH_JWT_CACHE_TTL=60m + - GF_AUTH_JWT_AUTO_SIGN_UP=true + - GF_LOG_LEVEL=debug entrypoint: - sh - -euc @@ -29,6 +32,7 @@ services: version: 1 editable: true EOF + curl -k https://bentov2-auth:8443/realms/bentov2/protocol/openid-connect/certs -o /etc/grafana/provisioning/jwks.json /run.sh user: ${BENTO_UID} @@ -38,6 +42,7 @@ services: - "3000:3000" networks: - monitoring-net + - auth-net loki: container_name: ${BENTOV2_LOKI_CONTAINER_NAME} @@ -65,6 +70,9 @@ services: - monitoring-net networks: + auth-net: + external: true + name: ${BENTO_AUTH_NETWORK} monitoring-net: external: true name: ${BENTO_MONITORING_NETWORK} \ No newline at end of file From b092e1cc6ba60b17edaa4f0b396c61ff765728b9 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 22 Jul 2024 10:59:24 -0400 Subject: [PATCH 09/75] add jwt authenitcation --- docker-compose.dev.yaml | 4 +++- lib/gateway/services/grafana.conf.tpl | 10 +++++++++- lib/logs/docker-compose.logs.yaml | 11 ++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index f73d872c..1cb74d8c 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -229,6 +229,7 @@ services: "http.cors.allow-origin": "http://localhost:8081" "http.cors.allow-headers": X-Requested-With,Content-Type,Content-Length,Authorization + cbioportal: ports: - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" @@ -239,5 +240,6 @@ services: - GF_AUTH_JWT_JWK_SET_FILE=/etc/grafana/provisioning/jwks.json depends_on: - auth - - gateway + - event-relay + diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl index d1ac4693..8c624db2 100644 --- a/lib/gateway/services/grafana.conf.tpl +++ b/lib/gateway/services/grafana.conf.tpl @@ -4,9 +4,17 @@ location /api/grafana/ { include /gateway/conf/proxy.conf; include /gateway/conf/proxy_extra.conf; + set $auth $http_Authorization; proxy_pass_request_headers on; - + if ($cookie_jwt) { + set $auth $cookie_jwt; + } + proxy_set_header Authorization $auth; + proxy_pass http://bentov2-grafana:3000; + if ($http_Authorization) { + add_header Set-Cookie 'jwt=$http_Authorization; Path=/api/grafana; HttpOnly; Secure' always; + } error_log /var/log/bentov2_grafana_errors.log; } diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 27cc2fea..48a068b7 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -9,7 +9,7 @@ services: - GF_SECURITY_ALLOW_EMBEDDING=true - GF_AUTH_JWT_ENABLED=true - GF_AUTH_JWT_ENABLE_LOGIN_TOKEN=true - - GF_AUTH_JWT_HEADER_NAME=X-Forwarded-Access-Token + - GF_AUTH_JWT_HEADER_NAME=authorization - GF_AUTH_JWT_USERNAME_CLAIM=sub - GF_AUTH_JWT_CACHE_TTL=60m - GF_AUTH_JWT_AUTO_SIGN_UP=true @@ -32,12 +32,17 @@ services: version: 1 editable: true EOF - curl -k https://bentov2-auth:8443/realms/bentov2/protocol/openid-connect/certs -o /etc/grafana/provisioning/jwks.json + touch /etc/grafana/provisioning/jwks.json + curl -k https://bentov2-auth:8443/realms/bentov2/protocol/openid-connect/certs -o /etc/grafana/provisioning/jwks.json /run.sh user: - ${BENTO_UID} + ${BENTO_UID} #entrypoint is sooo messyyy!!!! volumes: - ${BENTOV2_GRAFANA_PROD_LIB_DIR}:/var/lib/grafana + healthcheck: + test: [ "CMD", "curl", "https://localhost:3000", "-k" ] + timeout: 5s + interval: 15s ports: - "3000:3000" networks: From 7417cb7a9e54ff59b293ce80a727fde384bd3d71 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Thu, 25 Jul 2024 13:29:39 -0400 Subject: [PATCH 10/75] add configurable option --- docker-compose.dev.yaml | 8 -------- docker-compose.prod.yaml | 2 +- lib/logs/docker-compose.logs.yaml | 15 ++++++++++----- py_bentoctl/config.py | 3 +++ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 1cb74d8c..30a97a53 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -234,12 +234,4 @@ services: ports: - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" - grafana: - environment: - # temporary workaround, cannot extract from url on self signed certificates - - GF_AUTH_JWT_JWK_SET_FILE=/etc/grafana/provisioning/jwks.json - depends_on: - - auth - - event-relay - diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 547a3d0f..a8e656cb 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -3,4 +3,4 @@ services: environment: - NODE_ENV=production volumes: - - $PWD/lib/web/branding.png:/web/dist/static/branding.png:ro + - $PWD/lib/web/branding.png:/web/dist/static/branding.png:ro \ No newline at end of file diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 48a068b7..4b3f808b 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -4,7 +4,8 @@ services: container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_SERVER_ROOT_URL=https://portal.bentov2.local:443/api/grafana + - GF_SERVER_ROOT_URL=https://${BENTOV2_PORTAL_DOMAIN}:443/api/grafana + - GF_AUTH_JWT_JWK_SET_URL=https://${BENTOV2_AUTH_CONTAINER_NAME}:8443/realms/bentov2/protocol/openid-connect/certs - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_ALLOW_EMBEDDING=true - GF_AUTH_JWT_ENABLED=true @@ -32,19 +33,19 @@ services: version: 1 editable: true EOF - touch /etc/grafana/provisioning/jwks.json - curl -k https://bentov2-auth:8443/realms/bentov2/protocol/openid-connect/certs -o /etc/grafana/provisioning/jwks.json /run.sh user: - ${BENTO_UID} #entrypoint is sooo messyyy!!!! + ${BENTO_UID} volumes: - ${BENTOV2_GRAFANA_PROD_LIB_DIR}:/var/lib/grafana healthcheck: - test: [ "CMD", "curl", "https://localhost:3000", "-k" ] + test: [ "CMD", "curl", "-k", "https://localhost:3000"] timeout: 5s interval: 15s ports: - "3000:3000" + profiles: + - monitoring networks: - monitoring-net - auth-net @@ -62,6 +63,8 @@ services: ${BENTO_UID} networks: - monitoring-net + profiles: + - monitoring promtail: container_name: ${BENTOV2_PROMTAIL_CONTAINER_NAME} @@ -73,6 +76,8 @@ services: command: "-config.file=/etc/promtail/config.yaml" networks: - monitoring-net + profiles: + - monitoring networks: auth-net: diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index be1c06dc..dbf33762 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -103,6 +103,9 @@ def __init__(self, enabled: bool, profile: str): enabled=_env_get_bool("BENTO_CBIOPORTAL_ENABLED", default=False), profile="cbioportal") BENTO_FEATURE_GOHAN = BentoOptionalFeature( enabled=_env_get_bool("BENTO_GOHAN_ENABLED", default=False), profile="gohan") +BENTO_FEATURE_MONITORING = BentoOptionalFeature( + enabled=_env_get_bool("BENTO_MONITORING_ENABLED", default=False), profile="monitoring") + BENTO_FEATURE_PUBLIC = BentoOptionalFeature(enabled=BENTOV2_USE_BENTO_PUBLIC, profile="public") BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=bool(BENTO_DOMAIN_REDIRECT), profile="redirect") From 1127b22482395ef0c1f73eb7c6e25d8e93bc7a4e Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 5 Aug 2024 10:57:34 -0400 Subject: [PATCH 11/75] configure to work with generic oauth --- docker-compose.dev.yaml | 10 ++++++++++ etc/bento.env | 2 +- lib/gateway/services/grafana.conf.tpl | 11 ----------- lib/logs/docker-compose.logs.yaml | 23 +++++++++++++++-------- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 30a97a53..f4800ee2 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -65,6 +65,11 @@ services: - ${BENTOV2_DOMAIN} - ${BENTOV2_PORTAL_DOMAIN} - ${BENTOV2_AUTH_DOMAIN} + monitoring-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + - ${BENTOV2_AUTH_DOMAIN} public-net: aliases: - ${BENTOV2_DOMAIN} @@ -234,4 +239,9 @@ services: ports: - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" + grafana: + environment: + # Workaround for self signed certificates in dev + - GF_AUTH_GENERIC_OAUTH_TLS_SKIP_VERIFY_INSECURE=true + diff --git a/etc/bento.env b/etc/bento.env index e0dd92bc..0b04c342 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -447,7 +447,7 @@ BENTOV2_LOKI_BASE_IMAGE_VERSION=3.0.0 BENTOV2_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki BENTOV2_LOKI_PROD_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp BENTOV2_GRAFANA_BASE_IMAGE=grafana/grafana -BENTOV2_GRAFANA_BASE_IMAGE_VERSION=11.0.0 +BENTOV2_GRAFANA_BASE_IMAGE_VERSION=11.1.1-ubuntu BENTOV2_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana BENTOV2_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib BENTOV2_PROMTAIL_BASE_IMAGE=grafana/promtail diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl index 8c624db2..6bc85817 100644 --- a/lib/gateway/services/grafana.conf.tpl +++ b/lib/gateway/services/grafana.conf.tpl @@ -4,17 +4,6 @@ location /api/grafana/ { include /gateway/conf/proxy.conf; include /gateway/conf/proxy_extra.conf; - set $auth $http_Authorization; - proxy_pass_request_headers on; - if ($cookie_jwt) { - set $auth $cookie_jwt; - } - proxy_set_header Authorization $auth; - proxy_pass http://bentov2-grafana:3000; - if ($http_Authorization) { - add_header Set-Cookie 'jwt=$http_Authorization; Path=/api/grafana; HttpOnly; Secure' always; - } - error_log /var/log/bentov2_grafana_errors.log; } diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 4b3f808b..2b186801 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -4,17 +4,24 @@ services: container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_SERVER_ROOT_URL=https://${BENTOV2_PORTAL_DOMAIN}:443/api/grafana - - GF_AUTH_JWT_JWK_SET_URL=https://${BENTOV2_AUTH_CONTAINER_NAME}:8443/realms/bentov2/protocol/openid-connect/certs + - GF_SERVER_ROOT_URL=https://${BENTOV2_PORTAL_DOMAIN}/api/grafana - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_ALLOW_EMBEDDING=true - - GF_AUTH_JWT_ENABLED=true - - GF_AUTH_JWT_ENABLE_LOGIN_TOKEN=true - - GF_AUTH_JWT_HEADER_NAME=authorization - - GF_AUTH_JWT_USERNAME_CLAIM=sub - - GF_AUTH_JWT_CACHE_TTL=60m - - GF_AUTH_JWT_AUTO_SIGN_UP=true - GF_LOG_LEVEL=debug + - GF_AUTH_GENERIC_OAUTH_ENABLED=true + - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth + - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true + - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana-oauth + - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=WEanNFtMXYy0cIBZxAFG9SrfNMNIUx9A + - GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile offline_access roles + - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email + - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=username + - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name + - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth + - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token + - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'grafanaadmin') && 'GrafanaAdmin' || contains(roles[*], 'admin') && 'Admin' || contains(roles[*], 'editor') && 'Editor' || 'Viewer' + - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true entrypoint: - sh - -euc From d1b94d8bc71977467ddccb5551fd4eb606041748 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Wed, 7 Aug 2024 16:23:26 -0400 Subject: [PATCH 12/75] make grafana keycloak adjustment automatic --- etc/bento.env | 2 ++ etc/bento_deploy.env | 5 +++ etc/bento_dev.env | 5 +++ lib/gateway/services/grafana.conf.tpl | 7 +++-- lib/logs/docker-compose.logs.yaml | 10 +++--- py_bentoctl/auth_helper.py | 44 +++++++++++++++++++++++++++ 6 files changed, 67 insertions(+), 6 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 0b04c342..ce27673b 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -442,6 +442,8 @@ BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/ses # Monitoring BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net +BENTOV2_PRIVATE_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/grafana +BENTOV2_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana BENTOV2_LOKI_BASE_IMAGE=grafana/loki BENTOV2_LOKI_BASE_IMAGE_VERSION=3.0.0 BENTOV2_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 86a5c98a..260a06ae 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -11,6 +11,7 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='false' # Set to true if using Beacon! BENTO_BEACON_UI_ENABLED='false' BENTO_CBIOPORTAL_ENABLED='false' +BENTO_MONITORING_ENABLED='true' BENTO_GOHAN_ENABLED='true' # - Switch to enable French translation in Bento Public @@ -53,6 +54,10 @@ BENTO_AUTHZ_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! + +# - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_GRAFANA_CLIENT_ID=grafana +BENTO_GRAFANA_CLIENT_SECRET= # --------------------------------------------------------------------- BENTO_WEB_CUSTOM_HEADER= diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 884d1d65..2dd13dd8 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -11,6 +11,7 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' +BENTO_MONITORING_ENABLED='true' BENTO_GOHAN_ENABLED='true' # - Switch to enable French translation in Bento Public @@ -53,6 +54,10 @@ BENTOV2_AUTH_TEST_PASSWORD= # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= + +# - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_GRAFANA_CLIENT_ID=grafana +BENTO_GRAFANA_CLIENT_SECRET= # -------------------------------------------------------------------- # Gohan diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl index 6bc85817..03c84e5d 100644 --- a/lib/gateway/services/grafana.conf.tpl +++ b/lib/gateway/services/grafana.conf.tpl @@ -3,7 +3,10 @@ location /api/grafana/ { # Reverse proxy settings include /gateway/conf/proxy.conf; include /gateway/conf/proxy_extra.conf; - - proxy_pass http://bentov2-grafana:3000; + + # Immediate set/re-use means we don't get resolve errors if not up (as opposed to passing as a literal) + set $upstream_grafana http://bentov2-grafana:3000; + + proxy_pass $upstream_grafana; error_log /var/log/bentov2_grafana_errors.log; } diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 2b186801..c14f71e3 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -4,23 +4,25 @@ services: container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_SERVER_ROOT_URL=https://${BENTOV2_PORTAL_DOMAIN}/api/grafana + - GF_SERVER_ROOT_URL=${BENTOV2_PRIVATE_GRAFANA_URL} - GF_SERVER_SERVE_FROM_SUB_PATH=true + - GF_SECURITY_COOKIE_SAMESITE=none - GF_SECURITY_ALLOW_EMBEDDING=true - GF_LOG_LEVEL=debug - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true - - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana-oauth - - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=WEanNFtMXYy0cIBZxAFG9SrfNMNIUx9A + - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} + - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET} - GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile offline_access roles - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=username - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name + - GF_AUTH_GENERIC_OAUTH_USE_PKCE=true - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo - - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'grafanaadmin') && 'GrafanaAdmin' || contains(roles[*], 'admin') && 'Admin' || contains(roles[*], 'editor') && 'Editor' || 'Viewer' + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='Admin' - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true entrypoint: - sh diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 9d9b2437..916a81fe 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -41,6 +41,9 @@ WES_CLIENT_ID = os.getenv("BENTO_WES_CLIENT_ID") WES_WORKFLOW_TIMEOUT = int(os.getenv("BENTOV2_WES_WORKFLOW_TIMEOUT")) +GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID") +GRAFANA_PRIVATE_URL=os.getenv("BENTOV2_PRIVATE_GRAFANA_URL") + KC_CLIENTS_ENDPOINT = f"admin/realms/{AUTH_REALM}/clients" @@ -226,6 +229,42 @@ def create_web_client_if_needed(token: str) -> None: use_refresh_tokens=True, ) + def create_grafana_client_if_needed(token: str) -> None: + grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) + + if grafana_client_kc_id is None: + # Create the Bento WES client + create_keycloak_client_or_exit( + token, + GRAFANA_CLIENT_ID, + standard_flow_enabled=True, + service_accounts_enabled=False, + public_client=False, # Use client secret for this one + redirect_uris=[ + f"{GRAFANA_PRIVATE_URL}/*" + ], + web_origins=[GRAFANA_PRIVATE_URL], + access_token_lifespan=900, # default access token lifespan: 15 minutes + use_refresh_tokens=False, + ) + grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) + + # Fetch and print secret + + client_secret_res = get_keycloak_client_secret(grafana_client_kc_id, token) + + client_secret_data = client_secret_res.json() + if not client_secret_res.ok: + err(f" Failed to get client secret for {GRAFANA_CLIENT_ID}; {client_secret_res.status_code} " + f"{client_secret_data}") + exit(1) + + client_secret = client_secret_data["value"] + cprint( + f" Please set BENTO_GRAFANA_CLIENT_SECRET to {client_secret} in local.env and restart Grafana", + attrs=["bold"], + ) + # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) @@ -376,6 +415,11 @@ def success(): create_wes_client_if_needed(access_token) success() + if c.BENTO_FEATURE_MONITORING.enabled: + info(f" Creating Grafana client: {GRAFANA_CLIENT_ID}") + create_grafana_client_if_needed(access_token) + success() + info(f" Creating user: {AUTH_TEST_USER}") create_test_user_if_needed(access_token) success() From 04091d561897001f1f513447daa1ddbe22445685 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Wed, 7 Aug 2024 16:32:33 -0400 Subject: [PATCH 13/75] fix whitespace --- py_bentoctl/auth_helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 916a81fe..f79129e7 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -42,7 +42,7 @@ WES_WORKFLOW_TIMEOUT = int(os.getenv("BENTOV2_WES_WORKFLOW_TIMEOUT")) GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID") -GRAFANA_PRIVATE_URL=os.getenv("BENTOV2_PRIVATE_GRAFANA_URL") +GRAFANA_PRIVATE_URL = os.getenv("BENTOV2_PRIVATE_GRAFANA_URL") KC_CLIENTS_ENDPOINT = f"admin/realms/{AUTH_REALM}/clients" @@ -242,10 +242,10 @@ def create_grafana_client_if_needed(token: str) -> None: public_client=False, # Use client secret for this one redirect_uris=[ f"{GRAFANA_PRIVATE_URL}/*" - ], - web_origins=[GRAFANA_PRIVATE_URL], + ], + web_origins=[GRAFANA_PRIVATE_URL], access_token_lifespan=900, # default access token lifespan: 15 minutes - use_refresh_tokens=False, + use_refresh_tokens=False, ) grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) From aa8ff2860a5d72c1e6b6c3a999873f268f22e6b1 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Thu, 8 Aug 2024 13:27:54 -0400 Subject: [PATCH 14/75] remove default admin account and clean up config --- lib/logs/docker-compose.logs.yaml | 3 ++- lib/logs/promtail-config.yaml | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index c14f71e3..83517781 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -9,6 +9,7 @@ services: - GF_SECURITY_COOKIE_SAMESITE=none - GF_SECURITY_ALLOW_EMBEDDING=true - GF_LOG_LEVEL=debug + - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true @@ -22,7 +23,7 @@ services: - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo - - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='Admin' + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='GrafanaAdmin' - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true entrypoint: - sh diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml index fb594f44..9eae536d 100644 --- a/lib/logs/promtail-config.yaml +++ b/lib/logs/promtail-config.yaml @@ -18,9 +18,6 @@ scrape_configs: docker_sd_configs: - host: unix:///var/run/docker.sock refresh_interval: 5s - filters: - #- name: name - # values: [flog] relabel_configs: - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' From e4bed4720521f89a6b991789864e805da12ca6c8 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 11:01:02 -0400 Subject: [PATCH 15/75] make requested changes --- docker-compose.dev.yaml | 9 ++++--- docker-compose.prod.yaml | 2 +- docker-compose.yaml | 2 +- etc/bento.env | 26 ++++++++++---------- lib/gateway/docker-compose.gateway.yaml | 1 - lib/logs/docker-compose.logs.yaml | 32 +++++++++++-------------- lib/logs/promtail-config.yaml | 2 +- 7 files changed, 36 insertions(+), 38 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index f4800ee2..8e23f5af 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -234,14 +234,17 @@ services: "http.cors.allow-origin": "http://localhost:8081" "http.cors.allow-headers": X-Requested-With,Content-Type,Content-Length,Authorization - cbioportal: ports: - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" grafana: + ports: + - "3000:3000" environment: # Workaround for self signed certificates in dev - GF_AUTH_GENERIC_OAUTH_TLS_SKIP_VERIFY_INSECURE=true - - + + loki: + ports: + - "3100:3100" diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index a8e656cb..547a3d0f 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -3,4 +3,4 @@ services: environment: - NODE_ENV=production volumes: - - $PWD/lib/web/branding.png:/web/dist/static/branding.png:ro \ No newline at end of file + - $PWD/lib/web/branding.png:/web/dist/static/branding.png:ro diff --git a/docker-compose.yaml b/docker-compose.yaml index f00eae74..f91aaa52 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,4 +21,4 @@ include: - lib/service-registry/docker-compose.service-registry.yaml - lib/web/docker-compose.web.yaml - lib/wes/docker-compose.wes.yaml - project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory \ No newline at end of file + project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory diff --git a/etc/bento.env b/etc/bento.env index ce27673b..748de69a 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -442,16 +442,16 @@ BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/ses # Monitoring BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net -BENTOV2_PRIVATE_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/grafana -BENTOV2_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana -BENTOV2_LOKI_BASE_IMAGE=grafana/loki -BENTOV2_LOKI_BASE_IMAGE_VERSION=3.0.0 -BENTOV2_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki -BENTOV2_LOKI_PROD_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp -BENTOV2_GRAFANA_BASE_IMAGE=grafana/grafana -BENTOV2_GRAFANA_BASE_IMAGE_VERSION=11.1.1-ubuntu -BENTOV2_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana -BENTOV2_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib -BENTOV2_PROMTAIL_BASE_IMAGE=grafana/promtail -BENTOV2_PROMTAIL_BASE_IMAGE_VERSION=2.9.2 -BENTOV2_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail \ No newline at end of file +BENTO_PRIVATE_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/grafana +BENTO_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana +BENTO_LOKI_IMAGE=grafana/loki +BENTO_LOKI_IMAGE_VERSION=3.0.0 +BENTO_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki +BENTO_LOKI_PROD_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp +BENTO_GRAFANA_IMAGE=grafana/grafana +BENTO_GRAFANA_IMAGE_VERSION=11.1.1-ubuntu +BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana +BENTO_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib +BENTO_PROMTAIL_IMAGE=grafana/promtail +BENTO_PROMTAIL_IMAGE_VERSION=2.9.2 +BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail \ No newline at end of file diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 57582494..85533d3d 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -105,7 +105,6 @@ services: - ${BENTOV2_GATEWAY_CERTS_DIR}:${BENTOV2_GATEWAY_INTERNAL_CERTS_DIR}:ro - ${PWD}/lib/gateway/services:/gateway/services:ro - ${PWD}/lib/gateway/public_services:/gateway/public_services:ro - - /var/log:/var/log mem_limit: ${BENTOV2_GATEWAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 cpus: ${BENTOV2_GATEWAY_CPUS} cpu_shares: 512 diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 83517781..3a4d263f 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -1,10 +1,10 @@ services: grafana: - image: ${BENTOV2_GRAFANA_BASE_IMAGE}:${BENTOV2_GRAFANA_BASE_IMAGE_VERSION} - container_name: ${BENTOV2_GRAFANA_CONTAINER_NAME} + image: ${BENTO_GRAFANA_IMAGE}:${BENTO_GRAFANA_IMAGE_VERSION} + container_name: ${BENTO_GRAFANA_CONTAINER_NAME} environment: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_SERVER_ROOT_URL=${BENTOV2_PRIVATE_GRAFANA_URL} + - GF_SERVER_ROOT_URL=${BENTO_PRIVATE_GRAFANA_URL} - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_COOKIE_SAMESITE=none - GF_SECURITY_ALLOW_EMBEDDING=true @@ -47,27 +47,26 @@ services: user: ${BENTO_UID} volumes: - - ${BENTOV2_GRAFANA_PROD_LIB_DIR}:/var/lib/grafana + - ${BENTO_GRAFANA_PROD_LIB_DIR}:/var/lib/grafana + expose: + - 3000 healthcheck: test: [ "CMD", "curl", "-k", "https://localhost:3000"] timeout: 5s interval: 15s - ports: - - "3000:3000" profiles: - monitoring networks: - monitoring-net - - auth-net loki: - container_name: ${BENTOV2_LOKI_CONTAINER_NAME} - image: ${BENTOV2_LOKI_BASE_IMAGE}:${BENTOV2_LOKI_BASE_IMAGE_VERSION} + container_name: ${BENTO_LOKI_CONTAINER_NAME} + image: ${BENTO_LOKI_IMAGE}:${BENTO_LOKI_IMAGE_VERSION} volumes: - - ${BENTOV2_LOKI_PROD_TEMP_DIR}:/tmp/loki + - ${BENTO_LOKI_PROD_TEMP_DIR}:/tmp/loki - ${PWD}/lib/logs/loki-config.yaml:/etc/loki/loki-config.yaml - ports: - - "3100:3100" + expose: + - 3100 command: -config.file=/etc/loki/loki-config.yaml user: ${BENTO_UID} @@ -77,8 +76,8 @@ services: - monitoring promtail: - container_name: ${BENTOV2_PROMTAIL_CONTAINER_NAME} - image: ${BENTOV2_PROMTAIL_BASE_IMAGE}:${BENTOV2_PROMTAIL_BASE_IMAGE_VERSION} + container_name: ${BENTO_PROMTAIL_CONTAINER_NAME} + image: ${BENTO_PROMTAIL_IMAGE}:${BENTO_PROMTAIL_IMAGE_VERSION} volumes: - "/var/lib/docker/containers:/var/lib/docker/containers" - "/var/run/docker.sock:/var/run/docker.sock" @@ -90,9 +89,6 @@ services: - monitoring networks: - auth-net: - external: true - name: ${BENTO_AUTH_NETWORK} monitoring-net: external: true - name: ${BENTO_MONITORING_NETWORK} \ No newline at end of file + name: ${BENTO_MONITORING_NETWORK} diff --git a/lib/logs/promtail-config.yaml b/lib/logs/promtail-config.yaml index 9eae536d..fe6d644c 100644 --- a/lib/logs/promtail-config.yaml +++ b/lib/logs/promtail-config.yaml @@ -21,4 +21,4 @@ scrape_configs: relabel_configs: - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' - target_label: 'container' \ No newline at end of file + target_label: 'container' From 83defa9bee9ce6a6604fc8a58137fecf205abbf1 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 12 Aug 2024 12:21:47 -0400 Subject: [PATCH 16/75] chore: update dependencies in requirements.txt --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 50b093b2..f6aec9b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ certifi==2024.7.4 cffi==1.16.0 charset-normalizer==3.3.2 -cryptography==42.0.8 -debugpy==1.8.2 +cryptography==43.0.0 +debugpy==1.8.5 docker==7.1.0 -flake8==7.1.0 +flake8==7.1.1 idna==3.7 mccabe==0.7.0 packaging==24.1 @@ -12,9 +12,9 @@ pycodestyle==2.12.0 pycparser==2.22 pyflakes==3.2.0 pyhumps==3.8.0 -PyYAML==6.0.1 +PyYAML==6.0.2 requests==2.32.3 termcolor==2.4.0 -tqdm==4.66.4 +tqdm==4.66.5 urllib3==2.2.2 websocket-client==1.8.0 From c17742f2f4ac1916de75e6971b8bc568417f9b5d Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 12 Aug 2024 12:26:07 -0400 Subject: [PATCH 17/75] chore: update WES to 0.14.3 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 7ec69b2f..d6754876 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -201,7 +201,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=0.14.2 +BENTOV2_WES_VERSION=0.14.3 BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net From 4af80ff196b3750133ed9dc4ed4346c74b51e036 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 12 Aug 2024 12:44:22 -0400 Subject: [PATCH 18/75] chore: bump reference to 0.2.3 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index d6754876..9c0f84ad 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -179,7 +179,7 @@ BENTOV2_EVENT_RELAY_CPUS=1 # Reference # - Service BENTO_REFERENCE_IMAGE=ghcr.io/bento-platform/bento_reference_service -BENTO_REFERENCE_VERSION=0.2.2 +BENTO_REFERENCE_VERSION=0.2.3 BENTO_REFERENCE_VERSION_DEV=${BENTO_REFERENCE_VERSION}-dev BENTO_REFERENCE_CONTAINER_NAME=${BENTOV2_PREFIX}-reference BENTO_REFERENCE_NETWORK=${BENTOV2_PREFIX}-reference-net From b0159bcc1d770ae9eeb2a6949b0815ccc13917d3 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 14:15:12 -0400 Subject: [PATCH 19/75] more fixes --- etc/bento.env | 8 ++++---- lib/gateway/services/grafana.conf.tpl | 2 +- lib/gateway/services/loki.conf.tpl | 2 +- py_bentoctl/auth_helper.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 748de69a..648e3a44 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -445,13 +445,13 @@ BENTO_MONITORING_NETWORK=${BENTOV2_PREFIX}-monitoring-net BENTO_PRIVATE_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/grafana BENTO_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana BENTO_LOKI_IMAGE=grafana/loki -BENTO_LOKI_IMAGE_VERSION=3.0.0 +BENTO_LOKI_IMAGE_VERSION=3.1.1 BENTO_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki BENTO_LOKI_PROD_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp BENTO_GRAFANA_IMAGE=grafana/grafana -BENTO_GRAFANA_IMAGE_VERSION=11.1.1-ubuntu +BENTO_GRAFANA_IMAGE_VERSION=11.1.3 BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana BENTO_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib BENTO_PROMTAIL_IMAGE=grafana/promtail -BENTO_PROMTAIL_IMAGE_VERSION=2.9.2 -BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail \ No newline at end of file +BENTO_PROMTAIL_IMAGE_VERSION=2.9.10 +BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl index 03c84e5d..3cf777e6 100644 --- a/lib/gateway/services/grafana.conf.tpl +++ b/lib/gateway/services/grafana.conf.tpl @@ -5,7 +5,7 @@ location /api/grafana/ { include /gateway/conf/proxy_extra.conf; # Immediate set/re-use means we don't get resolve errors if not up (as opposed to passing as a literal) - set $upstream_grafana http://bentov2-grafana:3000; + set $upstream_grafana http://${BENTO_GRAFANA_CONTAINER_NAME}:3000; proxy_pass $upstream_grafana; error_log /var/log/bentov2_grafana_errors.log; diff --git a/lib/gateway/services/loki.conf.tpl b/lib/gateway/services/loki.conf.tpl index 3330c6b6..0cb1cd81 100644 --- a/lib/gateway/services/loki.conf.tpl +++ b/lib/gateway/services/loki.conf.tpl @@ -9,7 +9,7 @@ location /api/loki/ { rewrite ^ $request_uri; rewrite ^/api/loki/(.*) /$1 break; return 400; - proxy_pass http://bentov2-loki:3100/loki/api/v1$uri; + proxy_pass http://${BENTO_LOKI_CONTAINER_NAME}:3100/loki/api/v1$uri; # Errors error_log /var/log/bentov2_loki_errors.log; diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index f79129e7..90211a8e 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -42,7 +42,7 @@ WES_WORKFLOW_TIMEOUT = int(os.getenv("BENTOV2_WES_WORKFLOW_TIMEOUT")) GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID") -GRAFANA_PRIVATE_URL = os.getenv("BENTOV2_PRIVATE_GRAFANA_URL") +GRAFANA_PRIVATE_URL = os.getenv("BENTO_PRIVATE_GRAFANA_URL") KC_CLIENTS_ENDPOINT = f"admin/realms/{AUTH_REALM}/clients" From f2fca0cef9e0fdc986cd86c1610008380aae30d1 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 15:08:17 -0400 Subject: [PATCH 20/75] more small fixes (bento grafana env variable) --- etc/bento_deploy.env | 2 +- etc/bento_dev.env | 2 +- lib/gateway/docker-compose.gateway.yaml | 1 + lib/gateway/services/grafana.conf.tpl | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 260a06ae..8d0be71d 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -11,7 +11,7 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='false' # Set to true if using Beacon! BENTO_BEACON_UI_ENABLED='false' BENTO_CBIOPORTAL_ENABLED='false' -BENTO_MONITORING_ENABLED='true' +BENTO_MONITORING_ENABLED='false' BENTO_GOHAN_ENABLED='true' # - Switch to enable French translation in Bento Public diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 2dd13dd8..9e471dc2 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -11,7 +11,7 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' -BENTO_MONITORING_ENABLED='true' +BENTO_MONITORING_ENABLED='false' BENTO_GOHAN_ENABLED='true' # - Switch to enable French translation in Bento Public diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 85533d3d..fe1594ca 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -76,6 +76,7 @@ services: - BENTO_BEACON_INTERNAL_PORT - BENTO_CBIOPORTAL_CONTAINER_NAME - BENTO_CBIOPORTAL_INTERNAL_PORT + - BENTO_GRAFANA_CONTAINER_NAME networks: - aggregation-net - auth-net diff --git a/lib/gateway/services/grafana.conf.tpl b/lib/gateway/services/grafana.conf.tpl index 3cf777e6..214968a3 100644 --- a/lib/gateway/services/grafana.conf.tpl +++ b/lib/gateway/services/grafana.conf.tpl @@ -1,3 +1,4 @@ +# env: BENTO_MONITORING_ENABLED location /api/grafana { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/grafana/; } location /api/grafana/ { # Reverse proxy settings From 99ef0450e0e174861419e98aa1a68ceb42e1f935 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 12 Aug 2024 15:49:26 -0400 Subject: [PATCH 21/75] chore: rm proxy-controlled CORS from Katsu --- lib/gateway/services/katsu.conf.tpl | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/gateway/services/katsu.conf.tpl b/lib/gateway/services/katsu.conf.tpl index 81e47553..6e265483 100644 --- a/lib/gateway/services/katsu.conf.tpl +++ b/lib/gateway/services/katsu.conf.tpl @@ -11,9 +11,6 @@ location /api/metadata/ { return 400; proxy_pass http://${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_INTERNAL_PORT}$uri; - # CORS - include /usr/local/openresty/nginx/conf/cors.conf; - # Errors error_log /var/log/bentov2_metadata_errors.log; } From 17cdebc6daf9a47c4bc6e48cd8e3c9919af37bc5 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 12 Aug 2024 15:49:37 -0400 Subject: [PATCH 22/75] set katsu to pr-529 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 9c0f84ad..114ccb85 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -269,7 +269,7 @@ BENTOV2_KATSU_DB_CPUS=4 # Katsu BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu -BENTOV2_KATSU_VERSION=8.0.1 +BENTOV2_KATSU_VERSION=pr-529 BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net From ee0a83d3ad5c9cdb0a2a5c1eda4988b602cf63c8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 12 Aug 2024 15:50:34 -0400 Subject: [PATCH 23/75] configure Katsu for CORS + authz --- lib/katsu/docker-compose.katsu.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/katsu/docker-compose.katsu.yaml b/lib/katsu/docker-compose.katsu.yaml index e8da7e3f..9a5bfeca 100644 --- a/lib/katsu/docker-compose.katsu.yaml +++ b/lib/katsu/docker-compose.katsu.yaml @@ -27,8 +27,12 @@ services: - SERVICE_SECRET_KEY=${BENTOV2_KATSU_APP_SECRET} - DJANGO_SETTINGS_MODULE=chord_metadata_service.metadata.settings - BENTOV2_PORTAL_DOMAIN - # Allow access by container name or localhost for healthchecks: - - KATSU_ALLOWED_HOSTS=${BENTOV2_KATSU_CONTAINER_NAME},localhost + # Allow access by public, container name, or localhost for healthchecks: + - KATSU_ALLOWED_HOSTS=${BENTOV2_DOMAIN},${BENTOV2_KATSU_CONTAINER_NAME},localhost + # Authz + - BENTO_AUTHZ_ENABLED=True + - BENTO_AUTHZ_SERVICE_URL + - CORS_ORIGINS=${BENTO_CORS_ORIGINS} # configs: # - source: chord-metadata-settings # target: /katsu/metadata/settings.py From 2d5500a38c216ed5eacf888c7bd334971b74bbd6 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 16:08:13 -0400 Subject: [PATCH 24/75] fix loki enviornment variable --- lib/gateway/docker-compose.gateway.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index fe1594ca..1a2ca727 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -77,6 +77,7 @@ services: - BENTO_CBIOPORTAL_CONTAINER_NAME - BENTO_CBIOPORTAL_INTERNAL_PORT - BENTO_GRAFANA_CONTAINER_NAME + - BENTO_LOKI_CONTAINER_NAME networks: - aggregation-net - auth-net From b25e54b82651a1f111eb0fd3ad439363f3ec27c4 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 16:15:10 -0400 Subject: [PATCH 25/75] fix monitoring enabled flag --- lib/gateway/docker-compose.gateway.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 1a2ca727..845a8599 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -22,6 +22,7 @@ services: - BENTO_BEACON_ENABLED - BENTO_CBIOPORTAL_ENABLED - BENTO_GOHAN_ENABLED + - BENTO_MONITORING_ENABLED - BENTOV2_GATEWAY_CONTAINER_NAME From 256322549414996fcfd20398c639ad04329b7954 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 16:42:16 -0400 Subject: [PATCH 26/75] add monitoring env to web --- lib/web/docker-compose.web.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/web/docker-compose.web.yaml b/lib/web/docker-compose.web.yaml index 1de72f2f..0865f3be 100644 --- a/lib/web/docker-compose.web.yaml +++ b/lib/web/docker-compose.web.yaml @@ -9,6 +9,7 @@ services: environment: - BENTO_UID - BENTO_CBIOPORTAL_ENABLED + - BENTO_MONITORING_ENABLED - BENTO_CBIOPORTAL_PUBLIC_URL - BENTO_DROP_BOX_FS_BASE_PATH - BENTO_URL=${BENTOV2_PORTAL_PUBLIC_URL} From 254091746431d4c4b9f0e42891b8ea15afcf17cb Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Mon, 12 Aug 2024 17:03:17 -0400 Subject: [PATCH 27/75] remove email requirement --- lib/logs/docker-compose.logs.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 3a4d263f..6f238ca8 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -15,8 +15,7 @@ services: - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET} - - GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile offline_access roles - - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email + - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile offline_access roles - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=username - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name - GF_AUTH_GENERIC_OAUTH_USE_PKCE=true From ef1c4d82cea0ecce1d48c4538652a67b0810ad23 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Wed, 14 Aug 2024 14:45:04 -0400 Subject: [PATCH 28/75] remove unnecessary loki conf --- lib/gateway/services/loki.conf.tpl | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 lib/gateway/services/loki.conf.tpl diff --git a/lib/gateway/services/loki.conf.tpl b/lib/gateway/services/loki.conf.tpl deleted file mode 100644 index 0cb1cd81..00000000 --- a/lib/gateway/services/loki.conf.tpl +++ /dev/null @@ -1,16 +0,0 @@ -location /api/loki { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/loki/; } -location /api/loki/ { - # Reverse proxy settings - include /gateway/conf/proxy.conf; - include /gateway/conf/proxy_extra.conf; - include /gateway/conf/proxy_private.conf; - - # Forward request to the aggregation - rewrite ^ $request_uri; - rewrite ^/api/loki/(.*) /$1 break; - return 400; - proxy_pass http://${BENTO_LOKI_CONTAINER_NAME}:3100/loki/api/v1$uri; - - # Errors - error_log /var/log/bentov2_loki_errors.log; -} From d7eefb2c91f1cee4313835d44c6558191bad6268 Mon Sep 17 00:00:00 2001 From: CowTrainer Date: Wed, 14 Aug 2024 15:41:42 -0400 Subject: [PATCH 29/75] env var changes --- etc/bento.env | 4 ++-- etc/default_config.env | 3 +++ lib/gateway/docker-compose.gateway.yaml | 1 - lib/logs/docker-compose.logs.yaml | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 1977a8e3..2ff92af4 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -448,11 +448,11 @@ BENTO_PUBLIC_GRAFANA_URL=${BENTOV2_PORTAL_PUBLIC_URL}/grafana BENTO_LOKI_IMAGE=grafana/loki BENTO_LOKI_IMAGE_VERSION=3.1.1 BENTO_LOKI_CONTAINER_NAME=${BENTOV2_PREFIX}-loki -BENTO_LOKI_PROD_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp +BENTO_LOKI_TEMP_DIR=${BENTO_SLOW_DATA_DIR}/loki/tmp BENTO_GRAFANA_IMAGE=grafana/grafana BENTO_GRAFANA_IMAGE_VERSION=11.1.3 BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana -BENTO_GRAFANA_PROD_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib +BENTO_GRAFANA_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib BENTO_PROMTAIL_IMAGE=grafana/promtail BENTO_PROMTAIL_IMAGE_VERSION=2.9.10 BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail diff --git a/etc/default_config.env b/etc/default_config.env index 949f7309..3ff8966f 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -81,6 +81,9 @@ BENTO_AUTHZ_DB_PASSWORD= # - cBioPortal Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM BENTO_CBIOPORTAL_CLIENT_ID=cbioportal BENTO_CBIOPORTAL_CLIENT_SECRET= +# - Grafana Client ID/secret; secret to be filled by local.env client within BENTOV2_AUTH_REALM +BENTO_GRAFANA_CLIENT_ID=grafana +BENTO_GRAFANA_CLIENT_SECRET= # - WES Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 845a8599..5ccab6b7 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -78,7 +78,6 @@ services: - BENTO_CBIOPORTAL_CONTAINER_NAME - BENTO_CBIOPORTAL_INTERNAL_PORT - BENTO_GRAFANA_CONTAINER_NAME - - BENTO_LOKI_CONTAINER_NAME networks: - aggregation-net - auth-net diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 6f238ca8..f8b3c0bb 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -46,7 +46,7 @@ services: user: ${BENTO_UID} volumes: - - ${BENTO_GRAFANA_PROD_LIB_DIR}:/var/lib/grafana + - ${BENTO_GRAFANA_LIB_DIR}:/var/lib/grafana expose: - 3000 healthcheck: @@ -62,7 +62,7 @@ services: container_name: ${BENTO_LOKI_CONTAINER_NAME} image: ${BENTO_LOKI_IMAGE}:${BENTO_LOKI_IMAGE_VERSION} volumes: - - ${BENTO_LOKI_PROD_TEMP_DIR}:/tmp/loki + - ${BENTO_LOKI_TEMP_DIR}:/tmp/loki - ${PWD}/lib/logs/loki-config.yaml:/etc/loki/loki-config.yaml expose: - 3100 From 062bcb586f4d26322f0884db9c90d0cfe0f48797 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 14 Aug 2024 15:44:37 -0400 Subject: [PATCH 30/75] set public to pr-173 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 114ccb85..5e12951f 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=0.19.1 +BENTO_PUBLIC_VERSION=pr-173 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 0a45ee590f7928411441d5f6cb29c91089f1b4b3 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 15 Aug 2024 08:07:10 -0400 Subject: [PATCH 31/75] chore(deps): update requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f6aec9b5..f58be5b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ certifi==2024.7.4 -cffi==1.16.0 +cffi==1.17.0 charset-normalizer==3.3.2 cryptography==43.0.0 debugpy==1.8.5 @@ -8,7 +8,7 @@ flake8==7.1.1 idna==3.7 mccabe==0.7.0 packaging==24.1 -pycodestyle==2.12.0 +pycodestyle==2.12.1 pycparser==2.22 pyflakes==3.2.0 pyhumps==3.8.0 From 3f2f667d85267eb5369b96cdfcab1ba8a65dcfcc Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 15 Aug 2024 08:07:32 -0400 Subject: [PATCH 32/75] refact(bentoctl): factor out common client/secret creation code --- py_bentoctl/auth_helper.py | 154 ++++++++++++++----------------------- 1 file changed, 58 insertions(+), 96 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 90211a8e..857e63b6 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -160,6 +160,52 @@ def get_keycloak_client_secret(client_id: str, token: str): return keycloak_req(f"{KC_CLIENTS_ENDPOINT}/{client_id}/client-secret", bearer_token=token) +def create_client_and_secret_for_service( + client_id: str, + env_var_to_set: str, + private_url: str | None, + token: str, + is_service_account: bool = False, + to_restart: str = "the gateway", + token_lifespan: int = 900, # default access token lifespan: 15 minutes + use_refresh_tokens: bool = False, # by default, don't use refresh tokens! (they're less secure) +): + client_kc_id: Optional[str] = fetch_existing_client_id(token, client_id) + + if client_kc_id is None: + # Create the Bento WES client + create_keycloak_client_or_exit( + token, + client_id, + standard_flow_enabled=not is_service_account, + service_accounts_enabled=is_service_account, + public_client=False, + redirect_uris=[ + f"{private_url}/*" + ] if not is_service_account else [], # Not used for client credentials access + web_origins=[private_url] if not is_service_account else [], # " + access_token_lifespan=token_lifespan, + use_refresh_tokens=use_refresh_tokens, + ) + client_kc_id = fetch_existing_client_id(token, client_id) + + # Fetch and print secret + + client_secret_res = get_keycloak_client_secret(client_kc_id, token) + + client_secret_data = client_secret_res.json() + if not client_secret_res.ok: + err(f" Failed to get client secret for {client_id}; {client_secret_res.status_code} " + f"{client_secret_data}") + exit(1) + + client_secret = client_secret_data["value"] + cprint( + f" Please set {env_var_to_set} to {client_secret} in local.env and restart {to_restart}", + attrs=["bold"], + ) + + def init_auth(docker_client: docker.DockerClient): check_auth_admin_user() @@ -230,109 +276,25 @@ def create_web_client_if_needed(token: str) -> None: ) def create_grafana_client_if_needed(token: str) -> None: - grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) - - if grafana_client_kc_id is None: - # Create the Bento WES client - create_keycloak_client_or_exit( - token, - GRAFANA_CLIENT_ID, - standard_flow_enabled=True, - service_accounts_enabled=False, - public_client=False, # Use client secret for this one - redirect_uris=[ - f"{GRAFANA_PRIVATE_URL}/*" - ], - web_origins=[GRAFANA_PRIVATE_URL], - access_token_lifespan=900, # default access token lifespan: 15 minutes - use_refresh_tokens=False, - ) - grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) - - # Fetch and print secret - - client_secret_res = get_keycloak_client_secret(grafana_client_kc_id, token) - - client_secret_data = client_secret_res.json() - if not client_secret_res.ok: - err(f" Failed to get client secret for {GRAFANA_CLIENT_ID}; {client_secret_res.status_code} " - f"{client_secret_data}") - exit(1) - - client_secret = client_secret_data["value"] - cprint( - f" Please set BENTO_GRAFANA_CLIENT_SECRET to {client_secret} in local.env and restart Grafana", - attrs=["bold"], + create_client_and_secret_for_service( + GRAFANA_CLIENT_ID, "BENTO_GRAFANA_CLIENT_SECRET", GRAFANA_PRIVATE_URL, token, to_restart="Grafana" ) # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: - cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) - - if cbio_client_kc_id is None: - # Create the cBioportal client - create_keycloak_client_or_exit( - token, - CBIOPORTAL_CLIENT_ID, - standard_flow_enabled=True, - service_accounts_enabled=False, - public_client=False, - redirect_uris=[f"{CBIOPORTAL_URL}{AUTH_LOGIN_REDIRECT_PATH}"], - web_origins=[CBIOPORTAL_URL], - access_token_lifespan=900, # 15 minutes - use_refresh_tokens=True, - ) - cbio_client_kc_id = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) - - # Fetch and print secret - - client_secret_res = get_keycloak_client_secret(cbio_client_kc_id, token) - - client_secret_data = client_secret_res.json() - if not client_secret_res.ok: - err(f" Failed to get client secret for {CBIOPORTAL_CLIENT_ID}; {client_secret_res.status_code} " - f"{client_secret_data}") - exit(1) - - client_secret = client_secret_data["value"] - cprint( - f" Please set BENTO_CBIOPORTAL_CLIENT_SECRET to {client_secret} in local.env and restart the " - f"gateway", - attrs=["bold"], + create_client_and_secret_for_service( + GRAFANA_CLIENT_ID, "BENTO_CBIOPORTAL_CLIENT_SECRET", CBIOPORTAL_URL, token, use_refresh_tokens=True ) def create_wes_client_if_needed(token: str) -> None: - wes_client_kc_id: Optional[str] = fetch_existing_client_id(token, WES_CLIENT_ID) - - if wes_client_kc_id is None: - # Create the Bento WES client - create_keycloak_client_or_exit( - token, - WES_CLIENT_ID, - standard_flow_enabled=False, - service_accounts_enabled=True, - public_client=False, # Use client secret for this one - redirect_uris=[], # Not used with a standard/web flow - just client credentials - web_origins=[], # " - access_token_lifespan=WES_WORKFLOW_TIMEOUT, # WES workflow lifespan - use_refresh_tokens=False, # No refreshing these allowed! - ) - wes_client_kc_id = fetch_existing_client_id(token, WES_CLIENT_ID) - - # Fetch and print secret - - client_secret_res = get_keycloak_client_secret(wes_client_kc_id, token) - - client_secret_data = client_secret_res.json() - if not client_secret_res.ok: - err(f" Failed to get client secret for {WES_CLIENT_ID}; {client_secret_res.status_code} " - f"{client_secret_data}") - exit(1) - - client_secret = client_secret_data["value"] - cprint( - f" Please set BENTO_WES_CLIENT_SECRET to {client_secret} in local.env and restart WES", - attrs=["bold"], + create_client_and_secret_for_service( + WES_CLIENT_ID, + "BENTO_WES_CLIENT_SECRET", + None, + token, + is_service_account=True, + to_restart="WES", + token_lifespan=WES_WORKFLOW_TIMEOUT, ) def create_test_user_if_needed(token: str) -> None: From 8b249ca31b72665ba540dfbf6fb47cad2333fd11 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 19 Aug 2024 14:38:24 -0400 Subject: [PATCH 33/75] fix: allow grafana auth with keycloak users that dont have emails --- lib/logs/docker-compose.logs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index f8b3c0bb..ca99197f 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -23,6 +23,7 @@ services: - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='GrafanaAdmin' + - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH='@.email || @.sub' - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true entrypoint: - sh From 1f9e2aebf051b85169ba8ce43f0e5122c53da866 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 19 Aug 2024 15:30:49 -0400 Subject: [PATCH 34/75] set gateway to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 7a7dfd2d..babf9aa5 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=0.12.0 +BENTOV2_GATEWAY_VERSION=edge BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway From 44a6e47851a06fff02c5a5f23c3ed9c7f23caa46 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 19 Aug 2024 17:13:02 -0400 Subject: [PATCH 35/75] add preferred_username paths --- lib/logs/docker-compose.logs.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index ca99197f..1b20b7a8 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -16,14 +16,15 @@ services: - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET} - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile offline_access roles - - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=username - - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name + - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=preferred_username + - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=preferred_username - GF_AUTH_GENERIC_OAUTH_USE_PKCE=true - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='GrafanaAdmin' - - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH='@.email || @.sub' + # Allows authentication for users that don't have an email + - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email || preferred_username || sub - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true entrypoint: - sh From 772f5de52d6a4fce53fa4d1b5854616a65ad01e5 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Thu, 22 Aug 2024 17:08:27 -0400 Subject: [PATCH 36/75] secure cookie --- lib/logs/docker-compose.logs.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 1b20b7a8..384bfc30 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -7,12 +7,14 @@ services: - GF_SERVER_ROOT_URL=${BENTO_PRIVATE_GRAFANA_URL} - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_COOKIE_SAMESITE=none + - GF_SECURITY_COOKIE_SECURE=true - GF_SECURITY_ALLOW_EMBEDDING=true - GF_LOG_LEVEL=debug - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true + - GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=true - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth - - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true + - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET} - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile offline_access roles From 370bd21fc87ea1d5431ad406d27dc50aac722f5a Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Fri, 23 Aug 2024 13:41:56 -0400 Subject: [PATCH 37/75] feat: grafana role mapping from keycloak client roles --- etc/bento.env | 2 ++ lib/logs/docker-compose.logs.yaml | 15 +++++++++------ py_bentoctl/other_helpers.py | 3 +++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 2ff92af4..0e1f23b6 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -453,6 +453,8 @@ BENTO_GRAFANA_IMAGE=grafana/grafana BENTO_GRAFANA_IMAGE_VERSION=11.1.3 BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana BENTO_GRAFANA_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib +BENTO_GRAFANA_ROLE_ATTRIBUTE_PATH="contains(resource_access.grafana.roles[*], 'admin') && 'Admin' || contains(resource_access.grafana.roles[*], 'editor') && 'Editor' || contains(resource_access.grafana.roles[*], 'viewer') && 'Viewer' || 'None'" +BENTO_GRAFANA_SIGNOUT_REDIRECT_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/${BENTOV2_AUTH_REALM}/protocol/openid-connect/logout?post_logout_redirect_uri=https%3A%2F%2F${BENTOV2_PORTAL_DOMAIN}%2Fapi%2Fgrafana%2Flogin BENTO_PROMTAIL_IMAGE=grafana/promtail BENTO_PROMTAIL_IMAGE_VERSION=2.9.10 BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 384bfc30..ff2784c3 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -9,25 +9,28 @@ services: - GF_SECURITY_COOKIE_SAMESITE=none - GF_SECURITY_COOKIE_SECURE=true - GF_SECURITY_ALLOW_EMBEDDING=true - - GF_LOG_LEVEL=debug - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true - GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=true - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth - - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false + - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${BENTO_GRAFANA_CLIENT_SECRET} - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile offline_access roles - GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=preferred_username - GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=preferred_username - GF_AUTH_GENERIC_OAUTH_USE_PKCE=true - - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/auth - - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/token - - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/bentov2/protocol/openid-connect/userinfo - - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH='GrafanaAdmin' + - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/${BENTOV2_AUTH_REALM}/protocol/openid-connect/auth + - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/${BENTOV2_AUTH_REALM}/protocol/openid-connect/token + - GF_AUTH_GENERIC_OAUTH_API_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/${BENTOV2_AUTH_REALM}/protocol/openid-connect/userinfo + # Role mapping based on Grafana client role membership + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=${BENTO_GRAFANA_ROLE_ATTRIBUTE_PATH} + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_STRICT=true # Allows authentication for users that don't have an email - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email || preferred_username || sub + - GF_AUTH_GENERIC_OAUTH_SIGNOUT_REDIRECT_URL=${BENTO_GRAFANA_SIGNOUT_REDIRECT_URL} - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true + - GF_LOG_LEVEL=debug entrypoint: - sh - -euc diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 9f0e3eeb..dd765222 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -222,6 +222,9 @@ def init_dirs(): **({"auth": "BENTOV2_AUTH_VOL_DIR"} if not c.BENTOV2_USE_EXTERNAL_IDP else {}), # - cBioPortal **({"cbioportal": "BENTO_CBIOPORTAL_STUDY_DIR"} if c.BENTO_FEATURE_CBIOPORTAL.enabled else {}), + # - Monitoring: Grafana/Loki + **({"grafana": "BENTO_GRAFANA_LIB_DIR"} if c.BENTO_FEATURE_MONITORING else {}), + **({"loki": "BENTO_LOKI_TEMP_DIR"} if c.BENTO_FEATURE_MONITORING else {}), } # Some of these don't use the Bento user inside their containers, so ignore if need be From 41a96af471bfe1e14f4c8ad48717d0bd39641ec8 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Fri, 23 Aug 2024 16:45:34 -0400 Subject: [PATCH 38/75] lint --- py_bentoctl/other_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index dd765222..225d43d3 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -222,7 +222,7 @@ def init_dirs(): **({"auth": "BENTOV2_AUTH_VOL_DIR"} if not c.BENTOV2_USE_EXTERNAL_IDP else {}), # - cBioPortal **({"cbioportal": "BENTO_CBIOPORTAL_STUDY_DIR"} if c.BENTO_FEATURE_CBIOPORTAL.enabled else {}), - # - Monitoring: Grafana/Loki + # - Monitoring: Grafana/Loki **({"grafana": "BENTO_GRAFANA_LIB_DIR"} if c.BENTO_FEATURE_MONITORING else {}), **({"loki": "BENTO_LOKI_TEMP_DIR"} if c.BENTO_FEATURE_MONITORING else {}), } From 8eed564ef10ebedbd99dc4f61cca5bedc22b7b63 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 26 Aug 2024 11:17:59 -0400 Subject: [PATCH 39/75] set public to pr-175 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index babf9aa5..c2b2247e 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=pr-173 +BENTO_PUBLIC_VERSION=pr-175 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 748d4312b63526ba65363b3e18795f3f5ef627fa Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 26 Aug 2024 17:47:18 -0400 Subject: [PATCH 40/75] feat(bentoctl): grafana roles and groups creation --- py_bentoctl/auth_helper.py | 93 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 90211a8e..c0a5de68 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -44,7 +44,9 @@ GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID") GRAFANA_PRIVATE_URL = os.getenv("BENTO_PRIVATE_GRAFANA_URL") -KC_CLIENTS_ENDPOINT = f"admin/realms/{AUTH_REALM}/clients" +KC_ADMIN_API_ENDPOINT = f"admin/realms/{AUTH_REALM}" +KC_ADMIN_API_GROUP_ENDPOINT = f"{KC_ADMIN_API_ENDPOINT}/groups" +KC_ADMIN_API_CLIENTS_ENDPOINT = f"{KC_ADMIN_API_ENDPOINT}/clients" def check_auth_admin_user(): @@ -87,7 +89,7 @@ def keycloak_req( def fetch_existing_client_id(token: str, client_id: str) -> Optional[str]: - existing_clients_res = keycloak_req(KC_CLIENTS_ENDPOINT, bearer_token=token) + existing_clients_res = keycloak_req(KC_ADMIN_API_CLIENTS_ENDPOINT, bearer_token=token) existing_clients = existing_clients_res.json() if not existing_clients_res.ok: @@ -102,6 +104,66 @@ def fetch_existing_client_id(token: str, client_id: str) -> Optional[str]: return None +def fetch_existing_group_id(token: str, group_name: str, + parent_id: str | None = None, verbose: bool = True) -> Optional[str]: + endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{parent_id}/children" if parent_id else KC_ADMIN_API_GROUP_ENDPOINT + existing_groups_res = keycloak_req(endpoint, bearer_token=token) + + if not existing_groups_res.ok: + err(f" Failed to fetch group id associated with name: {group_name}") + exit(1) + + exising_groups = existing_groups_res.json() + for group in exising_groups: + if group["name"] == group_name: + if verbose: + warn(f" Found existing group: {group_name}; using that.") + return group["id"] + + return None + + +def create_client_role_or_exit(token: str, client_id: str, role_name: str) -> None: + client_roles_endpoint = f"{KC_ADMIN_API_CLIENTS_ENDPOINT}/{client_id}/roles" + + # Check if client role exists + existing_role_res = keycloak_req(f"{client_roles_endpoint}/{role_name}", bearer_token=token,) + if existing_role_res.ok: + warn(f" Found existing role: {role_name}; using that.") + return + + # Create client role if needed + res = keycloak_req(client_roles_endpoint, bearer_token=token, method="post", json={ + "clientRole": True, + "name": role_name, + }) + + if not res.ok: + err(f" Failed to create {client_id} client role : {role_name}; {res.status_code} {res.json()}") + exit(1) + + warn(f" Created role: {role_name}.") + + +def create_group_or_exit(token: str, group_representation: dict, parent_id: str = None) -> Optional[str]: + group_endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{parent_id}/children" if parent_id else KC_ADMIN_API_GROUP_ENDPOINT + group_type = f"sub-group of {parent_id}" if parent_id else "group" + + if existing_group_id := fetch_existing_group_id(token, group_representation["name"], parent_id): + return existing_group_id + + res = keycloak_req(f"{group_endpoint}", bearer_token=token, method="post", json=group_representation) + + # group creation doesn't return the ID of the newly created group, extra request is required to obtain it + created_group_id = fetch_existing_group_id(token, group_representation["name"], parent_id, verbose=False) + if not res.ok or not created_group_id: + err(f" Failed to create {group_type}: {group_representation}; {res.status_code}") + exit(1) + + warn(f" Created {group_type}: {group_representation['name']}") + return created_group_id + + def create_keycloak_client_or_exit( token: str, client_id: str, @@ -113,7 +175,7 @@ def create_keycloak_client_or_exit( access_token_lifespan: int, use_refresh_tokens: bool, ) -> None: - res = keycloak_req(KC_CLIENTS_ENDPOINT, bearer_token=token, method="post", json={ + res = keycloak_req(KC_ADMIN_API_CLIENTS_ENDPOINT, bearer_token=token, method="post", json={ "clientId": client_id, "enabled": True, "protocol": "openid-connect", @@ -157,7 +219,7 @@ def create_keycloak_client_or_exit( def get_keycloak_client_secret(client_id: str, token: str): - return keycloak_req(f"{KC_CLIENTS_ENDPOINT}/{client_id}/client-secret", bearer_token=token) + return keycloak_req(f"{KC_ADMIN_API_CLIENTS_ENDPOINT}/{client_id}/client-secret", bearer_token=token) def init_auth(docker_client: docker.DockerClient): @@ -265,6 +327,25 @@ def create_grafana_client_if_needed(token: str) -> None: attrs=["bold"], ) + def create_grafana_roles_if_needed(token: str) -> None: + grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) + + for role_name in ["admin", "editor", "viewer"]: + create_client_role_or_exit(token, grafana_client_kc_id, role_name) + + def create_grafana_groups_if_needed(token: str) -> None: + parent_group = {"name": "grafana"} + parent_group_id = create_group_or_exit(token, parent_group) + + sub_groups = [ + {"name": "admin"}, + {"name": "editor"}, + {"name": "viewer"} + ] + + for subgroup in sub_groups: + create_group_or_exit(token, subgroup, parent_id=parent_group_id) + # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) @@ -418,6 +499,10 @@ def success(): if c.BENTO_FEATURE_MONITORING.enabled: info(f" Creating Grafana client: {GRAFANA_CLIENT_ID}") create_grafana_client_if_needed(access_token) + create_grafana_roles_if_needed(access_token) + create_grafana_groups_if_needed(access_token) + # TODO: group client-role mappings + # /admin/realms/{realm}/groups/{group-id}/role-mappings/clients/{client} success() info(f" Creating user: {AUTH_TEST_USER}") From 49322e2f63af6fe61b4ff9db6328c81b0610ed80 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 27 Aug 2024 17:15:11 -0400 Subject: [PATCH 41/75] feat(bentoctl): client role group mappings for grafana --- py_bentoctl/auth_helper.py | 146 +++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 45 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index c0a5de68..f24fd8a9 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -5,6 +5,7 @@ import requests import subprocess import urllib3 +import json from termcolor import cprint from urllib3.exceptions import InsecureRequestWarning @@ -88,7 +89,7 @@ def keycloak_req( raise NotImplementedError -def fetch_existing_client_id(token: str, client_id: str) -> Optional[str]: +def fetch_existing_client_id(token: str, client_id: str, verbose: bool = True) -> Optional[str]: existing_clients_res = keycloak_req(KC_ADMIN_API_CLIENTS_ENDPOINT, bearer_token=token) existing_clients = existing_clients_res.json() @@ -98,70 +99,117 @@ def fetch_existing_client_id(token: str, client_id: str) -> Optional[str]: for client in existing_clients: if client["clientId"] == client_id: - warn(f" Found existing client: {client_id}; using that.") + if verbose: + warn(f" Found existing client: {client_id}; using that.") return client["id"] return None -def fetch_existing_group_id(token: str, group_name: str, - parent_id: str | None = None, verbose: bool = True) -> Optional[str]: - endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{parent_id}/children" if parent_id else KC_ADMIN_API_GROUP_ENDPOINT +def fetch_existing_client_role(token: str, client_id: str, role_name: str, verbose: bool = True) -> Optional[dict]: + client_roles_endpoint = f"{KC_ADMIN_API_CLIENTS_ENDPOINT}/{client_id}/roles" + + existing_role_res = keycloak_req(f"{client_roles_endpoint}/{role_name}", bearer_token=token) + if not existing_role_res.ok: + return + + if verbose: + warn(f" Found existing role: {role_name}; using that.") + return existing_role_res.json() + + +def fetch_existing_group_rep_or_exit( + token: str, + group_name: str, + parent_rep: dict | None = None, + verbose: bool = True + ) -> Optional[dict]: + endpoint = KC_ADMIN_API_GROUP_ENDPOINT + if parent_rep: + endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{parent_rep['id']}/children" existing_groups_res = keycloak_req(endpoint, bearer_token=token) if not existing_groups_res.ok: err(f" Failed to fetch group id associated with name: {group_name}") exit(1) - exising_groups = existing_groups_res.json() - for group in exising_groups: + existing_groups = existing_groups_res.json() + for group in existing_groups: if group["name"] == group_name: if verbose: - warn(f" Found existing group: {group_name}; using that.") - return group["id"] + warn(f" Found existing group: {group['path']} ; using that.") + return group return None -def create_client_role_or_exit(token: str, client_id: str, role_name: str) -> None: - client_roles_endpoint = f"{KC_ADMIN_API_CLIENTS_ENDPOINT}/{client_id}/roles" - - # Check if client role exists - existing_role_res = keycloak_req(f"{client_roles_endpoint}/{role_name}", bearer_token=token,) - if existing_role_res.ok: - warn(f" Found existing role: {role_name}; using that.") - return +def create_client_role_or_exit(token: str, client_id: str, role_name: str) -> Optional[dict]: + # Check if client role alread exists + if existing_role := fetch_existing_client_role(token, client_id, role_name): + return existing_role # Create client role if needed + client_roles_endpoint = f"{KC_ADMIN_API_CLIENTS_ENDPOINT}/{client_id}/roles" res = keycloak_req(client_roles_endpoint, bearer_token=token, method="post", json={ "clientRole": True, "name": role_name, }) if not res.ok: - err(f" Failed to create {client_id} client role : {role_name}; {res.status_code} {res.json()}") + err(f" Failed to create {client_id} client role : {role_name}; {res.status_code}") exit(1) - warn(f" Created role: {role_name}.") + # role creation response returns no data, fetch the created RoleRepresentation for later use + created_role_rep = fetch_existing_client_role(token, client_id, role_name, verbose=False) + warn(f" Created client role: {role_name}.") + return created_role_rep -def create_group_or_exit(token: str, group_representation: dict, parent_id: str = None) -> Optional[str]: - group_endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{parent_id}/children" if parent_id else KC_ADMIN_API_GROUP_ENDPOINT - group_type = f"sub-group of {parent_id}" if parent_id else "group" +def create_group_or_exit(token: str, group_rep: dict, parent_group_rep: dict = None) -> Optional[dict]: + # try to get existing group first + if existing_group := fetch_existing_group_rep_or_exit(token, group_rep["name"], parent_group_rep): + return existing_group - if existing_group_id := fetch_existing_group_id(token, group_representation["name"], parent_id): - return existing_group_id + # group creation endpoint + group_endpoint = KC_ADMIN_API_GROUP_ENDPOINT + if parent_group_rep: + # use sub-group creation endpoint if a parent group is passed + group_endpoint = f"{group_endpoint}/{parent_group_rep['id']}/children" - res = keycloak_req(f"{group_endpoint}", bearer_token=token, method="post", json=group_representation) + res = keycloak_req(f"{group_endpoint}", bearer_token=token, method="post", json=group_rep) + if not res.ok: + err(f" Failed to create group: {group_rep}; {res.status_code}") + exit(1) + + # group creation response returns no data, fetch the created GroupRepresentation for later use + created_group = fetch_existing_group_rep_or_exit(token, group_rep["name"], parent_group_rep, verbose=False) + + warn(f" Created group: {created_group['path']}") + return created_group + + +def add_client_role_mapping_to_group_or_exit(token: str, group_rep: dict, client_id: str, role_rep: dict) -> None: + role_mappings_endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{group_rep['id']}/role-mappings/clients/{client_id}" + existing_mappings_res = keycloak_req(role_mappings_endpoint, bearer_token=token) + if existing_mappings_res.ok: + target_role_name = role_rep["name"] + for role_map in existing_mappings_res.json(): + if target_role_name == role_map["name"]: + warn(f" Found existing client-level group role: {group_rep['path']}; using that.") + return - # group creation doesn't return the ID of the newly created group, extra request is required to obtain it - created_group_id = fetch_existing_group_id(token, group_representation["name"], parent_id, verbose=False) - if not res.ok or not created_group_id: - err(f" Failed to create {group_type}: {group_representation}; {res.status_code}") + # create client role-mapping for given group + client_res = keycloak_req( + role_mappings_endpoint, + method="post", + bearer_token=token, + data=json.dumps([role_rep]) # RoleRepresentation needs to be in an array and sent as data + ) + if not client_res.ok: + err(f" Failed to add client-level role {role_rep['name']} to group {group_rep['path']}") exit(1) - warn(f" Created {group_type}: {group_representation['name']}") - return created_group_id + warn(f" Created client-level role mapping for group: {group_rep['path']}") def create_keycloak_client_or_exit( @@ -291,7 +339,7 @@ def create_web_client_if_needed(token: str) -> None: use_refresh_tokens=True, ) - def create_grafana_client_if_needed(token: str) -> None: + def create_grafana_client_if_needed(token: str) -> Optional[str]: grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) if grafana_client_kc_id is None: @@ -309,7 +357,7 @@ def create_grafana_client_if_needed(token: str) -> None: access_token_lifespan=900, # default access token lifespan: 15 minutes use_refresh_tokens=False, ) - grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) + grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID, verbose=False) # Fetch and print secret @@ -326,16 +374,18 @@ def create_grafana_client_if_needed(token: str) -> None: f" Please set BENTO_GRAFANA_CLIENT_SECRET to {client_secret} in local.env and restart Grafana", attrs=["bold"], ) + return grafana_client_kc_id - def create_grafana_roles_if_needed(token: str) -> None: - grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) - + def create_grafana_client_roles_if_needed(token: str, client_id: str) -> Optional[dict]: + role_representations = {} for role_name in ["admin", "editor", "viewer"]: - create_client_role_or_exit(token, grafana_client_kc_id, role_name) + client_role = create_client_role_or_exit(token, client_id, role_name) + role_representations[role_name] = client_role + return role_representations - def create_grafana_groups_if_needed(token: str) -> None: + def create_grafana_client_groups_if_needed(token: str, role_mappings: dict, client_id: str) -> None: parent_group = {"name": "grafana"} - parent_group_id = create_group_or_exit(token, parent_group) + parent_group = create_group_or_exit(token, parent_group) sub_groups = [ {"name": "admin"}, @@ -343,8 +393,12 @@ def create_grafana_groups_if_needed(token: str) -> None: {"name": "viewer"} ] + # Add client-level role mappings to groups + # grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID, verbose=False) for subgroup in sub_groups: - create_group_or_exit(token, subgroup, parent_id=parent_group_id) + group_rep = create_group_or_exit(token, subgroup, parent_group_rep=parent_group) + role_rep = role_mappings[subgroup["name"]] + add_client_role_mapping_to_group_or_exit(token, group_rep, client_id, role_rep) # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: @@ -498,11 +552,13 @@ def success(): if c.BENTO_FEATURE_MONITORING.enabled: info(f" Creating Grafana client: {GRAFANA_CLIENT_ID}") - create_grafana_client_if_needed(access_token) - create_grafana_roles_if_needed(access_token) - create_grafana_groups_if_needed(access_token) - # TODO: group client-role mappings - # /admin/realms/{realm}/groups/{group-id}/role-mappings/clients/{client} + grafana_client_id = create_grafana_client_if_needed(access_token) + role_mappings = create_grafana_client_roles_if_needed(access_token, grafana_client_id) + create_grafana_client_groups_if_needed(access_token, role_mappings, grafana_client_id) + cprint( + " Add users to the relevant Grafana sub-groups to give them access: admin, editor, viewer", + attrs=["bold"], + ) success() info(f" Creating user: {AUTH_TEST_USER}") From e78cbec1a6d3a067d6dd409a83455dee4b48c236 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 27 Aug 2024 17:19:03 -0400 Subject: [PATCH 42/75] grafana log level info --- lib/logs/docker-compose.logs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index ff2784c3..0a3c71a0 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -30,7 +30,7 @@ services: - GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email || preferred_username || sub - GF_AUTH_GENERIC_OAUTH_SIGNOUT_REDIRECT_URL=${BENTO_GRAFANA_SIGNOUT_REDIRECT_URL} - GF_AUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true - - GF_LOG_LEVEL=debug + - GF_LOG_LEVEL=info entrypoint: - sh - -euc From 26cc6cc1b08b912973fb8206376d1527ce8d95e9 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 27 Aug 2024 17:39:44 -0400 Subject: [PATCH 43/75] comment --- etc/bento.env | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/bento.env b/etc/bento.env index 0e1f23b6..d0c69824 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -453,6 +453,7 @@ BENTO_GRAFANA_IMAGE=grafana/grafana BENTO_GRAFANA_IMAGE_VERSION=11.1.3 BENTO_GRAFANA_CONTAINER_NAME=${BENTOV2_PREFIX}-grafana BENTO_GRAFANA_LIB_DIR=${BENTO_SLOW_DATA_DIR}/grafana/lib +# JMESPath to recover the user's role from the ID token BENTO_GRAFANA_ROLE_ATTRIBUTE_PATH="contains(resource_access.grafana.roles[*], 'admin') && 'Admin' || contains(resource_access.grafana.roles[*], 'editor') && 'Editor' || contains(resource_access.grafana.roles[*], 'viewer') && 'Viewer' || 'None'" BENTO_GRAFANA_SIGNOUT_REDIRECT_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/${BENTOV2_AUTH_REALM}/protocol/openid-connect/logout?post_logout_redirect_uri=https%3A%2F%2F${BENTOV2_PORTAL_DOMAIN}%2Fapi%2Fgrafana%2Flogin BENTO_PROMTAIL_IMAGE=grafana/promtail From 9dcf5be08d6792e30c921cd3308be05474dc2e7f Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Wed, 28 Aug 2024 13:21:12 -0400 Subject: [PATCH 44/75] feat(bentoctl): configure keycloak to include client roles in ID tokens --- py_bentoctl/auth_helper.py | 53 +++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index f24fd8a9..51018baa 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -48,7 +48,7 @@ KC_ADMIN_API_ENDPOINT = f"admin/realms/{AUTH_REALM}" KC_ADMIN_API_GROUP_ENDPOINT = f"{KC_ADMIN_API_ENDPOINT}/groups" KC_ADMIN_API_CLIENTS_ENDPOINT = f"{KC_ADMIN_API_ENDPOINT}/clients" - +KC_ADMIN_API_CLIENT_SCOPES = f"{KC_ADMIN_API_ENDPOINT}/client-scopes" def check_auth_admin_user(): if not AUTH_ADMIN_USER: @@ -85,6 +85,12 @@ def keycloak_req( **(dict(data=data) if data else {}), **(dict(json=json) if json else {}), **kwargs) + if method == "put": + return requests.put( + make_keycloak_url(path), + **(dict(data=data) if data else {}), + **(dict(json=json) if json else {}), + **kwargs) raise NotImplementedError @@ -400,6 +406,50 @@ def create_grafana_client_groups_if_needed(token: str, role_mappings: dict, clie role_rep = role_mappings[subgroup["name"]] add_client_role_mapping_to_group_or_exit(token, group_rep, client_id, role_rep) + # Modifies the "roles" client scope mapper, so that client-level roles are included in the ID token + def set_include_client_roles_in_id_tokens(token: str): + # Retrieve the 'roles' client-scope + client_scopes_res = keycloak_req(KC_ADMIN_API_CLIENT_SCOPES, bearer_token=token) + if not client_scopes_res.ok: + err(f" Failed to retrieve client scopes: {client_scopes_res.status_code}") + exit(1) + client_scopes = client_scopes_res.json() + + roles_client_scope = None + for scope in client_scopes: + if "roles" == scope["name"]: + roles_client_scope = scope + break + + if not roles_client_scope: + # 'roles' is a predefined scope, so it should always be there by default + err(" Failed to retrieve the 'roles' client scope.") + exit(1) + + # Find the 'client roles' protocol mapper + roles_mapper = None + for mapper in roles_client_scope["protocolMappers"]: + if "client roles" == mapper["name"]: + roles_mapper = mapper + + if not roles_mapper: + # 'client roles' is a predefined mapper, so it should always be there by default + err(" Failed to retrieve the 'client roles' protocol mapper.") + exit(1) + + # Update mapper config's id.token.claim if needed + if "id.token.claim" not in roles_mapper["config"] or roles_mapper["config"]["id.token.claim"] == "false": + roles_mapper["config"]["id.token.claim"] = "true" + mapper_endpoint = f"{KC_ADMIN_API_CLIENT_SCOPES}/{roles_client_scope['id']}" + \ + f"/protocol-mappers/models/{roles_mapper['id']}" + update_mapper_res = keycloak_req(mapper_endpoint, bearer_token=token, method="put", json=roles_mapper) + if not update_mapper_res.ok: + err(f" Failed to modify 'client roles' mapper: {update_mapper_res.status_code}") + exit(1) + warn(" Updated 'client roles' scope mapper to include roles in the ID token.") + elif roles_mapper["config"]["id.token.claim"] == "true": + warn(" The 'client roles' scope mapper already includes roles in the ID token.") + # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) @@ -555,6 +605,7 @@ def success(): grafana_client_id = create_grafana_client_if_needed(access_token) role_mappings = create_grafana_client_roles_if_needed(access_token, grafana_client_id) create_grafana_client_groups_if_needed(access_token, role_mappings, grafana_client_id) + set_include_client_roles_in_id_tokens(access_token) cprint( " Add users to the relevant Grafana sub-groups to give them access: admin, editor, viewer", attrs=["bold"], From 2dcd039d72153ed4876ebb9f456ae2df5561278a Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Wed, 28 Aug 2024 13:22:13 -0400 Subject: [PATCH 45/75] lint --- py_bentoctl/auth_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 51018baa..dda2273e 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -50,6 +50,7 @@ KC_ADMIN_API_CLIENTS_ENDPOINT = f"{KC_ADMIN_API_ENDPOINT}/clients" KC_ADMIN_API_CLIENT_SCOPES = f"{KC_ADMIN_API_ENDPOINT}/client-scopes" + def check_auth_admin_user(): if not AUTH_ADMIN_USER: err("Missing environment value for BENTOV2_AUTH_ADMIN_USER") From c492445052addcb75ca5279f1ec25dd144bdd212 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Wed, 28 Aug 2024 13:31:59 -0400 Subject: [PATCH 46/75] rm commented code --- py_bentoctl/auth_helper.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index dda2273e..4e2bff1f 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -391,17 +391,16 @@ def create_grafana_client_roles_if_needed(token: str, client_id: str) -> Optiona return role_representations def create_grafana_client_groups_if_needed(token: str, role_mappings: dict, client_id: str) -> None: + # create parent grafana group (no role mapping) parent_group = {"name": "grafana"} parent_group = create_group_or_exit(token, parent_group) + # create subgroups with client-role mappings sub_groups = [ {"name": "admin"}, {"name": "editor"}, {"name": "viewer"} ] - - # Add client-level role mappings to groups - # grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID, verbose=False) for subgroup in sub_groups: group_rep = create_group_or_exit(token, subgroup, parent_group_rep=parent_group) role_rep = role_mappings[subgroup["name"]] From 20998b0c277b6fd51111eaeb479e4ccf71a1a68d Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Wed, 28 Aug 2024 13:56:37 -0400 Subject: [PATCH 47/75] strict cookies samesite --- lib/logs/docker-compose.logs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 0a3c71a0..98843e41 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -6,7 +6,7 @@ services: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - GF_SERVER_ROOT_URL=${BENTO_PRIVATE_GRAFANA_URL} - GF_SERVER_SERVE_FROM_SUB_PATH=true - - GF_SECURITY_COOKIE_SAMESITE=none + - GF_SECURITY_COOKIE_SAMESITE=strict - GF_SECURITY_COOKIE_SECURE=true - GF_SECURITY_ALLOW_EMBEDDING=true - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true From c51a53a6615294155da865e5adc718195bce938a Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 15 Aug 2024 08:07:32 -0400 Subject: [PATCH 48/75] refact(bentoctl): factor out common client/secret creation code --- py_bentoctl/auth_helper.py | 154 ++++++++++++++----------------------- 1 file changed, 58 insertions(+), 96 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 90211a8e..857e63b6 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -160,6 +160,52 @@ def get_keycloak_client_secret(client_id: str, token: str): return keycloak_req(f"{KC_CLIENTS_ENDPOINT}/{client_id}/client-secret", bearer_token=token) +def create_client_and_secret_for_service( + client_id: str, + env_var_to_set: str, + private_url: str | None, + token: str, + is_service_account: bool = False, + to_restart: str = "the gateway", + token_lifespan: int = 900, # default access token lifespan: 15 minutes + use_refresh_tokens: bool = False, # by default, don't use refresh tokens! (they're less secure) +): + client_kc_id: Optional[str] = fetch_existing_client_id(token, client_id) + + if client_kc_id is None: + # Create the Bento WES client + create_keycloak_client_or_exit( + token, + client_id, + standard_flow_enabled=not is_service_account, + service_accounts_enabled=is_service_account, + public_client=False, + redirect_uris=[ + f"{private_url}/*" + ] if not is_service_account else [], # Not used for client credentials access + web_origins=[private_url] if not is_service_account else [], # " + access_token_lifespan=token_lifespan, + use_refresh_tokens=use_refresh_tokens, + ) + client_kc_id = fetch_existing_client_id(token, client_id) + + # Fetch and print secret + + client_secret_res = get_keycloak_client_secret(client_kc_id, token) + + client_secret_data = client_secret_res.json() + if not client_secret_res.ok: + err(f" Failed to get client secret for {client_id}; {client_secret_res.status_code} " + f"{client_secret_data}") + exit(1) + + client_secret = client_secret_data["value"] + cprint( + f" Please set {env_var_to_set} to {client_secret} in local.env and restart {to_restart}", + attrs=["bold"], + ) + + def init_auth(docker_client: docker.DockerClient): check_auth_admin_user() @@ -230,109 +276,25 @@ def create_web_client_if_needed(token: str) -> None: ) def create_grafana_client_if_needed(token: str) -> None: - grafana_client_kc_id: Optional[str] = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) - - if grafana_client_kc_id is None: - # Create the Bento WES client - create_keycloak_client_or_exit( - token, - GRAFANA_CLIENT_ID, - standard_flow_enabled=True, - service_accounts_enabled=False, - public_client=False, # Use client secret for this one - redirect_uris=[ - f"{GRAFANA_PRIVATE_URL}/*" - ], - web_origins=[GRAFANA_PRIVATE_URL], - access_token_lifespan=900, # default access token lifespan: 15 minutes - use_refresh_tokens=False, - ) - grafana_client_kc_id = fetch_existing_client_id(token, GRAFANA_CLIENT_ID) - - # Fetch and print secret - - client_secret_res = get_keycloak_client_secret(grafana_client_kc_id, token) - - client_secret_data = client_secret_res.json() - if not client_secret_res.ok: - err(f" Failed to get client secret for {GRAFANA_CLIENT_ID}; {client_secret_res.status_code} " - f"{client_secret_data}") - exit(1) - - client_secret = client_secret_data["value"] - cprint( - f" Please set BENTO_GRAFANA_CLIENT_SECRET to {client_secret} in local.env and restart Grafana", - attrs=["bold"], + create_client_and_secret_for_service( + GRAFANA_CLIENT_ID, "BENTO_GRAFANA_CLIENT_SECRET", GRAFANA_PRIVATE_URL, token, to_restart="Grafana" ) # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: - cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) - - if cbio_client_kc_id is None: - # Create the cBioportal client - create_keycloak_client_or_exit( - token, - CBIOPORTAL_CLIENT_ID, - standard_flow_enabled=True, - service_accounts_enabled=False, - public_client=False, - redirect_uris=[f"{CBIOPORTAL_URL}{AUTH_LOGIN_REDIRECT_PATH}"], - web_origins=[CBIOPORTAL_URL], - access_token_lifespan=900, # 15 minutes - use_refresh_tokens=True, - ) - cbio_client_kc_id = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) - - # Fetch and print secret - - client_secret_res = get_keycloak_client_secret(cbio_client_kc_id, token) - - client_secret_data = client_secret_res.json() - if not client_secret_res.ok: - err(f" Failed to get client secret for {CBIOPORTAL_CLIENT_ID}; {client_secret_res.status_code} " - f"{client_secret_data}") - exit(1) - - client_secret = client_secret_data["value"] - cprint( - f" Please set BENTO_CBIOPORTAL_CLIENT_SECRET to {client_secret} in local.env and restart the " - f"gateway", - attrs=["bold"], + create_client_and_secret_for_service( + GRAFANA_CLIENT_ID, "BENTO_CBIOPORTAL_CLIENT_SECRET", CBIOPORTAL_URL, token, use_refresh_tokens=True ) def create_wes_client_if_needed(token: str) -> None: - wes_client_kc_id: Optional[str] = fetch_existing_client_id(token, WES_CLIENT_ID) - - if wes_client_kc_id is None: - # Create the Bento WES client - create_keycloak_client_or_exit( - token, - WES_CLIENT_ID, - standard_flow_enabled=False, - service_accounts_enabled=True, - public_client=False, # Use client secret for this one - redirect_uris=[], # Not used with a standard/web flow - just client credentials - web_origins=[], # " - access_token_lifespan=WES_WORKFLOW_TIMEOUT, # WES workflow lifespan - use_refresh_tokens=False, # No refreshing these allowed! - ) - wes_client_kc_id = fetch_existing_client_id(token, WES_CLIENT_ID) - - # Fetch and print secret - - client_secret_res = get_keycloak_client_secret(wes_client_kc_id, token) - - client_secret_data = client_secret_res.json() - if not client_secret_res.ok: - err(f" Failed to get client secret for {WES_CLIENT_ID}; {client_secret_res.status_code} " - f"{client_secret_data}") - exit(1) - - client_secret = client_secret_data["value"] - cprint( - f" Please set BENTO_WES_CLIENT_SECRET to {client_secret} in local.env and restart WES", - attrs=["bold"], + create_client_and_secret_for_service( + WES_CLIENT_ID, + "BENTO_WES_CLIENT_SECRET", + None, + token, + is_service_account=True, + to_restart="WES", + token_lifespan=WES_WORKFLOW_TIMEOUT, ) def create_test_user_if_needed(token: str) -> None: From 782522b0edb6d3116119b616d71b8cf1332188c0 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 28 Aug 2024 15:17:44 -0400 Subject: [PATCH 49/75] address review comments --- py_bentoctl/auth_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 857e63b6..fdf0f3e0 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -173,7 +173,6 @@ def create_client_and_secret_for_service( client_kc_id: Optional[str] = fetch_existing_client_id(token, client_id) if client_kc_id is None: - # Create the Bento WES client create_keycloak_client_or_exit( token, client_id, @@ -283,7 +282,7 @@ def create_grafana_client_if_needed(token: str) -> None: # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: create_client_and_secret_for_service( - GRAFANA_CLIENT_ID, "BENTO_CBIOPORTAL_CLIENT_SECRET", CBIOPORTAL_URL, token, use_refresh_tokens=True + CBIOPORTAL_CLIENT_ID, "BENTO_CBIOPORTAL_CLIENT_SECRET", CBIOPORTAL_URL, token, use_refresh_tokens=True ) def create_wes_client_if_needed(token: str) -> None: From 58108a32cab2e9cb5e32bc979109901a33a8887d Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Wed, 28 Aug 2024 15:27:45 -0400 Subject: [PATCH 50/75] chore: add migration guide todo --- docs/migrating_to_17.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/migrating_to_17.md diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md new file mode 100644 index 00000000..f44426b4 --- /dev/null +++ b/docs/migrating_to_17.md @@ -0,0 +1,6 @@ +# Migrating to Bento v17 + +Key points: +* Bento now has observability tools to help monitor the services (Grafana) + +* ... From 9d5aa154be9f46e849c20801c9adf802d36051c3 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Thu, 29 Aug 2024 13:48:34 -0400 Subject: [PATCH 51/75] adress review comments --- py_bentoctl/auth_helper.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 4e2bff1f..954cd6e0 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import docker +import json import os import requests import subprocess import urllib3 -import json from termcolor import cprint from urllib3.exceptions import InsecureRequestWarning @@ -44,6 +44,7 @@ GRAFANA_CLIENT_ID = os.getenv("BENTO_GRAFANA_CLIENT_ID") GRAFANA_PRIVATE_URL = os.getenv("BENTO_PRIVATE_GRAFANA_URL") +GRAFANA_ROLES = ("admin", "editor", "viewer") KC_ADMIN_API_ENDPOINT = f"admin/realms/{AUTH_REALM}" KC_ADMIN_API_GROUP_ENDPOINT = f"{KC_ADMIN_API_ENDPOINT}/groups" @@ -129,8 +130,7 @@ def fetch_existing_group_rep_or_exit( token: str, group_name: str, parent_rep: dict | None = None, - verbose: bool = True - ) -> Optional[dict]: + verbose: bool = True) -> Optional[dict]: endpoint = KC_ADMIN_API_GROUP_ENDPOINT if parent_rep: endpoint = f"{KC_ADMIN_API_GROUP_ENDPOINT}/{parent_rep['id']}/children" @@ -168,7 +168,7 @@ def create_client_role_or_exit(token: str, client_id: str, role_name: str) -> Op # role creation response returns no data, fetch the created RoleRepresentation for later use created_role_rep = fetch_existing_client_role(token, client_id, role_name, verbose=False) - warn(f" Created client role: {role_name}.") + cprint(f" Created client role: {role_name}.", "green") return created_role_rep @@ -191,7 +191,7 @@ def create_group_or_exit(token: str, group_rep: dict, parent_group_rep: dict = N # group creation response returns no data, fetch the created GroupRepresentation for later use created_group = fetch_existing_group_rep_or_exit(token, group_rep["name"], parent_group_rep, verbose=False) - warn(f" Created group: {created_group['path']}") + cprint(f" Created group: {created_group['path']}", "green") return created_group @@ -216,7 +216,7 @@ def add_client_role_mapping_to_group_or_exit(token: str, group_rep: dict, client err(f" Failed to add client-level role {role_rep['name']} to group {group_rep['path']}") exit(1) - warn(f" Created client-level role mapping for group: {group_rep['path']}") + cprint(f" Created client-level role mapping for group: {group_rep['path']}", "green") def create_keycloak_client_or_exit( @@ -385,7 +385,7 @@ def create_grafana_client_if_needed(token: str) -> Optional[str]: def create_grafana_client_roles_if_needed(token: str, client_id: str) -> Optional[dict]: role_representations = {} - for role_name in ["admin", "editor", "viewer"]: + for role_name in GRAFANA_ROLES: client_role = create_client_role_or_exit(token, client_id, role_name) role_representations[role_name] = client_role return role_representations @@ -396,11 +396,7 @@ def create_grafana_client_groups_if_needed(token: str, role_mappings: dict, clie parent_group = create_group_or_exit(token, parent_group) # create subgroups with client-role mappings - sub_groups = [ - {"name": "admin"}, - {"name": "editor"}, - {"name": "viewer"} - ] + sub_groups = [{"name": g} for g in GRAFANA_ROLES] for subgroup in sub_groups: group_rep = create_group_or_exit(token, subgroup, parent_group_rep=parent_group) role_rep = role_mappings[subgroup["name"]] @@ -446,7 +442,7 @@ def set_include_client_roles_in_id_tokens(token: str): if not update_mapper_res.ok: err(f" Failed to modify 'client roles' mapper: {update_mapper_res.status_code}") exit(1) - warn(" Updated 'client roles' scope mapper to include roles in the ID token.") + cprint(" Updated 'client roles' scope mapper to include roles in the ID token.", "green") elif roles_mapper["config"]["id.token.claim"] == "true": warn(" The 'client roles' scope mapper already includes roles in the ID token.") @@ -607,7 +603,7 @@ def success(): create_grafana_client_groups_if_needed(access_token, role_mappings, grafana_client_id) set_include_client_roles_in_id_tokens(access_token) cprint( - " Add users to the relevant Grafana sub-groups to give them access: admin, editor, viewer", + f" Add users to the relevant Grafana sub-groups to give them access: {' '.join(GRAFANA_ROLES)}", attrs=["bold"], ) success() From e956b9d8890d1758ff2a178d4ca2ee7570fb39c2 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Thu, 29 Aug 2024 14:53:41 -0400 Subject: [PATCH 52/75] only oauth login --- lib/logs/docker-compose.logs.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index 98843e41..c7fc99bc 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -6,12 +6,13 @@ services: - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - GF_SERVER_ROOT_URL=${BENTO_PRIVATE_GRAFANA_URL} - GF_SERVER_SERVE_FROM_SUB_PATH=true - - GF_SECURITY_COOKIE_SAMESITE=strict + - GF_SECURITY_COOKIE_SAMESITE=none - GF_SECURITY_COOKIE_SECURE=true - GF_SECURITY_ALLOW_EMBEDDING=true - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true - GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=true - GF_AUTH_GENERIC_OAUTH_ENABLED=true + - GF_AUTH_DISABLE_LOGIN=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} From ec144a645199f70260ebda78cf4d1866fa9a92b0 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Thu, 29 Aug 2024 16:28:13 -0400 Subject: [PATCH 53/75] rm embedding, comment --- lib/logs/docker-compose.logs.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logs/docker-compose.logs.yaml b/lib/logs/docker-compose.logs.yaml index c7fc99bc..7cbb41a0 100644 --- a/lib/logs/docker-compose.logs.yaml +++ b/lib/logs/docker-compose.logs.yaml @@ -8,11 +8,12 @@ services: - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_SECURITY_COOKIE_SAMESITE=none - GF_SECURITY_COOKIE_SECURE=true - - GF_SECURITY_ALLOW_EMBEDDING=true - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=true + # --- Only allow logins through OAUTH - GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=true - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_DISABLE_LOGIN=true + # --- - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak-OAuth - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${BENTO_GRAFANA_CLIENT_ID} From 64d77735a70704d2da8f5772694f7bf44579d1a4 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 15:25:05 -0400 Subject: [PATCH 54/75] docs: link to v17 migration guide from README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2664dba9..78cd5b88 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ that make up the Bento platform. ### Migration documents +* [v16 to v17](./docs/migrating_to_17.md) * [v15.2 to v16](./docs/migrating_to_16.md) * [v15.1 to v15.2](./docs/migrating_to_15_2.md) * [v15 to v15.1](./docs/migrating_to_15_1.md) From eb20934025bacde3034b1387b550978d6dbe33e7 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 15:25:17 -0400 Subject: [PATCH 55/75] set Katsu to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index fbf6e368..71816c07 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -269,7 +269,7 @@ BENTOV2_KATSU_DB_CPUS=4 # Katsu BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu -BENTOV2_KATSU_VERSION=pr-529 +BENTOV2_KATSU_VERSION=edge BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net From 0dc03d0646e4cafb607d2a583385dad69290aa14 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 15:25:32 -0400 Subject: [PATCH 56/75] rm now-unused env var from public --- etc/bento.env | 1 - lib/public/docker-compose.public.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 71816c07..057e4c45 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -382,7 +382,6 @@ BENTO_PUBLIC_EXTERNAL_PORT=8090 BENTO_PUBLIC_DEBUG=false BENTO_PUBLIC_SERVICE_ID=${BENTOV2_PREFIX}-public BENTO_PUBLIC_CLIENT_NAME=BentoPublicDev -BENTO_PUBLIC_KATSU_URL=http://${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_INTERNAL_PORT} BENTO_PUBLIC_WES_URL=http://${BENTOV2_WES_CONTAINER_NAME}:${BENTOV2_WES_INTERNAL_PORT} BENTO_PUBLIC_GOHAN_URL=http://${BENTOV2_GOHAN_API_CONTAINER_NAME}:${BENTOV2_GOHAN_API_INTERNAL_PORT} BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} diff --git a/lib/public/docker-compose.public.yaml b/lib/public/docker-compose.public.yaml index 25ec8f7a..16a7f79f 100644 --- a/lib/public/docker-compose.public.yaml +++ b/lib/public/docker-compose.public.yaml @@ -11,7 +11,6 @@ services: - BENTO_UID - BENTO_PUBLIC_SERVICE_ID - BENTO_PUBLIC_CLIENT_NAME - - BENTO_PUBLIC_KATSU_URL - BENTO_PUBLIC_WES_URL - BENTO_PUBLIC_GOHAN_URL - BENTO_PUBLIC_PORTAL_URL From 90719de6fa8c25a94d51f8819331df3bfb680998 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 15:27:51 -0400 Subject: [PATCH 57/75] docs: WIP migration guide content for v17 --- docs/migrating_to_17.md | 44 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index f44426b4..0411fb67 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -1,6 +1,46 @@ # Migrating to Bento v17 Key points: -* Bento now has observability tools to help monitor the services (Grafana) - + +* Bento now has observability tools to help monitor the services (Grafana). Some setup is required for this feature to + work. +* Katsu discovery endpoints now have an authorization layer. Data that used to be completely public by default (i.e., + censored counts) now requires a permission (`query:project_level_counts` and/or `query:dataset_level_counts`), and + thus a grant in the authorization service. * ... + + +## 1. Stop Bento + +```bash +./bentoctl.bash stop +``` + + +## 2. Update images + +```bash +./bentoctl.bash pull +``` + + +## 3. *(Optional)* Set up Grafana + +TODO: environment + +```bash +./bentoctl.bash start auth +./bentoctl.bash init-auth +``` + + +## 4. Set up public data access grants + +TODO + + +## 5. Start Bento + +```bash +./bentoctl.bash start +``` From 28ed30170130d4067b99ec6444566048de0052cc Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 15:43:22 -0400 Subject: [PATCH 58/75] docs: add public data access configuration step for v17 migration --- docs/migrating_to_17.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index 0411fb67..cc6c31bc 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -36,7 +36,19 @@ TODO: environment ## 4. Set up public data access grants -TODO +Starting from Bento v17, anonymous visitors do not have access to see censored counts data by default, even if a +discovery configuration has been set up. For anonymous visitors to access data, a level (`bool`, `counts`, `full`) +must be chosen and passed to the `bento_authz` CLI command below. + +```bash +./bentoctl.bash shell authz +# The level below (counts) preserves previous functionality. Other possible options are: +# - none - will do nothing. +# - bool - for censored true/false discovery, but in effect right now forbids access. +# - counts - for censored count discovery. +# - full - allows full data access (record-level, including sensitive data such as IDs), uncensored counts, etc. +bento_authz public-data-access counts +``` ## 5. Start Bento From 65c9c1a8fb59846f6bd0b62ace84316aa31ab982 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 15:51:20 -0400 Subject: [PATCH 59/75] docs: instructions for public data access setup in installation guide --- docs/installation.md | 20 ++++++++++++++++++-- docs/migrating_to_17.md | 4 +++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index d70f84ce..f278c39a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -297,7 +297,7 @@ utilize new variables generated during the OIDC configuration. ## 6. Configure permissions -### a. Create superuser permissions in the new Bento authorization service +### a. Create superuser permissions in the Bento authorization service First, run the authorization service and then open a shell into the container: @@ -334,7 +334,23 @@ bento_authz create grant \ 'query:data' 'ingest:data' 'ingest:reference_material' 'delete:reference_material' ``` -### c. *Optional step:* Assign portal access to all users in the instance realm +### c. Configure public data access for all users, including anonymous visitors (if desired): + +To configure public data access, run the following command in the authorization service container. Note that with the +`full` value, **THIS GIVES FULL DATA ACCESS TO EVERYONE WHO VISITS YOUR INSTANCE!** + +```bash +# Configure public data access +# ---------------------------- +# The level below ("counts") preserves previous functionality. Other possible options are: +# - none - will do nothing. +# - bool - for censored true/false discovery, but in effect right now forbids access. +# - counts - for censored count discovery. +# - full - allows full data access (record-level, including sensitive data such as IDs), uncensored counts, etc. +bento_authz public-data-access counts +``` + +### d. *Optional step:* Assign portal access to all users in the instance realm We added a special permission, `view:private_portal`, to Bento v12/v13 in order to carry forward the current 'legacy' authorization behaviour for one more major version. This permission currently behaves as a super-permission, diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index cc6c31bc..4e6e247b 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -42,7 +42,9 @@ must be chosen and passed to the `bento_authz` CLI command below. ```bash ./bentoctl.bash shell authz -# The level below (counts) preserves previous functionality. Other possible options are: +# Configure public data access +# ---------------------------- +# The level below ("counts") preserves previous functionality. Other possible options are: # - none - will do nothing. # - bool - for censored true/false discovery, but in effect right now forbids access. # - counts - for censored count discovery. From 249012d59a980dc4305e80f08cb05e5125454f37 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 9 Sep 2024 16:05:44 -0400 Subject: [PATCH 60/75] set authz to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 057e4c45..1858a8a7 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -77,7 +77,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net" # - Authz service BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service -BENTO_AUTHZ_VERSION=0.9.2 +BENTO_AUTHZ_VERSION=edge BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net From 50fec773fa5afbea909ad2520c1b917befde5c91 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 10 Sep 2024 11:35:43 -0400 Subject: [PATCH 61/75] set public to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 1858a8a7..f178e04e 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=pr-175 +BENTO_PUBLIC_VERSION=edge BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 2c634bf96c73ab246ad11e866ae9454b281b9b8e Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 12 Sep 2024 11:15:51 -0400 Subject: [PATCH 62/75] work on configuring beacon OIDC for data access --- etc/bento.env | 3 +-- etc/bento_deploy.env | 6 +++++- etc/bento_dev.env | 4 ++++ etc/default_config.env | 3 +++ lib/beacon/docker-compose.beacon.yaml | 8 +++++--- py_bentoctl/auth_helper.py | 16 ++++++++++++++++ 6 files changed, 34 insertions(+), 6 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index f178e04e..ace283fe 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -391,7 +391,7 @@ BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon -BENTO_BEACON_VERSION=0.15.2 +BENTO_BEACON_VERSION=pr-107 BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_BEACON_EXTERNAL_PORT=5000 @@ -404,7 +404,6 @@ BENTO_BEACON_CONFIG_DIR=${PWD}/lib/beacon/config BENTO_BEACON_GOHAN_BASE_URL=http://${BENTOV2_GOHAN_API_CONTAINER_NAME}:${BENTOV2_GOHAN_API_INTERNAL_PORT} BENTO_BEACON_KATSU_TIMEOUT=60 BENTO_BEACON_GOHAN_TIMEOUT=60 -BENTO_BEACON_OIDC_ISSUER=${BENTOV2_AUTH_PUBLIC_URL}/auth/realms/${BENTOV2_AUTH_REALM} # cBioPortal diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 8d0be71d..3f2f9e25 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -51,13 +51,17 @@ BENTOV2_AUTH_TEST_PASSWORD= BENTO_AUTH_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! BENTO_AUTHZ_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! +# - Aggregation/Beacon client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_AGGREGATION_CLIENT_ID=aggregation +BENTO_AGGREGATION_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! + # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! # - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_GRAFANA_CLIENT_ID=grafana -BENTO_GRAFANA_CLIENT_SECRET= +BENTO_GRAFANA_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING IF GRAFANA IS ENABLED! # --------------------------------------------------------------------- BENTO_WEB_CUSTOM_HEADER= diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 9e471dc2..941f4ed1 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -51,6 +51,10 @@ BENTOV2_AUTH_ADMIN_PASSWORD= BENTOV2_AUTH_TEST_USER= BENTOV2_AUTH_TEST_PASSWORD= +# - Aggregation/Beacon client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_AGGREGATION_CLIENT_ID=aggregation +BENTO_AGGREGATION_CLIENT_SECRET= + # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/etc/default_config.env b/etc/default_config.env index 3ff8966f..d7356946 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -78,6 +78,9 @@ BENTOV2_AUTH_TEST_PASSWORD= # - Auth (Keycloak) DB credentials BENTO_AUTH_DB_PASSWORD= BENTO_AUTHZ_DB_PASSWORD= +# - Aggregation/Beacon client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM +BENTO_AGGREGATION_CLIENT_ID=aggregation +BENTO_AGGREGATION_CLIENT_SECRET= # - cBioPortal Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM BENTO_CBIOPORTAL_CLIENT_ID=cbioportal BENTO_CBIOPORTAL_CLIENT_SECRET= diff --git a/lib/beacon/docker-compose.beacon.yaml b/lib/beacon/docker-compose.beacon.yaml index 3a512284..37ee9e92 100644 --- a/lib/beacon/docker-compose.beacon.yaml +++ b/lib/beacon/docker-compose.beacon.yaml @@ -15,16 +15,18 @@ services: - BENTO_BEACON_DEBUGGER_INTERNAL_PORT - BENTO_BEACON_DEBUGGER_EXTERNAL_PORT - CONFIG_ABSOLUTE_PATH=/config/ - - OIDC_ISSUER=${BENTO_BEACON_OIDC_ISSUER} - - CLIENT_ID=${BENTOV2_AUTH_CLIENT_ID} - BEACON_BASE_URL=${BENTOV2_PUBLIC_URL}/api/beacon - BENTO_BEACON_VERSION=${BENTO_BEACON_VERSION} - BENTO_PUBLIC_CLIENT_NAME - BENTOV2_DOMAIN - BENTOV2_PUBLIC_URL - BENTO_BEACON_UI_ENABLED - - BENTO_AUTHZ_SERVICE_URL - DRS_URL=${BENTOV2_PUBLIC_URL}/api/drs + # Authorization + - BENTO_AUTHZ_SERVICE_URL + - BENTO_OPENID_CONFIG_URL + - BEACON_CLIENT_ID=BENTO_AGGREGATION_CLIENT_ID + - BEACON_CLIENT_SECRET=BENTO_AGGREGATION_CLIENT_SECRET volumes: - ${BENTO_BEACON_CONFIG_DIR}:/config:ro networks: diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 3d4d2b6f..b8667e7f 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -37,6 +37,8 @@ AUTH_TEST_PASSWORD = os.getenv("BENTOV2_AUTH_TEST_PASSWORD") AUTH_CONTAINER_NAME = os.getenv("BENTOV2_AUTH_CONTAINER_NAME") +AGGREGATION_CLIENT_ID = os.getenv("BENTO_AGGREGATION_CLIENT_ID") + CBIOPORTAL_CLIENT_ID = os.getenv("BENTO_CBIOPORTAL_CLIENT_ID") WES_CLIENT_ID = os.getenv("BENTO_WES_CLIENT_ID") @@ -459,6 +461,16 @@ def set_include_client_roles_in_id_tokens(token: str): elif roles_mapper["config"]["id.token.claim"] == "true": warn(" The 'client roles' scope mapper already includes roles in the ID token.") + def create_aggregation_client_if_needed(token: str) -> None: + create_client_and_secret_for_service( + AGGREGATION_CLIENT_ID, + "BENTO_AGGREGATION_CLIENT_SECRET", + None, + token, + is_service_account=True, + to_restart="Aggregation and Beacon", + ) + # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: create_client_and_secret_for_service( @@ -544,6 +556,10 @@ def success(): create_web_client_if_needed(access_token) success() + info(f" Creating aggregation/Beacon client: {AGGREGATION_CLIENT_ID}") + create_aggregation_client_if_needed(access_token) + success() + # TODO: if cBioPortal ever needs auth implemented, re-enable this and set up Bento Gateway to handle cBioPortal # client authorization. # - David L, 2024-03-25 From 5971afd90f8e3d40a683c3a7f25249801f328a4f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 12 Sep 2024 11:39:38 -0400 Subject: [PATCH 63/75] fix missing BENTO_MONITORING_ENABLED from default_config + reorder --- etc/bento_deploy.env | 2 +- etc/bento_dev.env | 2 +- etc/default_config.env | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 3f2f9e25..b120f6ff 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -11,8 +11,8 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='false' # Set to true if using Beacon! BENTO_BEACON_UI_ENABLED='false' BENTO_CBIOPORTAL_ENABLED='false' -BENTO_MONITORING_ENABLED='false' BENTO_GOHAN_ENABLED='true' +BENTO_MONITORING_ENABLED='false' # - Switch to enable French translation in Bento Public BENTO_PUBLIC_TRANSLATED='true' diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 941f4ed1..895618df 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -11,8 +11,8 @@ BENTO_GATEWAY_USE_TLS='true' BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' -BENTO_MONITORING_ENABLED='false' BENTO_GOHAN_ENABLED='true' +BENTO_MONITORING_ENABLED='false' # - Switch to enable French translation in Bento Public BENTO_PUBLIC_TRANSLATED='true' diff --git a/etc/default_config.env b/etc/default_config.env index d7356946..21c9175c 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -17,6 +17,7 @@ BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' BENTO_GOHAN_ENABLED='true' +BENTO_MONITORING_ENABLED='false' # - Switch to enable French translation in Bento Public BENTO_PUBLIC_TRANSLATED='true' From 269d1b655289938fedeaffc9f60b548fd78b1b4f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 12 Sep 2024 11:52:48 -0400 Subject: [PATCH 64/75] configure external URLs for Beacon Katsu/Gohan connection also factors out repeated URL vars for services into bento.env vars --- etc/bento.env | 15 ++++++++++++++- lib/aggregation/docker-compose.aggregation.yaml | 4 ++-- lib/beacon/docker-compose.beacon.yaml | 6 +++--- lib/drs/docker-compose.drs.yaml | 2 +- lib/wes/docker-compose.wes.yaml | 8 ++++---- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index ace283fe..bd9dcb3d 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -133,6 +133,8 @@ BENTOV2_SERVICE_REGISTRY_EXTERNAL_PORT=5010 BENTOV2_SERVICE_REGISTRY_MEM_LIM=1G BENTOV2_SERVICE_REGISTRY_CPUS=1 +BENTO_SERVICE_REGISTRY_URL=${BENTOV2_PUBLIC_URL}/api/service-registry + # Notification BENTOV2_NOTIFICATION_IMAGE=ghcr.io/bento-platform/bento_notification_service BENTOV2_NOTIFICATION_VERSION=3.1.4 @@ -249,6 +251,10 @@ BENTOV2_DRS_DEBUGGER_EXTERNAL_PORT=5682 BENTOV2_DRS_MEM_LIM=2G BENTOV2_DRS_CPUS=2 +# Canonical/world-resolvable URL for DRS +# TODO: services should use the service registry instead +BENTO_DRS_URL=${BENTOV2_PUBLIC_URL}/api/drs + # Katsu-DB BENTOV2_KATSU_DB_IMAGE=postgres BENTOV2_KATSU_DB_VERSION=13 @@ -290,6 +296,10 @@ BENTOV2_KATSU_CPUS=4 # urls in templates. CHORD_METADATA_SUB_PATH=/api/metadata +# Canonical/world-resolvable URL for Katsu +# TODO: services should use the service registry instead +BENTO_KATSU_URL=${BENTOV2_PORTAL_PUBLIC_URL}${CHORD_METADATA_SUB_PATH} + # Redis BENTOV2_REDIS_BASE_IMAGE=redis BENTOV2_REDIS_BASE_IMAGE_VERSION=7.0.15-alpine @@ -336,6 +346,10 @@ BENTOV2_GOHAN_API_AUTHZ_ENABLED=false #BENTOV2_GOHAN_API_AUTHZ_AGREED_DISABLED_RISK=false BENTOV2_GOHAN_API_AUTHZ_REQHEADS=X-CUSTOM-1,X-CUSTOM-2 +# Canonical/world-resolvable URL for Gohan +# - TODO: services should use the service registry instead +BENTO_GOHAN_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/gohan + # -- Elasticsearch BENTOV2_GOHAN_ES_USERNAME=elastic # BENTOV2_GOHAN_ES_PASSWORD comes from default_config @@ -401,7 +415,6 @@ BENTO_BEACON_MEM_LIM=2G BENTO_BEACON_CPUS=2 BENTO_BEACON_CONFIG_DIR=${PWD}/lib/beacon/config -BENTO_BEACON_GOHAN_BASE_URL=http://${BENTOV2_GOHAN_API_CONTAINER_NAME}:${BENTOV2_GOHAN_API_INTERNAL_PORT} BENTO_BEACON_KATSU_TIMEOUT=60 BENTO_BEACON_GOHAN_TIMEOUT=60 diff --git a/lib/aggregation/docker-compose.aggregation.yaml b/lib/aggregation/docker-compose.aggregation.yaml index 3b673e88..3a90e025 100644 --- a/lib/aggregation/docker-compose.aggregation.yaml +++ b/lib/aggregation/docker-compose.aggregation.yaml @@ -8,8 +8,8 @@ services: - BENTO_DEBUG=False - USE_GOHAN=true - CORS_ORIGINS=${BENTO_CORS_ORIGINS} - - KATSU_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/metadata/ - - SERVICE_REGISTRY_URL=${BENTOV2_PUBLIC_URL}/api/service-registry/ + - KATSU_URL=${BENTO_KATSU_URL}/ + - SERVICE_REGISTRY_URL=${BENTO_SERVICE_REGISTRY_URL}/ - BENTO_AUTHZ_SERVICE_URL networks: - aggregation-net diff --git a/lib/beacon/docker-compose.beacon.yaml b/lib/beacon/docker-compose.beacon.yaml index 37ee9e92..46cdb249 100644 --- a/lib/beacon/docker-compose.beacon.yaml +++ b/lib/beacon/docker-compose.beacon.yaml @@ -5,9 +5,9 @@ services: container_name: ${BENTO_BEACON_CONTAINER_NAME} environment: - BENTO_UID - - GOHAN_BASE_URL=${BENTO_BEACON_GOHAN_BASE_URL} + - GOHAN_BASE_URL=${BENTO_GOHAN_URL} - KATSU_TIMEOUT=${BENTO_BEACON_KATSU_TIMEOUT} - - KATSU_BASE_URL=http://${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_INTERNAL_PORT} + - KATSU_BASE_URL=${BENTO_KATSU_URL} - GOHAN_TIMEOUT=${BENTO_BEACON_GOHAN_TIMEOUT} - BENTO_BEACON_INTERNAL_PORT - INTERNAL_PORT=${BENTO_BEACON_INTERNAL_PORT} @@ -21,7 +21,7 @@ services: - BENTOV2_DOMAIN - BENTOV2_PUBLIC_URL - BENTO_BEACON_UI_ENABLED - - DRS_URL=${BENTOV2_PUBLIC_URL}/api/drs + - DRS_URL=${BENTO_DRS_URL} # Authorization - BENTO_AUTHZ_SERVICE_URL - BENTO_OPENID_CONFIG_URL diff --git a/lib/drs/docker-compose.drs.yaml b/lib/drs/docker-compose.drs.yaml index 09c56b9d..7d131a41 100644 --- a/lib/drs/docker-compose.drs.yaml +++ b/lib/drs/docker-compose.drs.yaml @@ -8,7 +8,7 @@ services: - BENTO_DRS_CONTAINER_DATA_VOLUME_DIR # Special container-only variable to specify where the volume is mounted - DATABASE=${BENTO_DRS_CONTAINER_DATA_VOLUME_DIR}/db/ # slightly confused naming, folder for database to go in - DATA=${BENTO_DRS_CONTAINER_DATA_VOLUME_DIR}/obj/ # DRS file objects, vs. the database - - SERVICE_BASE_URL=${BENTOV2_PUBLIC_URL}/api/drs + - SERVICE_BASE_URL=${BENTO_DRS_URL} - INTERNAL_PORT=${BENTOV2_DRS_INTERNAL_PORT} - DRS_INGEST_TMP_DIR=${BENTO_DRS_CONTAINER_TMP_VOLUME_DIR} # Volume for writing possibly large temporary files to - CORS_ORIGINS=${BENTO_CORS_ORIGINS} diff --git a/lib/wes/docker-compose.wes.yaml b/lib/wes/docker-compose.wes.yaml index 8699ec57..73556ddf 100644 --- a/lib/wes/docker-compose.wes.yaml +++ b/lib/wes/docker-compose.wes.yaml @@ -24,11 +24,11 @@ services: - WORKFLOW_HOST_ALLOW_LIST=${BENTOV2_GOHAN_API_CONTAINER_NAME}:${BENTOV2_GOHAN_API_INTERNAL_PORT},${BENTOV2_DOMAIN},${BENTOV2_PORTAL_DOMAIN},${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_INTERNAL_PORT} # Service URLS - - DRS_URL=${BENTOV2_PUBLIC_URL}/api/drs - - GOHAN_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/gohan - - KATSU_URL=${BENTOV2_PORTAL_PUBLIC_URL}/api/metadata + - DRS_URL=${BENTO_DRS_URL} + - GOHAN_URL=${BENTO_GOHAN_URL} + - KATSU_URL=${BENTO_KATSU_URL} - BENTO_AUTHZ_SERVICE_URL - - SERVICE_REGISTRY_URL=${BENTOV2_PUBLIC_URL}/api/service-registry + - SERVICE_REGISTRY_URL=${BENTO_SERVICE_REGISTRY_URL} - INTERNAL_PORT=${BENTOV2_WES_INTERNAL_PORT} - WORKFLOW_TIMEOUT=${BENTOV2_WES_WORKFLOW_TIMEOUT} From 9d01c0574bec972c6075e5568b58675be0f0d2f9 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 12 Sep 2024 12:46:10 -0400 Subject: [PATCH 65/75] docs: v17 migration and install guides for agg/Beacon, Grafana --- docs/installation.md | 32 +++++++++++++++++++++-------- docs/migrating_to_17.md | 45 ++++++++++++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index f278c39a..93cd05a4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -276,8 +276,10 @@ specified in the step above. ./bentoctl.bash init-auth ``` -**If using an external identity provider**, only start the cluster's gateway -after setting `CLIENT_SECRET` in your local environment file: +After running `init-auth`, be sure to put all client secrets into your `local.env` file! + +**If using an external identity provider**, only start the cluster's gateway after setting various `*_CLIENT_SECRET` +variables in your local environment file: ```bash ./bentoctl.bash run gateway @@ -317,24 +319,38 @@ which in Keycloak should be a UUID. ### b. Create grants for the Workflow Execution Service (WES) OAuth2 client -Run the following commands to set up authorization for the WES client. Don't forget to replace `ISSUER_HERE` by the -issuer URL! +Run the following commands to set up authorization for the WES client. +**Don't forget to replace `` with the issuer URL!** ```bash # This grant is a temporary hack to get permissions working for v12/v13. In the future, it should be removed. bento_authz create grant \ - '{"iss": "ISSUER_HERE", "client": "wes"}' \ + '{"iss": "", "client": "wes"}' \ '{"everything": true}' \ 'view:private_portal' # This grant gives permission to access and ingest data into all projects and the reference genome service bento_authz create grant \ - '{"iss": "ISSUER_HERE", "client": "wes"}' \ + '{"iss": "", "client": "wes"}' \ '{"everything": true}' \ 'query:data' 'ingest:data' 'ingest:reference_material' 'delete:reference_material' ``` -### c. Configure public data access for all users, including anonymous visitors (if desired): +### c. Create a grant for the aggregation and Beacon services + +Run the following commands to set up authorization for the aggregation/Beacon client. +**Don't forget to replace `` with the issuer URL!** + +```bash +# In the future, view:private_portal will need to be removed from this grant. +bento_authz create grant \ + '{"iss": "", "client": "aggregation"}' \ + '{"everything": true}' \ + 'query:data' 'view:private_portal' +``` + + +### d. Configure public data access for all users, including anonymous visitors (if desired): To configure public data access, run the following command in the authorization service container. Note that with the `full` value, **THIS GIVES FULL DATA ACCESS TO EVERYONE WHO VISITS YOUR INSTANCE!** @@ -350,7 +366,7 @@ To configure public data access, run the following command in the authorization bento_authz public-data-access counts ``` -### d. *Optional step:* Assign portal access to all users in the instance realm +### e. Assign portal access to all users in the instance realm We added a special permission, `view:private_portal`, to Bento v12/v13 in order to carry forward the current 'legacy' authorization behaviour for one more major version. This permission currently behaves as a super-permission, diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index 4e6e247b..4ae9d5b8 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -4,9 +4,11 @@ Key points: * Bento now has observability tools to help monitor the services (Grafana). Some setup is required for this feature to work. -* Katsu discovery endpoints now have an authorization layer. Data that used to be completely public by default (i.e., - censored counts) now requires a permission (`query:project_level_counts` and/or `query:dataset_level_counts`), and - thus a grant in the authorization service. +* Katsu discovery endpoints now have an authorization layer. + * Data that used to be completely public by default (i.e., + censored counts) now requires a permission (`query:project_level_counts` and/or `query:dataset_level_counts`), and + thus a grant in the authorization service. + * Beacon now requires a client ID/secret and an authorization service grant to access uncesored data. * ... @@ -24,24 +26,49 @@ Key points: ``` -## 3. *(Optional)* Set up Grafana +## 3. Set up credentials for aggregation/Beacon and, optionally, set up Grafana -TODO: environment +If you wish to enable Grafana, you first must enable the monitoring feature in your `local.env` file: + +```bash +BENTO_MONITORING_ENABLED='true' +``` + +To create the client secrets for aggregation/Beacon and Grafana (if the latter is enabled), run the following commands: ```bash ./bentoctl.bash start auth ./bentoctl.bash init-auth ``` +Aggregation/Beacon data access authorization will not work until an authorization service grant is configured; +see step 4 below. -## 4. Set up public data access grants -Starting from Bento v17, anonymous visitors do not have access to see censored counts data by default, even if a -discovery configuration has been set up. For anonymous visitors to access data, a level (`bool`, `counts`, `full`) -must be chosen and passed to the `bento_authz` CLI command below. +## 4. Set up aggregation/Beacon permissions and public data access grants + +Now that Beacon uses a client ID/secret to get authorized, uncensored data access for discovery, a grant must be +configured to give the aggregation/Beacon client data access. + +Another change to permissions: starting from Bento v17, anonymous visitors do not have access to see censored counts +data by default, even if a discovery configuration has been set up. For anonymous visitors to access data, a level +(`bool`, `counts`, `full`) must be chosen and passed to the `bento_authz` CLI command below. ```bash ./bentoctl.bash shell authz + +# Configure aggregation/Beacon permissions +# ---------------------------------------- +# This assumes the aggregation/Beacon client ID is "aggregation". +# MUST be replaced with your actual issuer value. +# - The query:data permission gives access to Katsu endpoints which are properly authz-enabled. +# - The view:private_portal permission gives access to Katsu and Gohan endpoints where the proxy still manages access. +# This permission will be removed in an uncoming version. +bento_authz create grant \ + '{"iss": "", "client": "aggregation"}' \ + '{"everything": true}' \ + 'query:data' 'view:private_portal' + # Configure public data access # ---------------------------- # The level below ("counts") preserves previous functionality. Other possible options are: From 64588d6b8f3e1ef4057cd5e78c9fb956a34eb57e Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 12 Sep 2024 14:37:32 -0400 Subject: [PATCH 66/75] fix env vars passing to beacon --- lib/beacon/docker-compose.beacon.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/beacon/docker-compose.beacon.yaml b/lib/beacon/docker-compose.beacon.yaml index 46cdb249..bd4ad00b 100644 --- a/lib/beacon/docker-compose.beacon.yaml +++ b/lib/beacon/docker-compose.beacon.yaml @@ -25,8 +25,8 @@ services: # Authorization - BENTO_AUTHZ_SERVICE_URL - BENTO_OPENID_CONFIG_URL - - BEACON_CLIENT_ID=BENTO_AGGREGATION_CLIENT_ID - - BEACON_CLIENT_SECRET=BENTO_AGGREGATION_CLIENT_SECRET + - BEACON_CLIENT_ID=${BENTO_AGGREGATION_CLIENT_ID} + - BEACON_CLIENT_SECRET=${BENTO_AGGREGATION_CLIENT_SECRET} volumes: - ${BENTO_BEACON_CONFIG_DIR}:/config:ro networks: From c552d3e18c2faaa5f89fba8fe03b3dcbe352e059 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 12 Sep 2024 14:46:26 -0400 Subject: [PATCH 67/75] whitespace --- docs/installation.md | 2 +- docs/migrating_to_17.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 93cd05a4..07f0c8e8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -346,7 +346,7 @@ Run the following commands to set up authorization for the aggregation/Beacon cl bento_authz create grant \ '{"iss": "", "client": "aggregation"}' \ '{"everything": true}' \ - 'query:data' 'view:private_portal' + 'query:data' 'view:private_portal' ``` diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index 4ae9d5b8..f8195192 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -67,7 +67,7 @@ data by default, even if a discovery configuration has been set up. For anonymou bento_authz create grant \ '{"iss": "", "client": "aggregation"}' \ '{"everything": true}' \ - 'query:data' 'view:private_portal' + 'query:data' 'view:private_portal' # Configure public data access # ---------------------------- From c4e7cf4bfd33abd7b25747571290a5d779b5cacf Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Fri, 13 Sep 2024 16:43:03 -0400 Subject: [PATCH 68/75] chore(docs): discovery documentation --- docs/public_discovery.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/public_discovery.md diff --git a/docs/public_discovery.md b/docs/public_discovery.md new file mode 100644 index 00000000..5665b937 --- /dev/null +++ b/docs/public_discovery.md @@ -0,0 +1,38 @@ +# Public data discovery configuration + +New in Bento v17. + +Previously, the public data configuration given to Katsu was applied on all the metadata contained in the service. +This configuration declares which fields can be queried publicly for discovery purposes, which charts to display +and which censorship rules to apply on the results. + +Katsu can hold multiple projects/datasets that may use different fields, require specific charts or custom +`extra_properties` schemas at the project level. +Therefore, there is a need to tailor the discovery configuration at different levels. + +Bento v17 gives the ability to specify a scoped Discovery configuration at the following levels: +- Dataset + - Optional at dataset creation + - For scoped queries on public endpoints targeting a project and dataset: + - Katsu will use the dataset's discovery configuration + - If no configuration is found, fallsback on the parent project's discovery +- Project + - Optional at project creation + - For scoped queries on public endpoints targeting a project only: + - Katsu will use the project's discovery configuration + - If no configuration is found, fallsback on the node's config +- Node + - Optional during Katsu deployment + - Uses the legacy `lib/katsu/config.json` file mount + - For non-scoped queries on public endpoints: + - Katsu will use the node's discovery, if there is one. + - If no node configuration is found, Katsu will respond with a 404 status. + - For scoped queries on public endpoints: + - Katsu will fallback on the node's discovery if the project and/or dataset in the scope don't have one + - If no node configuration is found, Katsu will respond with a 404 status. + +In previous versions of Bento, the bento_public web application could only show the aggregated data of all projects and datasets. +Now, bento_public users can select a project/dataset scope in order to only retrieve the data contained in it. + +Given that projects/datasets use different fields or may have custom extra properties, depending on the study, you can now +declare the fields and charts of interest at the relevant level. From cac2cf9a9a64dc7ffb8dc478498ae663d09309ca Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 16 Sep 2024 13:39:57 -0400 Subject: [PATCH 69/75] chore(docs): discovery documentation details --- docs/deployment.md | 8 ++++ docs/img/discovery_proj_creation.png | Bin 0 -> 27158 bytes docs/img/discovery_proj_edit.png | Bin 0 -> 20573 bytes docs/public_discovery.md | 64 ++++++++++++++++++++++----- 4 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 docs/img/discovery_proj_creation.png create mode 100644 docs/img/discovery_proj_edit.png diff --git a/docs/deployment.md b/docs/deployment.md index 1f3e2d6a..41e869f5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -99,3 +99,11 @@ ls certs/gateway/letsencrypt/live/ ``` If all went well, the `old-bento.example.com` domain should be redirected to `bento.example.com` in a browser. + +## Discovery configuration + +Bento can serve censored data publicly if configured to do so, this allows anonymous users to take a glimpse into the +data hosted by a Bento node. + +When deploying a Bento instance, make sure that the discovery settings are configured properly at the necessary levels. +Consult the [public discovery](./public_discovery.md) documentation for more details. diff --git a/docs/img/discovery_proj_creation.png b/docs/img/discovery_proj_creation.png new file mode 100644 index 0000000000000000000000000000000000000000..f2508f9f8117e58b72ee6c76d385c2f04d16648c GIT binary patch literal 27158 zcmdSBWl&vRv@J+Nf&>e}-QC?G5Zv8egS&fh5AG1$-JRg>?(Xh>-sbz>?XJFc>sEDF z-yd&R!8)?`UcTlSW6pI#WTi#mzF>X<0|SE-6BU#P1N$ff{9!{u0H3^k`O*y}pX>$1 z6rq621IjQMc#q*Ar0SqxZS3HzXJ-UvVr6Y4%+> zo`b2i6~3aWr4g7E@PdIAU%|)*pOK!C8J~fXlb(r_i2+|$N~0ai9SjT~OiYkp(IxF< z)!6}McCGL7JS{<+B8w!e;VV?Ste{}y4I!&+CoSPoT^Rn|njGTi+fbFBz@gnxe5v1S zEzF1h6I08k*Khn-F8lZL+sBQM68$$VoqcWX8M^itD;?-y95)(J!~aK#aS z5g`OF)c@(^szE_f(Lwv}a{v({BI0A}<)epW620%?bOGqFeY!x#q{|=r!K1yVyuyYU zsE;?5Q%ibW9I|qmdT%%e(^%s=@L#pg426P%V&U|zru-aAkVpt<32iO}9OlOLV-kxw zDsam%UB^rADUB*Pxa)^l05-K&uqZe*%)T)&} zfBxXNYSRrT(Bv2mC-i<1{<^^-Vthxa-QLdS-1ik~fV>_^!7Twfg=JRme%Z6txoW;OEic&*mz!Z4b`Ld3#S zw{ZLM0YJkt1o<(_Yk%D!NHCzira%UF2ATKm+O=fh)m;$#s(xDF<_?(Ak+Aha( zPdClU{947jy49d-tU|RW#yYFa&Mg_M{p#H2o3{;T4K%Pq`<YQIf(JI8{ z+(?_M_flBB%6wBpP2^3aD z_?QP*FKEWs)?%GTkL0+DC~zQo$Gz)ORWY|oEYF^~Z7jJkXN&kp-DnQ%o+TvL)WKZT z-nw>YjvjeC+_Qczc@is`cKR``0LxeN3o5m0XSLC=sFYNmP#7w;Mt%5a7*ymI?apSh zR;hGB>|9wQ!C{<;GfE%TVvgkE7>7@d>xGCb>m(6Xkt1S{+emJB@Y*7#M)oiHx5 zjOEXl?w5}rPS~+oJ41#)KQJZTF4S1nP~9!IBj%N35XM1%@E?*@1kh1;{!ipV0@>u907@ zUeivE)#N#E2{$JQ^QAyO&&sDgY^KB!soFvnMwwTu)jt-#El)Lgev{g&3-t>QetD(Q zZcTc;-k)x?HBhg&fcEL%)_Z$-{Pg*=)(Z6E+Yd@XJnBU>+Abi~+Wty@`<&Ch5W1YuNn;&u zBv^w#@Y2Oy0mWQ))T-KWQDDx+ zeY$?}hX>Cs0hnY3WCK_Dl+iEx0WvbK{qX&(Qmvulk4Qwhq%1?Yq%?3BWU^dqV<$8c zs*RHp%Md2*#2gB^Io&ZtQF)Jc2)~`!@$hJiZ(D+RQ7mmdf~UoD%C*GUl_{N}ua(jC zmc%FX?J1)neE1cQBXa}2yL(RJsnxXvJMIe7CK^(Aho>*}3iM9}Jq}HCKjfLu;0K?U zsx^kEl}MphTi0zrHvt#MkG6z#qFuyJ}%x2d)@s&*!*#)?GQ#pl6((({~P1r;Mm_!++jNT z8b`xq=V>j8JZhxI1J)kbTlOCGjwD?0yW5v|5(eHY2D^iU`4gX*Kqi6OZ`TxXi~Sj% zUhLa-OqaW-CL*{#zMD{_`QrBFC+>U*t5aJ)H#n4s#UzVFBRzM<$MA>Ef6>Gohz{E% z@ProHU>TxB$18P$hU1Lk)tbPxSv_6A%Xqn6Yltino*91(-X!b7kP(is*Zl$-NYcN5 zU1hf`E-x?tbNy{5BLIoh=_FVWy;QR)#_jef{g!g^1B^%{rqoY#BZK43^(00kCg-_A zPkGz%X6HjDn=eN0aC?&?ZnAd^5ZxO+GR^#gio_5>Hq4CyZ0D9jg}21@=w<=#YHv@v zfnM!6IpPSo-fx^S!|bYWccZQBEf)yO2jSs_Jz&m1sm?sFQI9emB2MDU?nTf>@t=;O zR9}`*(ngPd1d zV8XYntIOWaQ*^_4H?Ten}Bb(tn0En&OwL)|%&2$t27O z$$~jYHU+UqfKXL*YP#Oeqmmof7poD>M;-&U&p%hb7%^WfSM~b+$=O0Iwwd4Xgr=*f z=lmAu)R*I1^GG)LRTS^uUg2`^jxKl&IXF=}&FOMO!RxML2>C#_6ngaLzG;k|Ad)&M zq#m%_a#>DJItEHI?!WbPaU&6xQj{r3+xwly|6HiVc6u?~5t-Tvy{onM{ya*Bxr6*r zDI)w0H(nrC9Uc~>hdQeEgTSXLGvvq8M6D_1eE5>h4neVfrm~XJ}p*W{Ys| zMyHdRt2uh;ptA!G@V=rP1{WH9V++v zfEr+1LX2xsaI%83X5d2-!O7 z?#0G5`a>12=vAjp$9;y_=z{&i25g;e7g}fS-ZVJR@Y{2HDG4+{hZT1^#rw}}-+8|# zUdpTs20>g3#z+{~ix~?10QPCdbq^3H59OO!gR(5+z1I~*cZHQ?Px+w-uN9#LjQP}Q zRuuZZs(lw!)+l&|oyu~tpfQcXmWus==_@Q=IL4S1Mb*M|;>GrhmP?8DXxl56JZoXF zmV~!n@0tKT;m_BeeUFFna$*ivhQ}IPNo{Mg-6b-N`$_yGc5pWqYS^w+1f!RZ%v@_T zHF&zT9+g-XS=lC@hPEe?%(e89WX7_#tB6+6Bg~DpQe^ja`_449J-ccakKa1Zco>L0 zz?Z)$l8*81)%_r5NB_9C;!)3)H$-J+3`-|`CqT z3w)64lbxoI9!!Dc;*@_M&MKcfZf5HhkBZZg@ao1i3ecm7&}p3kM9u{4egp&1PS-sq zksAsS=f6VA;zUoPaUOV9BK4@G1a^n8kJ0U9TB3Fk5k2VhIJTF02Wqin)o2uwxIHj) zBs9z)7x?ZyGhDZ^P_5QvHHgjdI$16Y{axudRo}gg ztzV#;BO67yB)L-Ke!pe#BYNZty$Q=r^SLNx(Hp6S!wE)2`F+GVYQEt` zp3HJiv|N7RZEc2oZd$xy@p^mEvq}byV1$X zVn2V9+uGWC7io8PPoAx|*m9VFsbzQHYZ6LBO&$L0*Drk>tV)|5rPC|;%~iQ;Jp3$`Leo{1 zw^E^e8R=kAdedVObm`brNmvLPqpJ;#*&W{aU!RUiX`n(M|Jqhz={6W$Oq-uq{#l)_ zrA8bk4w!iQi-0jT1MPD-sCS-{`QUOxZ?&S?gRl)D->bc{Bz<)6nNq+b1;zGpc8%K$ zdn(X+c~svbll-Hy^84t_2*Hc6W-r|l8}%l(K1|+9wl=_zg)QTE3Kyx5nof3A8!M#V z)kg9Rpmc*q8?dZ4vfO&xES$bAY03zEUviCehB2Q*6KMI4iUO>Ob;sy^?Bs`RuFH!J zLodANk2de!^MY(QqWDz{3UKi9GA*gi+Y{sN`t4@8%H>YNlVak@c(a0}p4;y5bb#eaMmjcDh z-QVRq`Yp{AE{97uboAY<;@W@)R>gQ+LH8cU+zUQtQh|*3T@eDn`qhk~ee$=+T>rBa z5ygaWg=pQu8qsZ%6t=z2;_D6>uB$AIr~T%{$<*!|Jr|7Vr>sv*1&S`Hb>*Z{_r)eD?RG2=WRXF)miM#)ajzy_(j!=rTL=g*zczf zS+qarDaO^1C-WkgZxx@3u7oH_U6+?~>4G2L1)l2AR_&AKatkI_LV5Pr&KBR^;jqH> zW)4JedXD}Z_e-~^H~amV`YZDcw^On&M>qK<)!_8>7d3QOc-*$*d_SlDVCLayD@TIj zb`jgq-z51`R4PP!k7y1+Cw{f}qXsYc8kgyA8QlsPTzZf!=Pv05Ps~~39!e>i&&)E~ zI76*fI+b%UpF6zz_ESsjk82Pn%S^S7n9ur%#(5>jQ>>f{mZ75HyN!IPr~{J@D#LiB z0ULYOpu*UN2r;R`-N~n6a?yJY($Be%8{N~JD_aaG)25UnlF7x?NG4@2J24&&jd;Lz z{JBW@OG-i|24;s&XQZ!q^k6Cf0|8ayRVtcCOG4$%cNX=$gjYYhjQ(7H$0F^vr)Y1M zna%8lB4ZNA>8q=E}*)T3H>bGLt@^71|S3v@)exLHA~0&;ny< zB5TKqX|a2Fy?_&be*1yWb2^N{l4kkE)o}mmvJ^wwtPGJ1XYbz4Mwur(5no=N_3vGm z{G3~}F10$p;&y6Lnx^kgxObIBMdHwKVkHM{JfinQqZ*WDkms1+Xw==;#dQBfDf}wQ z0}9_Ed}ijGo=juhCRfsWc|CtM_q=e~E&;e!S6Ac=M#F>gQG)S?AluiYE9+whFFaf` zyN$}k))nrNjjz9M(SiMR;Io`v7ve^tBU1O1*C?EI$Jj_5@=5Z(v#u0`%A2>e$cTd; zx0#Cbr=0Xic8X5{h=^7jOrOCDZb{p*cu476=KU?R%yjchIfMT*Y;n zG7-l4-eYBnjDikBk zW+UH9)H7^E(2a%z^}y;GN5#sT(@FSbOeV{lna$BZ*4r_gDTXS+v>UD2u|;UG?O^Zg zXX0pJ9FeH>bm(!E(8zQptCgr-wnU^1IB4LFRk>t?V za{6{OOF-$3GVIO8Fn6%b82Pu°6tjF(BOjFN(o@Ea5fHMTF}a!AmbW#3!Kdgac8 z!6E#SmNyJdb-Q5?l1}B%VeKiWa)YPm`>AKVR~K`&&FIw=B~eScK$dN?O4cms2z|It z{<1ze@~2&U(sQwVshZ&H89PY0C-_{kzr@-ePv!|$5=55A9=xtEQeM!fouh3?9FFwk z!G{B*^9cr%SO^SuOX^Pfr$)zH6&MVEn0WA#2DJJ7Yma2sG2I_59;Q87e&QUXf`aA) zJ4VCPj+o2Oz)aAyX!$fy$MkOlEwN{EUfwq$u@D#$U81+8tQGa?PLo5}Dw3PsRom{L zt@lJ{XSNuYoHyHpMbb~!r5;guw_)_6Byc>V|C#s0*7MN^NXVs$As13Xh-cwn}bV~;=O6F7g-4UTMPs~Cd) zsn2|JVh%3xyXe(L56+BRV>(T0Hdl%fET35E@y1k0fUw8Y`(o?K5yb(pfE{mb+Zr+G z6OLvYJOMHDE|PA<2g zVEfV~I4q3HDZ;1CWe$LE=>R&*&dGs>fuXwdCx0)2Z`-M5aCn%Ai79qsVxj{%e1kiH z$fv7Ipt`!6oSb|@!Q%Z{YmJ(;0s;cTp`ny_vMIB!F#q_lyhYSM$pZ+fM_}_E1^yqJ zY<+_l(N5r=PHFZdQ*bE{daA$BzHh$4Nel#eFuyZLaq4&Mj33{6?3!baAZ&H>5%5NR zMKq#}oyi8DUvt9ZiOHt5`NlN3*d7XZ_9BOda_7`BKY#!x(+%%ixHnE2`64YqcD3vaUZ2p2z& zCXqYa^R%Ji2Im5)9O5I^pHkQ;k+qG3qxPYgNMo%zwFGP}oRrH|s9Pr)%VRo+6e9^@l6fC*uf+wRppA@%TDf z(#!ui)+ll!JSN3+kY+T=*gRV5#MB?bHRs-Jt=?kE;3{9^=}#WGA1pE^FV)5T^yrNU zPihSL2)52Dd@sJ0z@UKPs)E<7xfq&H(|kSAuzUgWwDnHhT%F%Un!{~URycp>V@xNt z=0b6syWEe7Bqt6MFF`4=84jY;3YU2X?cl1YJy-?Z;^M+8XH`EtX#gZ?Rn9oC%b#A$&k zXI>3&!gk-B`(lkUoK@y}HYWmF%*zK)#cTYxy3=s9*PWC{9KV760fQN8 z$sC=*)Wi@qHfvHe6_4(MEju@L3cX*C9@2yDlO`#jW+cXV$1vUJ{nsO)WuiV>zfHdj z7nvzWjhdXF!LCq|GBT9W+;h+P_Rw=h7B*{oJJS&w%jjFxha|@emD3~*9MEIe;}0>n z`^oHqdb;7TT~s{!om*1vhc|``krAbvjGeE#xPBaJ4kDm$&}B4Wf}&2es6${kJbR$j+rUgPczKi@~3eb$D^k0TwxmF&H9wl^5E%71CR`+5F5 zg7LJXz?_6kH^)Jo=-U^k>g_-I_F;gdW|f_GKur>lmv7;e^~&i?*LzPAf<%flkK}eC z{j7iS1QoaS1U}dK?h!O+9CeBw$n-ALpd~dlt%p&aZgZ8yEy``I8KEl zwJByfzGfg=1s%QpDzJ`rx|;Ae2RfW_jv4hui@*6&UfIhW}c#lr&w8@BcgYl&?*rCH~n(zcI^_7}jtUW(9&|iKY*X?mq%jL*K_#2DKxf&0&$)0LZ^9%Mu1Kt*; z{8?1_uQtN?ZN2^kOKd#PYdpjLF^?g#KKmt)chljm@c}_(Ee-rcLV}3K5-l}RO7G^? z6N8Z=&UbuODhJ;t_hzq{vIPBkN|1H`@p9L~cq(>&`Y{9g@Hk83d_`bWmHo2=>5y_wV*LJoQZDR>lLCJtYO;Y!NU%~6Jf zZy(9~;0QopShpn35>@2yiqgYE^2_ZO&1S>McssJGkO<*I(~r5qK5o#2X1Mvaj|N8n zKvvSY-7TAck{>ko!_wXg&?0gvlTkPuyhGH?M=h&xk0s+0F<{5ETYnypW_v+-%3TX0 zD`r)JbxS1U6x7>!b3G0}pgPp@#PnNU9q)rh3pvHxu4_5k1jNo`ZOzL!C@bq4pHyC4 zsgcVZi=JDr5V_~foky;viz~g@0ZN+2r%|~TrFq|k2~`CMC*7Hx8J(%;|zQ*rHk?eN~E z6ecF5hGnUMFDN#g>Al5B6E?L5{gFPz?+7P)i<9+Nd{UBX={7j)J~3%zpa2xp zxO(=FQj=RS1(?cQVhStPNzVT|!lggi&{Az+#qs$v^E&JCM3pr$-aT|;gbJuuN5l2i z)9)Jny=u!tMyDN|2&6!`AS(ahgQ>W%r~->d@aQe4*NsBY$@w z5phgm&B}acdS89pU%?)sP3DtNB59}5!2R+uT01tM0=tD|WxqmvMaC9)aiEsdP^mP# zW*rioC!G86_RG&=z1i2={(S4f#^1m>k2 z`v>ngFR%6L^jEcj2W-8#V#U`1RvF9^v@@b?bz+R=sY(?qN&UZ+bI3}0O7d##l^L!b zT!reB{coNe$W5N?+1o{GHQzQl0CyE>gut}!Vyho<;4=+UXl>Dp_>C%Gmpf{Qz@Xg-ZKu`XXN zozcbFTHiU+b!Xu=br8NnI%g(+f3j-A{?N|wXtEY&9q z&7hJ~=ctOOH=0Q|i+$rO-Arm^-_zQ8zhf8vh&K zShJHoIS-PErV*uZb0r4rF9{NPg6!Ymkakfr3r_KrGK;!cx*MV4#0xs#eMP35fL)Nb zK8&~2Nj)E$R2Ti>3yh6Be3jMNTkJgpAvrZ;m5@@l_=!3zo!Y=!A-cZ)uq zD(aT4|5pn>r{5713+w_PT75Q&$p^-g(Ia(#Md=WL7Yv6S(so&PVd%u^qsqxb5g7T& z+z4Vw3kaMv!oXZ^b88>7r@~>8svBwo`|uW3^yUEd5J~?wG2x7_9MmVJT>fj!p0xT* zsyI8x6nRR%*ltJ!TN{vZ!ugWOZ5**U=rGx0VN5S3stYju19=sn;7eQw7Mc=QHmc(y zYV{DH|LzVFty_wb^0EbFuUnf!#cg`O2NTkz@m13Lwetcw8u_i&JG-Tve| zpDUzO)xOQ9TA7HUTiFjS(lbJ~PoEKQM0|&!NQu$ETz!#F>%Nhd-5??-&L!5i0d}gY z#)FR8{CO!kIYc3$fE^k*MpdQjC8s{g8;taLboKA)`=*bM&We@a^7~^H+PYYYFee2Hnx8XB`fkVNB9-?Gyp@TJ zG&(RNZkYicMHJdOI%rnX&Z$&)6CMmal_~h*fKkypNwn!AN2*5X_isf=P=aM?N(VF@ zynq3vY=P6~UvS`*Gug(Qr1fR0oo*E&t0=;fkDQ8h+n>6{c5_o6SYOAkJ_jU4+K7|P zPi@eUB$sEl{wn&?2IDi={v_ve% z9Y$lGcFQ>`gorrB&hh9Qp4A_cT3{ZEvvxCsGsPND=S$6$sxf`d6aNJI3pcWEL005C z$~58#u!`uI(#AzWzGq+~{J+8zNv`DHVmg(Z;mbfCG+@BEH1!nj{~t{D{s)*UEroUk z+B9>RXX`(1O-)8SJ3G~S3ygq(fZpbL=j&U(y@7p099vlRs zq&~+{qDa`;F@~YiWSh+v`v(M|wB)L_~gh`ENjgCO$83#=?T? zi_cA)Hhz3xEtK;PkAN@dUP7pf9sHA6!~Syffh$E#g% zxpyDR4v2%bpPZfj-~l`5;o%{NM>woVr-N5dFKfYqwX3^3BqW4m-~LnSJ~=5V1OQ%y zg@ymsII#Dx9~W9(ndIc;a`W@6e@^OeZEuH!g&E8i&({8&Oq-ba3Ph|1ET7UU)Bggy z!~WpHjeB>t8U{GOI5|^(|Ni|w9)NSvAlrOXc|}G4Rjd9(hnUn<2~p93%W=5wjg7y4 zz#ZSdeG6@yll$I1$$)`@VQ69kh$5h=mUr?mIM!FiAsihY-FwEsseSC>|L)t0?w~6B z{{`*+cS!Jm{yW>h-O;W;KY|}QZH>aM!FV1?1P^X5j!%}DTADSuN9P@q?57;Pg#tlT z>3S*o7{A5WZd)hoAc@%AK7gkZ1qU<6in?7Fk_ zy(z_!0&B0!mZ74^8*h5uJ2unn9_R7-TSo^Vb{)bzRHhJR!|o0x7I@?p6X)IOUSR1k z)OW<53%CAFTTuwo@;G_kg|j-J3C{%rN7JVg%!9lJtsh0qlADgv`EJx{Ws0Omcjh$h zr|MZe-VW+&Emecm9Wm_|Sr9G{i!CHUPJK5;Klf1_Z79< z+qf(-N#q*%&E5De7>1dm9R4&F@O|&TlVLbp#rhdNldJ-|r60vw8N3go!Li^_Xe^Lp z!p%r_9%^u|vAa?5(4I(-Mp`qrX+R~OEtFn0UTr)+wQcoKQk!Zx;h3TY5uir22W8O2 z7HLt1^r@^21c#)OgI!n}I8+c@VKBuJqGv3n!(!#P;?2)Hb17;CG)x$OB$B@ zi=WeI>XcQd2K5M(q{(}%vuzrPLqwQe$_htxI^0xJTi)TSK3m82c$3a{h1<8c!_x54 zJ2U6fnvO}o%jv}!-d?9MY~HzU`_^<=^={onRQnayxveRVkSR4;X8rH5M#^=UcVd;- zEwi?1d%OxH&-ls9Cr+pG;c^V71-Duz@N;9YjP{@%+VtT2vdvX4JU!_=_s9E{+0u+_ z0#CbsoBa!r-N2{r%}#du_`#s3%f%BqbPp*!PLYxQphHNGyV!I#Z^~;w&WReMg~9dy z7GG1o^Q=CWh6&caxC|r>j%FUFK(uGYe&0+_Bs`;u1#GwQl-}ao;Uwl-MwlQt;~jAX zbSG?hgfP+(eA|UjUTM6Eg9wQReb?t#5^_dX7<}WsPxEmcr9d?I$~~b|x0!#KjJG3qDeHg#2tFZtD2j zDuzhlT;%26?{sS-X37}dmh_2GOP8Hg5)W4iBa6rDho_E!m+dhPQ%%)D1z zP&In1>-B?|86wVEbC@!UKo!^Ug#JXCX2LU5EShWZ`9v~sU?HjLbwH-ndYIX3ZPv#s zopM8mAv(m=vS~jHo;YyPR&aLrsl^?U!Tp1hcG_~3 z90gfeKPafFCgK|VEFpxindY^;Ttw3Xot7xZyOr5b4R1yGRTV9=lG5RH4tY8M_T`NX zy069VTkIX{_S`dKLtIVHi-ySpo zK{AaGZO@ctI%pihspWNNa6s=A{brn#+T#yRnfpptPKEbaT6zH3!=1HBVheWs5MCGt zM6@H3$9u`;)IJvN#a(;05 zt)z1;WO9V?i^UgdJ`Qvo@vQIPceR~J^F764EBcD(MQLF|-n!ip>-8c?E^Y_C1W3NJ zgpToGZgSqbV~Vt=uY<_C1=`zA7T_+Gkfi;|HB7&4{SaHPgB*`W2&Tbte}TaGR~S5x zGE3dbFNoD4^%c+Rr-OBjG2eLb%aJysQ;EJxaM)kt@M}0f8#Eflnl7|OQs-fg_sBe# z`4}#CR-SIyqfEeX)uZLyqmuv2n~v)qh)^NOBge6|XyKRrjnpw81~CUrQ2qZ~b1c*( zd5R;e>nikK?jK3HW}#b|_&RT$Y3rRx%guR;PYR)Hm!D>9c#OfiKslKuq2-mz8_RqS z@0C%XF056Nn>N}>4{Nde#N>7DRBzqzvdoxFz!F9Ju>dJ3FY6AI=RnIGC)RsOY%RO! z1#>yz>{c3y?c`Fi{6Me!C?f5S->KsMCNp#CZ5zaQFE*NQ`b&>1ZzOYNPk}af({77f z`Hu}^#mXbGL@`h2eoaE9$D;gR$6CNejn%{~17bi(#;iuLnS7@Q`B-HJiJfN!*D@>pOBq9Y zTJ9IrNJyR+|+QB{j z^?l8K<^IU}-we@)HMj+C$8YPB84}6mt3{s8<>(@ooKB@9krg7>nY9Xiz=;P**GBO4 zjAW6F|IkUv_U0ZCstrtdcagR^68S4*G5ixLg(m*;%j;q9G_W2!t^iA1;uRZ^w8y3{9I6uwT2cz1Ta3O%eo zCq1vB@AWtpIM)!7RVkf(FW@wPysHCMmzlCHCGt-F*c3e+RZ>!BxXTN%7`xscUL46K zt0=hUL0a(avn9mZ<+{&5>|Ip6R^sYTBZM4u<{tEs@1U<;J~WQEn-OY z_O4H`uqH6=XdPY9S?zCFrUC>HbGTt|XK*igx~DWTtWDTgn}zIca`W}xD-!q1q|YZQ zj}Yf0Od;e8<8RnVuw=9*i0;lO1XL9pxgj%=fjoz5wiKD+X@Ac=@YcGi9)8{+znr?; zyT-YN1M;Ea7aR7gYPX%)T1lxqqhTk9j@!@q$o0F2747<}B?>_^opRP+t>Magt2ecN zD%L;@oYss-W6w<4F&@tWM(p8#oyg&GG1U#zFaDN!5FmWZa z#f!ZEPjvSoQb&?rI^OPnwqf&4`CuR#*VJ9c`Jy#kQhDIY{o8e}m1hQS#IbC}YTF38 zGaerA@?*~{zziw$J~jy4<&8DO-#=@1Phk}>-W@TLxaEf*()38t^TUJTE%P4yn8)CG zbnO~doLRmB_|V&LFG^4xo;5#bm`Kz{&WZNQWRoEz(U(2OQYnLScLQG2&$xD8plCTH zR)sc8iJ?QQcjpnsE%qxklR2Stt&4vE%G+N*>$Z{3or@TpNP3RepUgxVSpB7NkQc%| z(~bR}++4M2v(j3L?^O2nQwKSIP7|G%F7 z?p2J)n0xvn83lpiz^k4cq*AOYA90!S&92de?WKOQdzI{E=jg_^Nok;#gLbXeo}$L~ zXDageuKcnUBURYFnNX z@>v$rFBYgDB@}a%n=}s_RlUC`l?(O0u;G*d{YxAHC?*qGKZr?4Ubh*<#Kf>U9S-VS zaWacG$qzTM1D&j;N5yxcil#I6+ZRMndd=G_K`Ma`2sy@iSBi!5QFpJrdb5LZe`)%& z6N@G-==o&aV?~x5vQ1CjO_vOq&GjW?KpuX9Z3VWBXDUxhB}3i?LV38#5-X!KDqON; zx!eS|xP}|!_K$z0pk&54y&b_7;7Xc;pX`y_H0B9&Eifn*%_S4(3#z#Yc4e8~=t%F+ z1-6e+D0HegBRD&{8Z@@)KWIW?tTXs)BkeQgpZnEXYG=A1uZ&ai;q|y;oz(!WMHz7R zq>;(|@EBD=b#=;T!CDl10h4uN{>Yq%*YM}=u*M>}b zYMUs9%s-f}!q?M1r`_$hZzZ!C+jYSOj(&-*c{qIV^I$dSl}M1zy$!i+<-7fw(ATZX zTLt6lw{$(2Q06g7f{a>#q!>zK&(!YkhS3^4o^r((~iG!g&=lE`8~O#b9ZA zb?_E=0lo94-CnD@kI;F<_PF5rnlT`waPfzHBLPBWusQ5@iqCcB{9rFmI~R6JbAG@#5BsD$}|U6SHW!V5gklA8UD`@rX4{ z=SU+L59y{zGy(QPC6^PEP81ZW>Ah9U6%#*vCs(Rqy_5*sNvFc%);gIy70K){$uW+Y zs7s&`iT_fnT#aHiV9r-&Vpj1@^_%k5@C&i}&8ftzz+&aoPTrV;%mlgD2m3kgBf{F> zY^7AEp$}lFSBfS4RXtGfnNy0^z|8Nahk}MCASC<+Xj~n9QBhI*1JOi)cW$M=-cX&L z6U{b>>2Cl<-02jVSDe*M^d5@6&70^ah9+@&?gP8bsIs5XUz(zj5O>8NJNejLKhk+x zZyX={Eg$YQGxXy~8P@V|ZUt0nzYNIJ`-{6y*!P+Gaay7UsF$YAp|5E<8mwLrDoZQ?#3tbEpIOJmnkR|A{_pE~E~6YX|~ zsto$!0cRrM0L0;Pl5O{J_3-qx*8C1JSZia54$=6yPjA1AwwHJPMoWNG*UTVAPo0ry z`MZ8|7_9i^o+^Plb*xgP*kGL}1Z}xK4}@1hPglQ}GQXvEXNgV*geW(~?vAD=rg7MR z!^Ir~g35uAB~qCT?uI(JWYoxNG%D%xq*rWicM5r8B;IrTb~RM7`s`k)$i;Erg<;!n z*B$)+QXvZM3AfXa(_I5n8JeVWB6JlRJCSR><+ZSu$?#(tA>ABCvp}3~gTazXG zW#zINsGOhw5evX`3S+%79Bseoyhq(XFSU{Gt5bC<0LDAZHIa1z(T-aaoC8gpX2N>C z`{3Hx%{fWjPjl0*8YbUhMR50e1s6t$!@liAzw4LD@v}oQf3Rpb^YZTCGj<0dU6WII zp2T!NdeaL#v=vf+GLOZgdjfXAml&Eqyy@!C)iy8tr6JN?r@{JLIhv+w{wWl;lzQmh;3$aSjmlXEG&!&@UsU!f~VQrh8+qXUV8o!ii|!Mmy{%CWkn6@=FbucxVXG*x)huLowa!YfE9G$goO<| zwo8`Fdrn2z?oO));5cO58E#I;@Rcw-z+VuX6in0KC2K^4e}8ZS_ur!^-2dVz7ukO@ z{U>Yh_wV^E$bXR+!Wy(lB)A911+@5_xxaM{f@<1H{JRHMkTGo(D}a+#dUtd^>)g5H z(k<`12dwoB&@F^9m#tIwaV|&9wj7ibf6{)vLzWe_)~}J0D^^IQsHI^8G%NUbm#)0_AGq2k$a+(|(Cb_0`dYOGoFUO_7Xh(IEk_>7l?hxpisLOXW9 zep=!uOpJd0D%lP<6lII0uJ&LeZ|`=xEhs8h%dATt7J67I<#%7$Nc7)|E35_kxGag9 zE7%7+yl1W7=a?*URC2JXqeELj)cCL8&=T|3*{ACY*cp{-G>oh-doe!L$x`eT?6w&J=tTng=Vp~;eCd#@WU#c zj{+Dw_+^xe=by+6XUhLQa{q6JT+Xd$Jg1un;p!@(w3Q%*yC-jC+YgwD$i541xx!0%T`8pPH zW4;!17DlJ}HWL5c;YKWbtIizQH3m-OK6gdOjw0#oPcD+Ku7d)3n0R+~xsMN)tzlud zK2#&l71Pcsg{j>*mx~&|cA*HWz~y*q!!+hRICWmba)0gmZ5zArJp6NwqAH*u^EnNr z=o-;@iZf7(o4-FRHdvb_HQu@|w{%s&w@;6my(JA}nKmfe-aPRFvhGlmj(!*1ohjc^ zGpuP3XGzB(ZYW%fzhmvZCU>G+GMzI5N2fzDmI -Q%-KsZkM@ncnf;t0so?N_5Mj zLp`jc04>@PCw;&i&9cIMY`O{#9WY93v5%VrO|j=Dc*Hx=XfAnH(BjAy;+Y?!CT~Xm zS8ZPz6-T#in*f2}ArK^3NFYFPcSuNZk2HkfPH=17HMqNkU>`0WAh^3jaHku0x9;Xu ze=^=X?~ZfFxMz&}tAA8g?_FEgntQD|SD`iQ#ErX?Xf#livh7~E?qA0{#5!{sZd2@3 zTguYm3SJHxz)~MK3hl`Wt20X<|47)nFi@FG|J)k&r7VQh&5c6XdMsMUlnRR~m zVyk4!>}ki4I4A_AZe_TKIHq-$AVYhk**7XEAA=7KRD4yxuJ^%pgMq1O(cTyls<#dI z9BcmgO8wDpS5$a{30pU zRQArU;l*3I8OXPyPmKA<{sLh&(`?)IM>t;1VWp9{oSS4OtZY&pJ(TaAWCW+869Y6j zvJBYf!<4?bwf=4v9>Sj^IfRbSu36Ngnp+A*T%7stptBwUO7 z%5#kJ=SiJ1mybRK>O54;-#gC~-hJrZ7H&17idcBO@gWJ@scNy)$;SrL_AbKQbN_V$ zgqv@OW2VA}(Qb9prDmTfu_UUq8cLt7|Kv!)K?#`lHMSIqNT_;9l`BVup$&am7;Q== z*41u6#`~@l#9|tWtqu9U$JEJbXau~GJ}8K7H}1}Wzh5UGFK^h!)b*-#Xqd}>D(AGx z4qk8j;*_k!7jwvrG%QZw7zTvzxcvH(_BtI&lqI_+j&bBIny0Lv|J%*N-nnJ zc9wVbsled{rGo?_dAp!^tj_Lexk;%8huWoZm_nT9N*p>(voc!wwnUpNxOSPmCtQPP z--^qAG~ZasU?WO@G;6eGQY_cJp6DKFXsu^5d$-mNLHb~$SdWmSnLwD=ZK8O`X2l4; z#>x9)d-f@~XqL|$AG2yT>658JgTx#(r+$qO2^+$K5nwBkF+DhYxk_*3GWWN#kto9p zIf0FoNjwkzXC3^VhV?H}g!ewV!G9&?Qj($M((1i%+WSU=1wfe zuiPesxIc#vgl0BmIzafn;H_s|mqBCvGf)wQH*S&S12dWh*G8{7`j?l(B0Ee^)b-@# zZ*{#}+4=mn@H@;BZKD@!+~Au!-cg5ICA5nVvPS2~ot9Roab+mZ%{;?L=dR8(hM0-N zSEM~Iy7Xx5M+!y^3DvR(aLG5ruFSRS!SV3Z z2y;Dq@vnp|0{eQ)Lbg4$hKT%9OVP{_^X@A50TAdwtS>OFHyKz)&fEZBUhLvPUOAD# z-3MHd3H|-Y#?|swz?T(+`GP1l=S9Tc%?@3tv2QOBRf*Mc3t#NQ>+o|p7QvXd?1CyZ z{j8nKJ~US?#;$%z)%tTQ3!tqnT>nL{TzhjjNe1ka{~odY8^0&dBOv4Lk?NyNJhOJ< zWhIA;8Cz;^6+T)YPgOZ7k~Sh-3(Rvpd3H6|R{}lx9|Es{hUy?pRL1&=jf7QZr(X#! z%0r_BUR8WD{tcg`_dKV?Ssp|-qF$NnZ;$)=1(i?~mmw@HfZy8BX2^uZ>{7R0c{XeU z8jVY&Kt6g%gfsay#CtK4*Tn*p63StXYno};<^0wR`5NnFVItZGlW^R2g#^@~uZ{0z{u4oA|m?e}^ zO01LXe~uv|wn1JOi5$La;-e5dwjga(^%c{Ngym=#GSbx%C}ukiT>M5D6*zC;pt?If z!`*OS&;Hd8eWcd8nNBHFpp~kiu6_|5(Ucal{E~;y-4t|TBlCpi^&8Cd`Bs?eW+i7f zoi^IZXyGA*#+Y>FT*iXyVo}M`t|m8dGk}`MhXA3ovU&o(_v*Km%kim(iI+&&ADHEp zXWRKOp7;IR0u|t$jk&}1N%eyqFYtF4bsG4F5dT4FM+r%I{yxy7h3l%o+D*9hKJ zIF~+k?Pw#&_@ES&6s-BFAeqA_ETiEvuELb~Q?at5XOW82kKuZ{)}d92mbC<+xyDP3 zN&+%#;&2~OdW(JU0LrE8sS9*GX`Aw7m#qK9Fyiz+lFx=n{#y5|8`$0?IG~eW@k)g1 z^T36{M54}i9ZGu?+Xr+3E$jWA{=q_hiB40dG36BJwSS)nr}sU|?j;pF+~mJ4ztSk* zu(On6&TTQ|;hso*1aYydXajyvmOt}VY8(qB2IW6k)U7JD3|UY5QLrASobX_SDz5^A zB;V^avQd1+6NG1e-y#XZP1Gu%hm*$%w$@i_uO4v-4jdMWGNF$CHyl@s>M?wYt><96VoDQEna@6Yi)m%lS{~3 zj{x0!A#TU2mXh0kyif)Uhm}_0^4PoTcnh20aCy+#F#mW{X}AbXoi5!_lVQf`l{C$0 zlx7OzISTZ}n|~Cw9ci>vp}u_#)rqyO^1M+M#!KlDzl<0F_IwP%F_z||t_|%+?<6)NTTb`~%=Np*s5qEEBzC(S-`Vl__-cRtLY zj4Pkm0SmD4Bo_#I^-I=GO){AV2xomHpI1I7cm7Oz zY7oHxeSc76&C4CX1#rfh9bXUPJZ|2h1HPC%`@CGJ#3Yk&bN4bc&mgD!YRXdXdMW4X zcqIaBL{wXlGpbokI6B9KI8;es(f- zw6QV9Z=o%L`vywHwb-Drt^vw+%?iyntD-WB!bX8e222)?`z%&O(|?YqyJ zl#7AVU2Ljr%$OXyt)9Cav`}#;KSPuI^;5;Ng&w56Nl{e^jka$N=p#$N46c1}HP^l| z*;HR-m3WoNCJP(yVii)0+_*+mNgAN$iq~06FqDwcRCi4}6T5hTr?9zlV*4lYFEv4K zA!itq8j=B<*^&9_m1?DynkCI@FJmr4)2qSt>R=jU<)){Rq=VQ=cXh)UChdvUK22*z5t3vEkg$ZSma&@0t`;cPT4j+ zr@hn4<&Xtetviy{RC@lW*akj2Uf9qJwx=_eQ@@+|y|VFctTEFh_N0Q>tofCxl@Vnf zDG4#cQH=T!WBb#0fto|E^@kR4 z{_O|t!zF9A+Uiv6;fyCPBt65zWuhWhtX?F=l+qfR#PDj4r2z+D_6iGK`Ukr~V?TXp zmktVm?bNq(5Nc~wJnEaYlfDY|J>>D)zfV9d(mD6U!R0o>ZZPCLoD?MacEG?oy=Qtp zCq(o~M;?6bLmgif$$VSqVJ#6fC^yIZXlwymxD=|ebV1DFUFWlYtMbU{Z zL2x;bJMPe_!D=X>5A)Fx^`4R}^J6HSU%5|nbm;;6!Zdd*CBDELQcJV@P9m6gQF_GG zKAGC%6{jt%INj}*`W3GeQL;n;dxT_#`=C&j(J6z;d9CB!22;#&bc2zS8=iMqvs6$F$zd1mtP59`0Q zyCdx-{jphPD&q3%+BtuGET~Mn09ia_XY1P8p{l!5AEmTfmD>yZ zYB#@39>TTWnKgjHexM2bMOET{S3aAC-LO}x`(a3_V_T)&D;p$%7$Z^|{$tu~Bd&7& zcH#6f*3@0M9~P@3rAWp)1ywVEY7Ria)Ahs*nZin1kvP0eWFb+j8Nzs9o_#tflwJDu zAzgJEIB?3Q8pYVc3=muF7-yM-HXnlCmm8BK`8t@!OmXHHR~C4DUtOLVYyHk!=n;dFn&J8WChlWe~uEr1O&zFu(SQ0+%i{wkL;?CwL-*?Z55kJ=Liq^q3jRH8S@V%CWAh?-AEd@b1WYW|1Qhk|Uw+6nyPO8+}iY*JOaA=2qq=b+;ABn*a2S{O;Cl#qg3*l`e zaOu<4s7IoWB>wqI2DYvojNHH;5z$C|3gYoV95t9|aLOVozil_yO`j`S@@`2GPhV6g z(@69|GJga;X_XwSc{?L?N!%@}5vawO#I=<|^ArB{uJlq1F(jGC*uvnoPA^_WHM~-Z z`|aYyEQtaV@+NC9Z_4LTs7b#H74`1LH!8F}Q^C`ffXh8!ZO>DQ4H14al5pI6blir8 z^-WzlzJ?@6lY$m(4G)_Tc=!T&&0^z8>Vfvils z8@~+GJ(Ane9Fz(3yq!6eSD`T96bAcfM}|6DYNu)~xv>F^dlpL(9b6c1=6+{Jl>x>GPfquLzFrn=wXaEyQ3g<$BQg>{kQK(ZoxKd zn};&T%W@6x-GUaz2C42Sph^4s-e=~|FG}=w>L+0A+OZym2eEy9LN*_oKM7>&rXaCr z!L+otFPHV~r2nz7dk$~miE@sfUQ`g3dajWYlNDTaiW<7i-F2~v6h&^(-k+HOb5w6V z4p$=t=2maA1*B4)^3Lf+tSC2|>yIyX@H}qm{KyFCc}u9JveNsP!kWbI^JVEHXNFeKo*N21Q&KZUiNgNZl@)RZ`BUhh(p$wI@v-HTA_Wv8IZ>+AJ z6SFc$#>EYffb)G(3qv;bZy}BUe+;|1IjQPqg;u%X7C~x1Pwz=15~A&-P?>;*mlTbX zHQ!F*L@%3fFwo&T(B3~tF^siVQMOwrsjcHHW{uBt-ZId`&FzVFJ5$ST)P<#u1R)tv z@skDX_=rExfzSk7qsvO)2{}v~UQ}3k%}M8>3@J`Lv)oZzg5n)ZP%d8j7%J_tW~H+& zbW%PMM})6@bdXcpM|yETY%&ma$@yWL0FIIdf)Ak%GhMmVJ^s3`b!E=SfC{a=b)l>I z^=-~OIyv@y`^TR$RA=(?cS!&$oUy!e`8y+YA}?rPr=?eEW&@xJqB^LNiS7%q(s;wZW5)ULTuJE4>;5%De+xq@1 zBZ|Q}9Fd%7vk7hZ@TAj9$vm{t>{p@_TT}hqgWk6-U|=}q|4ZJnLGH68Q0u(1vQm(W z0O!f8fT$>y)L{|FrT{8Qf zmW)2xo7B=HG(XB-$AlN`XJW>l-Mfdag8D`Pu#`GQ(#J~D;MmXt4xTwP*BMT z;?C2zU@rdU7n53*dk=fDR0RA9^0Hm{Wy6Cb8uZR2H7E}=ggsB*)KVRJ3jS*P(8Z=U z0<>u)>P!#OJ?PA#waIHm_fHlyq&dxxSf6#n9`vZ>@ob)t;(^RFVUPrFd#*|}bAq4p z(&9%A`az}iG-=J&@-V(sK0RZ`1s#%j5Bv$uDA>LFWP^}2y~H29zI&f5sLXK39$~ATF(fMAhQbF$JG*oQ{p!4bHmwOqA*M z2cNH%z7Z>ondlw-66d7x-PE1Q>z<}!5HzXYR=GXM-}XjAm7ff5-U>q=O;edI;0$vF z3>VYcd?uem+gZn2#;vfV#?wMmvu?$t*6d)J{GKb3F7_a9vpq!Y(Kwn3-+c*Uu)5y? zZQCmr(chSvMHmclx*ewQ6nqJ3li^Y|)H1|N>2LgA4tbm=a~oD4AwhGJk1bGf=Ek| zSkB$Q*j7s2oVycGXTY9)r%hQVX2%aaW?-Dqo5t=cVcHYKmI(>?K)=6OWps1M11^3c zj%IVGRL5W?`Ru3O&PWld^7ZNDMxn$q4fZK`;g_O+e3BnhSJ_`tnW9``7S7&#*w7a4w5-#_Xylj{Gv@(RviQtVdN1v!~ts&GO zMsumV*9d<4bU4uH`!V83S^Vm-pyCzHSH2*d`i`-&3V8RfrHLRuVE1IZxebNf&O}}N zlNc_YM9f)9_P8@8HW5+7#{NggeQ&x5N*~r$BrW!&d(+1K7haP+bU`#OMaE)ccUzhS zdr;X|)03S4!fB4g4!Gb?5fkBhkw8j9SYa@r2BbY4>20^mS)(SNC*7OUW3p@?-r}XQ zY>gbMfbpg)1&x2ozTqOTq5z_yNRn_JT)v{$MCg&_!E>}7@mukM(C@8JU&?S>t&9I4R$`TY zY!`|4@!DN7Hc-pe9)XZ>?KlWqv|V$o6Z(^P#lU>#yctiMN9mNy`?6Ak`lT?KOC=DV zS}*y9Nk+w}i@9}Lz@}%i>5VG~*c)M^FO19AhQVr1Va&TaGdtkTmPrX8XxHudK)Vs@ z+fFKI5FBZ>5;Gn>LSRj7uc5!oS8r7EMs<)(SPc8FX{|xR!7t`L!s<^sC&j?Se2wGR z$~McxcFtCz+SjjnGXxj;EfY01@OL_T80&fTy9IvtAL5$kk3rfxS^66;CgwTuY;||r z%=O4I6VVp0^{-WTuC+YNpsF|Q`_^lrz7Nrho&+2YYa^N_(aIEr#xZ)He6gcPK6t{| z5y+y#yMiP*l7Dmt#{>1;zt4P{Sk}ec=GYMoUGYxg?b(t?mE(=UGbxbXC56qMP(zb? zw1{-lsgRWd5wui27rE$`NEY9(gcsjIUb=YC%nu1J^33=u(bc zvrdPqpjTupZx0=IS6^awQcnO&@n~ z$a`DR8cWvilKapd+?&LGBRBNC!u*3gC-?njxat9|j*>%9TecBX)<_OJI?l#mBMlWQ zvC?OIA3t-ml)&1{EuWIZgFm=hqyu}oGBoS%QkTZnnQ)S3S60fM1_#DBTkC~hkZ!$p zIb?EG>%DsDkhm2mCs5J>D*d-)F5%RBy z&9Rs-+5z_cBf>fySwShq8ja2I+W6PeR(zdk?OHq6)&+&=JXPj4o)lu$gQ^6HE6g55 zE$6d5xml0c1W6? zjO;*Dzr+eZpLE%F=dFNuBx+pV{dWQB_gqN*F{wID&4UCL(4res_gT}W4vMn;zH)w_ z{^d&(Fi%xuXXm>;@GX8+vBnBNuXGvnKeu4x;Q`Dlen?u(dRKS%V{&qr4(MFKzv2vl zzy+WnpzI2F1M_DPP$C6jaee>(oswoghu|Z#2jx?S!jH`9_JXXmxgVE5@fu42(N}-ycVxB4tl2w#mT3Dww9c4_i;KeIZ zIT2cmeySY)?9o!vD~)GS-Z%a4|3qZ~b9i9=W*X~DQBa;3 zy?lW@^Yas62>J00Br2`)5_x#OGzmriCUF$ka8$N6b96CuFhv2{*jk%1gNz+aO>IEm zZ5@wM+eDCsK0X!_b1*e@G`F>(Rx!6WMUg{(VdtS%HnpSXVB_GTX6F##<`rOLr~W4Q zvqEMO1%(<#`pah(*YtxW7soeKUA?ErRx)pYf5ZEVC-9EuVi{M_9k=I43(A$=CzS&% z0s_U?Ud-_FY+Ay`uZr%+87MSQ#`llRw(kQ7jeOrt25o~zGR2&?mm4_oc}ABxER%E1 zX&(>04u@7U>c?+Ac_(zpk9F}k%U_U3b7*H;DDogubj8QT#XY4(w|y+7_hn5E`EPgl zhvqQkzpgRol*{Q8RPRcs>UZo<>_5bO2R(;p=UBim%G1W_x=@B2r)r^WS{06?xWxI zW7~mpD~U*84Fyw%gh7P~>@&M|II(>`M+UHWtJCYgCREM-=e8w|YmgKSXZ@3kfGMG? zbV%LbPsOa+syQ!(tDh_kYOphHg%ztszNN4n*~P)*rHuyr*ba0@fjNY`OytufWGO^4 zDz-4D6BNc5!NmM~9*Cv%FM0+}^SZD%qboC+Y?cL4qK{qQkGC0F_-%M{LRLQeYHEq? z)QK4Wtk^!W^}VV4`O%Dv@OvR5BkkQ31V{?`R)+4B7c9${i-8 z2mo4SoMyn1I{DP?rqBD&4X+Y7n)%m`RsSA$h!qq}3zm<)!WvZ@O?wjTr$36Ld1W|S zm4qvf>?(`hpMx@oc8G#lI$#xpSICPnfXp{2pESm`*2X05ZnkP9;MKt>_vpI!Kc$(7 zCJk~7T?}CFDvc!>Quv=Mk*XD5*8k}BpgZf`W|X$~=#R?k*{^fnn64nR%PN;ZPJK-H z&~t+2V(zB#wq0+LhDPB4kp%~x4^?Vp-5yj+{tu%8=2Y6?TrYi7PqPnE zYzK9*$tiC{{bQ5s{(EkZMj8{R{QqdIxIw&+=EnN#|HRV&^Tk%cU$KjeI^*Nx{R0CX zyN;P*4F8OR0Cor}008j!_dmJ3WLBjJHcy4_a?!H0#~d73`3D5Ndik<_%`RQu>9OfV z%y|>iYHEZ91qIi4cld;aYU={>jLnD3Eo4ngdW96>$gb)P;^E`lH|VI8Xw`h#`bx2k z{Q2a~B65Ddd@bN|D^$TCbO3<{Mn=@^9tVLR%h1r!1PF{vOVd8!_=9e1Zf^bz4J|1* zA6ZxEo7d#zvRYbN6?Lqtu0sO@G}P2Txx^nE{@d5r7sbKB;YW*CL1)^;?qs3T zcF*z=X#13gvCd=fUF~s!e`INc*6wIp?s7>bODq_X# zC&E5=KCQ8MO z$ZS4U*e<+3?~<*%^Mr*AK>#Rji|N}-x_YA-X5~W>1N@y4(ok3KuFK*m5|{7;SsyQd z2xwXMETHN1`Y&H?q)(8oi-&xp_3hpRyXW66uJ8i6!&ySdwtZ%^?k1GDdwev-vW|Or zjFX86(RoPCZ;6tl<~o{(?-Y0R*uV7R`jU0wX@sbRgFVxEc&`qmPki^OGObmU$a*2r z2|>ZvWFCSm@6&ukHhb`0;qxP0VWn~zvu2JBYv!#wG0v_U^`VB!Gn8g+9;{vqJil#9 z8oBoPqVrmHq&sl6T-Jqph5lwaW9=&n=00CKat}{w(p`l|dKllGjn!J23C~BTLNd*c z8puouk+YA{&97S?eaxQkHo>__UK%Vpl43p zYSrwr`*fJo{p4UFU>A=<+{BUS$YbMk>S0FrHOWT}TJ4EO2fU%A;|^5UD^^2AQPdY# zWdpbG83&%*l>x^pg-EL{rB|itLQ*JsZD4C5oN<8!E2nU6pJ@E0tX6xT_V)I!)=KrW zgM0PyVneyXg~j6JYab?zgT9j(-oKS<1cj3pJwJ)4{>UIa=@8Wk@XkBw%-(Bh!2%F$ zK?v}ZWa_QOU#nFNA(tA!dEWz=+onGS_Fmt%w1})1TENFl7GENm~*CnA7xk$c)2Mz0bH{B^7E4 z{=*^@WX4qnHMu;C++6r&KEV(zC!-eq%7j(ByO)%CXRejTn=!Gx1MB8AT(h!`&RSbt zXyCIMyQv*!jm4C}MZR*@^nC~?>jm918E1TQ&ThY`MgSiJRY8AMR*fahuwW+n*86!B z6I){a9-a1#Ep6G~%ro9HBWu0)TGfS$)Q&gf<)&}Ro75@=<{%^7`~4r3!O0eVYSOd6 zY}X@UArp+1lZ^Dp1#FYn!CSzVDS`N8a!K|8#$M*N0=B2XXoe2Z3Qq)BUfOBRP}u9R zgBOBYBsgq5zfb0pB&sKt_fDZ2uQ~XqE=$zZ4|f0Y92sx1DP-0jhL%r9@a;M%0tB${ zQYE!1j|{8kKPt1w=A$Emo@V*AwLMf&L6|tIWSO>}zWgv-1iZ#ywOt(7qHh`)9PYfh>{1_+Law))&qfyd(@Ks?Fq;s9HlD}UX1o@G zmr@25@!3{CpL^{0CBsFD;OGfe&jL44`BLlNvv}bR)!NYdNV8y3{pdZSlBk;3)~FUr z(F#|?{d)#|+5!}>culUF^#Y5WeqkqiLsSdJ&lFQACB-CV6|~9C88^&??{UL%O@MBY z*k1W;mUf5Vo!n*eRWf{Xlt_Z(^j8Uq%&B}O+q7*6ZIK9ZxA)R28!0Rozo z1M>CGXYEySE;|VEZ+*Ngfr{2RX-ehHbZAYxdlxKbs#Hvp|HaQ)&cC zv{el*yQPc6tR&*Tp_VJwnTwqvZc__9*1xaNo-oxcN_6epW^0bT(eqjpOteyVoE;BZ-{*V~(y$EF>99ua(&XD?mr&5}Dh+J0(!HQ~YjY&OB1u;@gX(RKX76<< z?8y&*?yXUuSQGBORD$)TmFTlwZgKC7y`x5hgab+G=kTRmLfRK1 z5DF*EGYm&MP1>^U0M_6!$K$-kg*?>5rsa0f!~rJpJqef_FrH-gwpNZaqWcSS-OPio zRc|holT-|N%<4BWGY@683>Z)E)v=j%8c(huXk46Z6+%RCPUVgWnh{%M7X=!V=2$;3 zC-m8%y_LgmYcCpWk3St}Q&kBc*^7{IpZqm?nH4O=g%GnTYL2Ko+EG~1&IBf|Of_>J zDW={$6)9;zKwp^1xuyH@wn5;Sm+eS6s$+p!^~TsPjmP?ze1-rpN0=za%9~^{S+Xh< z*g-S{A_V5za34L_#Wg{}BP7cEY)Lxm4_{hw@LA!V z)hH!^Esq}o7|k9r%2%1SL#A@Iwh$`bB~WYH2V5L27!LI)qpZ%*FBZUMrU2E+~ ze70MQEEcNckS;ryf0S@;zc60%F4>G$|}g(zo7RI4t9=~UGow$s=8vJqtkM8 z*Zdnenw(TYLb`teM`dNK2n6CC9bIB>{bQXdVPRoN__ndX@39w&!+7Z*R9?=e!;r!M`XcuhlZCJ7>tSyN?z<qYYDYHP@AnoFew-LdM9J zeEoW$Vg^AL-L-pz?SjX@6|Da9%Lmbs2&z>slSo$yWallrOfz`HA<}t8X}Ct4NbG1a z&cH(}Fl^64`sf=eS>(l3SaG?odJw{^d~bXZz1@mp=6y_=O^txjVJU&W7L6BvQ>0_R z_&TucSkk)F9<)UZ(b~a-_RRkv@@~d-HMA7w@~3MF42lI7SVr0~!<~Fv5q92#OD36fG&Ap#li9bWjXbl!z*oaJDAnEBEP-deEArLyI@YNFY6j; z*{CbgL|C%xz4A=WB)J|cB9kZBMq{}3!JTWb4{e`^5@5B*-``mQo2Zi`zufB`zo<+j zW_QEqN4Z%bSzt|E=#w+Ph<irzJyj#)ozGn;D3A^AX%WZjtq#5}E-fd@@FoC+D4}!=y_=-LR0e^sMD^C4L@FyflXxa?Kehgr-bf-Bjz zy!)Bc0o$q+-fiR3*-JvOXFXodJ%;C z!)!V@F?H=S`%R+E&d;#N3{+t^&h#r-ZoMuAlyd9JSEYyWZF3meNqK{Gbv}-hWTl$L z7fF}=wYoZz?`ZJkh{G3fcrextLoYYN__5aLaff@o{RWzwmP}%9qUHIKjNBKpE*6yQ zaz5t|aBa@Uf}K&_*t^KsP|f=l`kB78wZ|!Ie&AXvaz{ZGa{B@!lMTsth`+2N9E*N| zSzp3WLrDEoMv%6$Bu?0l6Q(H5d7OPqmsTv2|K;tMq56WpVO5Z`(sH)b2Pn z2wk@{2dFA&knNd6_XT=jwEcwH45mV{Pnu=Kes75au6$J%^|_MWGi9L!5QNI5awQGMK$fneX~zlwgMAzGj_ za2yC>ds?K#y(#6B9`yGtJ6ZZp>g>}cKEdG1(8$v3G0xZ=`@c$={V#p+Hejp27RMT+ ze(QCu1$5bMjIiD`;ygNzu>u=wkSZ5uOJ&i}5=wv5GQanpukp8ArN}%KwP!X{VY&7< zi9RWuAf763_e6wo+eVNrU>&^+1t*;ikA#Q3N6*+`{nWL6wSH|1kKab8cCrS!1cs(3 zatc~mP-Y>hoFlQRP)hQK2rvTjELjVpk#B}$y8*5WkWLXL0^*61~NwkuRisTl4mOo87ZQ;CT{#T82*qG<>RNXGZXfDT4ixM~ELb z-^ArDlF*k+8#5s0@$_d1^`y) zN>i_w1%D0=dmN?WE~MdelR$vC_&t0q@_F7>^{{0bC2NY=-1@fXvll_ zuqYRvBxYvH+u7Nf|9C_MQcpTMI*?5w^0-5SnY@VtejDT>O$+qR%A%6btD0R%VrVv3 zEkKs+Pf?Dw`g8X$tnGj9D$Cf3S#0f=KD1P>Q+fT@+<}fj5`o9r-(di<>HIVDcl#AT zdWKEzV*eZr3y`~DV(^4e`E_i)0*C%IA|dPLH}A)N@kG_=`|2sm(a_m zH8G8h7xAFkQTw!i9voFy)OKpNLfnc3FZN+ioraWm^an=gXZOD)cz?;e_3>kg@w8Pw zBLHbDxIbNh-Qp1w1}>GalW8o*{?n;xeoR|~S@P@aYXcOKRz&fTwYJEb`Pc@x7_X(`gI2utCj%y=Kzt$dZ?LGweQTk)*0; z((_55GdZcW?VdU9sr-@cH6tvMr=H=`5O_d|iQMb#HT@k}_tTJo3Ufl*0!~X4XN;-TYHFUv?dw$&;^?F=zpC&q z(`tXJ4&BC&&W!}??h@0~d{@a~nr^q?vR(+x_w|1-cZSDX>0Fmqo=jM6xspi74VPM~ zPR!V)Y;R|iRl>(!*RG<0XjT6o?ZSBn;teMK7ZO!%c+PxzZYh=B6y&Hd_0mL zWsqP6yLV1T7T-ExCqjY~VkzY>MeoUHi^VV9p5y6p6E{847<#S7?%TZirc$Aip<|$< zTgA$7Dqm^Zucv%fo8`x#U&FylmYd_}fE=Co47XY+sFHJ)*X|De_T~)%$Y&yck)MKx zIo90&74eIk27W$WG)+O?-D~TJPY&m@OrKQ?C>w(z;SZ*^-(J#W6THM>l=P)oY-p&r z^1jvrzo}fMA`J%klCd3JUu*E=c0t3B$EpksTxQub-1y}9)`C8z} zW)V!-L`CC|u1+yMz005aka(82824P}6MvPP7w2Ufk+Et2<}X6!qpJx(1Lb zJkZe7NR_tB!WVD8J`C~)cm}h2w?@Y$fUKx;!_(6}J>l>X*Z{%f%KDcdLX^88 zt*1xn=GI{H=MXq^OHeFRsr1MB-Dg6!r(G{$qtxEow3;_`xq{){5Jrtk7C)Vdcfe6n z!jjoegcUCx-BG$DM+Tv%=b^@++V5Ysq)jdd^b1>byxf`0Y0hy+AD4wUF)O7a6zFD9 zGn)erlgXKC@R|BgwT%y*L*zEW@1y%?zYA&CRylOgEzX|c6v#TZ z3|^DIP$`wkSd_EUN8s!)(y2+J&?NjSQ0>4D&akL(>R+jjG=11Knjiee!V#>sWgP-- z&iK(nZiqnBsQWIg$%5g%W$}%b<29;XV4$MSjM+4kY_`Sy?Q{85KLrg6qYUlYMl+fP z9Xg&igU%e|&=1Bx%M2a?94CYg5<7Y{Uu}->v^U95cF4~vgf?AeP?a#7=p@fJBmcO; z9ek9oRZ@A}P5wO^x7obHuqkTrkxH&fdBqZ0=*woa^6390aQC0Lfd8$i-~TTzcF3$> zvXPx&QY85&EEIE2{q}9ZADN_BWZ>kN!@q<(w>4~RSRUZ#IF#Tph#8}z=D!pL^>fL2 z-yOPmdU}4JuYTp+)beGE;2*3p5`&BhdBacX;rI8U;o%!#0D@XrxaEsed~w#}B!2n& z9(kAN7>I10hHvZZ>l*cTG~5=G)W|T*m&^R^$Jpv7(+~t=+Tri>5)<-)#!0cOiF(XTJ-et%2&?g0U)m5%B6B413JDz7)0xi0y6O> zUkR3x$bp}q|IbH;$8~w&gHWIKNIJic^PFk?N9AB-!LfRKW6UmOkmpU4d>7Mh#+q+@ z{2OFEsX(PbNjm%yW9Z;T4cnhzUh)g{F(c!{cHjkCQm*fxKKTL&!-F_X2VdjiDHXfR zOq~ncr$67%~G zzu!z?{S(A?INA~Uzl~=9f6uOO>(}G|O$+cp3dsLk7aY{O(%kIuc>5eTGA@s*8q&hk zrO&ch$I9G<_nf>F-iHy5@K0SLS4n;t<^&f6FUg_ISY=csVtMBUCKC_UQ%04ywW{hb zmfh&Ri~_L#h`QjBB_8I;-t!gSk^vhJiJmF!SvBak3T`4l6*dGMta$o+h z8JA8}gCeOGLM)bfz=7G|N(8wp^XP;5_K&{4lO-C1O`07%6Na1R00Zc*Ld(6+TAp8=v@6Qn#f&()}#mEJR`mVCC_7r~^s zWZpoVyRq;7fEZyCDoz}2ZIiH9uOsZ9E){Q)pdh)v;I--(QAtW^NDNcJ`K_GeJ{e}& z$KBUbn_8AdF4fmNHW_In8)8Yh@t~G#?C#8F^NCBnxh2sU)x~DIc`9M2sIO`Qbwqc< z^}cku+7?enw#)g#$K>gd5{?s(08m79gJtx*2aR`}p;pk=8^Om+w%nyK@Lb?6r}s93 zj@i}ISuH!qrPAo&+%sb$Z7got_fnQ#J+tWwT6)i<`I=y5+)HfLzi;EkY%x149 zYar&eNo<`AGXu-6KD=;8pBrg^n5!?koG@Rshd z-Pnu`mO#@?v651k$7;G;_BKqLJC^2{iDpz5c!l9|ew&!se73JNU6MLN#$0y>T}`^- z6pUqT?0XHcB-F-nd9k)U9FgJ?x<3{f3ZL?{8^wp~UOs6}ZTg|=#qzw_ejs*-pfsYftZt{E@!@@}B$PGwQ&7LHeZ_}xUEEI*F~+GWYsiSA z33AV5TDu_Sk5WgQ>iiOA&o{S{Q%unuR3dwCoEE{N) zn;Yy8(ZCub=H9K!xe}9+yRgWNUbefLJaZeF4OkN8`gyU4=NU|khbe6smB8#|GW1yL zCqtd5NTTK7^D&oU^u)FatgMaFV~*vPZF52u5C3DL5>vbR> zlw%n7YED)j`}U~4qYU|w<;nr|!xt8_5qGRcutLy`-J>u2WptaV0OvZ{c#*cL&=5T*fA(7#U^*cm zg|YT0acNT>m7kBip-tnf7tv&&VNH$7O`>d1o7v*I zjVa!CGO#~g_dsZsUlUTVju>c8uU&KpVhr9qdp&h&e`IdU+A+m(`pjiBxM&}rkkq&9 z!%#vQ#Au|?Wt@7_kNKy7um^Wvr8&idd`$TwV=kMiEyj`SHF{xEg&{`N2DGVZL-0Wz znoAkhiPf+Nm3T(#tB)yXMBaTIy1gm+NYy^ixS7f|6S<*8?wfrI(>OcBt0BK`U@=kE z`yfGu)_ZQBcDiSwQFFs zjC>0f+-$SDVcfUjoa2Y&i(=LV|DCYRR}-%P`*jF~bdleW-^Ztqk(Z~pVoiqfXe@v} zkE*>JMOHpO0K3KApNL6ycm8cK7AG)nb$K9SSMdAzxkD%Oi(o^8S5)2I5W(4soOa_O zm7^jMU&ZXPb~tXeRb>K{bZ)~OX2|i z`g_R--81QjTyuZJk1CLlEautx=6naUV3p{mXx<~Ys2*;rwSE3@&05D-7|TbWQ;c}? zce20~sf0X8;FDg;Nwz%v20?lEzK9X_OFvGu%Oy9idef0<%Xr80@V=SP27LofGv&68 z-$W&XcDy3QS}`3Q%agX0YbVWIP({6XW~ zuqICiiD2s3{@D-WPoGS<8GhTCog<9zCCr@hJmPfeMP69GX}RS!Ij{cc32M=XUnyP1MIjkNlW=cXNMQ>PW_)#bilPDBeE8ya0wAo{<}Dms&B1i+_v# zJhDL&|4|(~3)Mr4PoRYj9BElcjulB*M9`HIaxRsBMozuE8rBCm$5$&q`(*9DG8^{N z2V>Q2p%Kf3D32L8Ch)hFsrFvQv+byr5lYKviU~hBQ5BHL^v6bZXPTff;0_N@N-7k1 zKE6V7cXGMN+mofO*O){n5>%^uCxR-t(zyVF#U8TT)7hHzSaQNk2ObHmCz>8%4`rcGj@-VJ`h>s`x*3g>2AVvgKs%g zH%za^BGW)VEwa8TYXj=wb9!+ZK*H@tS;S*&*!8Qx)Aiv}=t`ZyroNW{iG*9@g{35P z)cD%S37tiONAQ6dLR4fqR_0vaJfQ;VI~5`cL^svM>aNzY+? z?uWGkN{IOwh*)M%q$>nm-QM@|_e2%dxk==6q2(HyOVUe&?!DmVlL7o{5ssN5r69(} z8MDLnL$2ewl)rBU>t?oF!7G1&P`*-ITsheyGLp3EG?$ge%tCs8ha4UC+d5vm>0PXXF)jLQ+@1wI3oH;=uCyoxn()w);@#e48#{8i39x9O&jdw6HDS|f3c$7{ADm z{x>h@f2QQ@cBW$(fI!|eGe@SUlOrSTDF0ETDU9C~gE5>Gz8dFNeotM0h9eBD6TNh> za}gw-T!+^xu;{su;>O6NAY;vL0tColer!gDoShvKFg!tW|0w@4aIc`DF-u}zu0uQz z_Lq~UU&)mhPrmDKFbC@hTx7nbO=?8?9!o7AyeLye>h$m4ZSJIfAisS>j)y0Dxi(Uv)J4BC8ZTe9+(m!YN+yQi|MKoQ^&2eT>Mu{E z+}7-a`drZXOBA&8S{}X-Mp{V@|1H>DH#;5k$s!{<@Q4cS=U3S~NlZ=-U9w@F&RakV z6%L#Qk@|(6qQ#adMfK@lS~V8pLKdK@`V4v;^=XC+pDmDtj^@!MT5`<~sy1!WADW#G zN%>bf)cj4?3-mxL24x`@D3;~tNG)AgbH(h8dgzJkgnV=L1yf)SM!Lt6g6EYw8w}0` z#<>eIN}st`r@n>aT*I?@_~dmjA_R(#`9Fq(cS?!Jkvw1K`I!_Qg(#p87;@IfWx- zv-T#7C?d)ZbtLOL@9JJ_l;AS&1!hj*@NGkd(U?rV=1xy#;I>Idsi8XcUoR19NL2Qit3%q*AKiPCf;srlg=a2d;d&|2(s4@ z0^2g?ldxq-YF#6q*QKkoyQ~r+c%#Gn@ViDGVa>+;_(MflG354`e;`F5F!_qLa`U@* zgDeH9SW`?m&Xjd-CH%av{^~4|D>Uqn=8U%67C0k440lEWN^lRi^fKJX2%#*yWY(N} zsQgws^X&yx)?ib8f!`3Y@CwYCMgr#gv(zshG6b!<&uN`31PSYtH`t9d2u&1ncWo2N zhS;dJdXtRJc=u*LO=5L9;;go?BZ`ZYJA>W#2)e+2s@~5}7A_>q&j9*Z7sC0fkn)a3 zj$rLpX%T_$UA+hFB_kWN@eZ+3J{P%-%O>}UR%s@7tc9_Mm{&|P_s72Z`(?oIJ~MR( z3R#_Ak)#gWb$1$PO7yA19DWaEi+;1yT`9*;IYHd|0RHHx$eaSQrDW_Gq27lQ*}iAP zlgACrwE)|DwqTF4Bl$e2+>YJ8B-5NwhPv&sWF0o-`|_b->k49}N_P@Ll<__Pf(dby zQ3NBGx!fn7cVo=azX|HbcRKdKuCzP2zp%!aY-z0EYB`{jkX`j5*5Y;$V%#>=({e>iORt0s@5RN5J?esnoQkxHQt8 z)Mk{n@Xl>Mt-0-nBh#xAzvHbdDL{j8$6ECMRlycqO@oncN&an{R59^W@quW8O z;#35!J){2Sz$-EtK3U_+ha_PAU5-kr=bcMpiSA&$Va@e+7>#HB)kO$d9Tof-RCXD( zpe06GAf6VmeZos<|8dO8ox0@SOd;`Ft*o`A{Nm7EZD`lWyYI zno!roQlX2c?d8V!t&!YnbB*$lDG94O#)4)uLLWMcIUuD>9)u=Oj#U6H@BB^%=hRJ& zO610anJiHb>W9whYSHcTX0AKVz62WJk-2(Yh;uTvs9Jn`hyTJPY9n*4&7u77ik(NF zda|GmHNoK3S{DH$37}laxhY}T4Ny@rL2UML%LTFC9rj+c(OVghRrl5%Y*d|a%h^&} zCWmwQnE4yl7!9p-$GfcwaqhZ6hg}osELXNhLZD*@HB{FdqbshJdsfT0j%z;tYKnKP zjmO6aNE@nxktc7fxi*AVKAA;{X_sMm~JAJFQ z@d;iUf9Voq+fGQ^=uGe2$7FWQ_onZxP8KNPDdz4}6wx#0PoFokwI8C53$^SjQ$lmE zVePxRO#3+2pY7&LvT`rpla{%f7@R$esn6ij4A#~@Fxe~HmtPE5l`W->-Q<@00n<@5 z4L36~dcC--NNHR1L$OSOzTt#lc-Ou(PxgFza;`R>AcbzT#ANNZ5tIB|26y<(TJJlz z8@#JYf?XmgMa1ggzAZ;47{Rfc&I4- z@?4|hd5_&v%l%6@2pu#LeonO5jlUQ}R$8i{@f{`V;0Dgc3@#N5{##oKQ1f`Mh1D8> zox+*icw#ro#AjRtMf!S=smFF2^)E)lb%$9*#27VC3yw z6@jpglsnpF#Nc;Af}?vL80UW8&9h4%Uv1KazP>$~1QwWsa?TPQv5?x;XPzyZw7;V{ zg{|>nCI3|nf77g6JLtuuHIOupF3cI`EK_8ib<`w)AAZ|jE7P2uaJFKR2vo9qm3jCR z=q_*0*oa@CYjp0Ow^P7xS0U9IxH1da8THLD)e8&FpiT-C@rwAOPYpR_0w<^JC!0<7 zREv_ac^q;@lx|}`tz^A^6=VD;rGI1zn!9YPTtEttLaQw9INivy9LxIWm_*VK-h+%( z4;_0;=Jl8uH**E`3tJn0Mm6}p@TG&$QPaak#{3)S4kFB_aQu_MKkKwE>{{)7Yo<6SM@7F5EDLI+TAwaoT*%uwhk*mR=va`O8*Z4gML zR-;j$WMqaDL1(&(@>E?GC@(LVc|;uO7Y3SPTK2dHL9J#ba~Imz2j4A>wFNT9$#J+G zzWeCJ6JVV8X5PoknckgxXHtEUR4?dj2=9=>{39cxo8uW}y-N03CCDoASiX1SY56l{ zmd$=3w;*;%(9gfCT@U)02iRnrO)_nRw??;1U0*NLB<$|MGaP4e2wCqBd$Te(9&-(3 zwC)m=cRuDH6-)vFVW%E;irD3^wGgxEpZ--s1xt^yWTi!J2F{xkR?>i1N!7#I{I-hp zyObBv6+oO_Y$_cO=h#huwye$t;=pRR2wOyJj4BAoZh7U{yU3rYV4L3VqQDJsdMiIE z2u$@={GdTfulMSE!Pgl|_J~xTw7>-c4&z(wR2>)oH9|Yv>*PDi`waK8D4@i#)-$ao zCa+zN7l#5)=h!t{8pc>%@@Yid=TkpVI7PA-ZSHMx=3R!ck@{2b()`NzrjgB2YcP2G z6lAppkxu&KY z@rvQO!plk&t3h%h81PDx=Pve(!4EfmnhwrBu#qW`5ydOwBT{c>yoS32u=Da;KTt==6mV6=>$q zBMHcTGfv=~quIfCiLxO^BZJrqcm9Pzm$h5HnwQm2Gw+M_vs5-cdBdlEtPOX$ZL-#E zqe!FS*qWDoy_CXO;I;b5<}x5=A^KDkxZ3M9HFOVa$ny>!MfeRji-AWG{1-a*gPEz3 zjdhW$bmua9*H&cE7=Ux0kUSM>jZOdZA5hA@t&lxr-w@P2Rz2-bh(O>Bfzl>&^~SQa zt;Uw>f31s}L3I18H8mX%=At-0d3U0#QRd4XlL_Lk**;mc(ysHPJx(>!DA$rpcXN|O zRYmx{!=`L~@g!y{Ke{mmatGFZ!zmcnGVmvD-nN8TJkS_N+vSahXf*!l$o-Qq5}7%w z*}xl@W6$R8Yu>gj!zCC8bSq0rD7L3qHJ|W_V4W4<>R|%po6%HR@P?qH;A}Rf^@`TP zoMGzD{oA+K$UM0BEJ>-UU%!3(R#E4YduR+hT6%Ch;ExXKlQo+zVy!CQQWhF7cvUCt zf$FxAwN)h-WHz%o3^XZYB<8UBcy}%<)DTvz+mNw%!Vsv9c_88}gyvlF_I*GxULb#t zmGXS|@3X$`xVgkbI?IJJq!1n*nHhs7lThjyuGfWQdp~#uZP`&;M!|%4b$*CzPVLdI zWY3Uhy4~c?{S(~j_vMP{tKD>=qKdpWq|#cxUF@v0ETs~+{aFrqlKv9O4D0a;!vCF4 zrVt*HcmLl^RJZ$;3H=WaE=|9(eD;9(H;dVliXDe0rV0afxMCqN=xROtF#lOy6LFtr zt{d5&O>)0FlC5cup(5_!vLf&N#p-P$xxvGAtAKN8So^(EU(qip3v8z5Ea_e=%*8%} zM=l|5DKtcQ++zxT4e)q#Jj(lOSemxrqS^qWDzp4aEm`0`O0e||?C##Pb)n{X89Z8Hui!3z zV6b-KQ*jLBEuFa2xVgA@q-+dakyl}f_FdRXSsh9OZJ*wsdM;jNX*|q7oepx&0WSWr zsSZpy^C&CMByklqRS)lDfHxNLkVj1S|@jNl}nR4uyiVTvf_diZqg}+w! zP%|M|<}1>llr)~7U5Q2r@k;*Po_kd-#Cc?vvB5z`-jFYbsCFk4MTNhT^Ev1Sp9%09wZ+>b*0Otc@q)uROs)(p2zbI}& zK6Ao9>oO^?^gN`%amOQijcj{tV|=%Le=1GqM3+z$mD*fscUp1~?C>y*i<_P7k(TMf zmizm+LDd{fYOHPDLdBfb@87?591pm=;yQlxt$)Ai7j;e_?6XoA5&+}@<=TdGJyk`vXSYNtW~0h)HXayC@LD6OU*`d zEM46?F%xZO73ex9E#}e`l_maW@2NIFm3ie3oM08C|DAs8L|6#ENg8;Cx)DC1%HZIJ z%u{?z`}F<4=yHBreGe_JYOozZLBaX;_+9`c#sA;s;{F#LnN&sH)ABhhWQMSD-NKmr zV=$FA2}t}xttbJh(-0LEMfdSmbkVTmBPZu~cvdv^6-jj=vt7g+9@E048jxwlGG=CU z&P^GGszvJ?8_KJoaPnnrN+FHS?hlW3wOttmhlCJar_)BdRn1vxl&C#_%({*-dim;A zheeH1)f~E~d`zb{lEy)5SH<~Nk!;`>mdCVYXJ=`H^b|RY6$^*Y zMOvlSWPrCQNtDyevodm8vuAqap49NwhHhHgGAJ+)ExQ#GxrkG}q zvd}n)sbVc-%1op9dHZ@!z065vrn1ZqQAYca^ax&aA+m4r)ke19OCShwY`hE?NR|ZX zjrcZx=1YY@nLV(IeVlumBNPiLHkN<~W{8icjj0Y#GtZ&Ss9WA|IZy>ri(seT+i{*$1vIH4d6t3saWnpJNHiJuM zapZt7QPts|Tj)*LYpA_V&8Xum=q&z^I)@o4n#m#v*{UdkgTl@xI}UZNp-ctq?eatBGFDvST@#@^_t-Wv=SFb zU-PEzyPZgf(*T(t;g9b`&@n~onKn+NTs$VNY34^xR8hqe;-mjJ8b9pd?;!W zl1`9Sbp^nX;^Rr$l~HD25=pqka98Hf7c8@0>F8WGgx?xFQ~=lFgj%xca`5Xaxb113 zTGM-Cxrod_Piz*-SB>o}jc;C92*ddG6^?!NX>8^aZn3+9y#s5GJzM8n8xQ zOj3R$+^T%qBM|to*X1$VGV~-bBw|#5b;|27nJ*WsBgq4><xer0k##fj=B(!j5*ZoTF6aPLn7ft`2$+`d-8CN|?FoUswP?(Y6sr+(Tq zQ#l=cXVq#+O!XfhXugO8yxDzPckbek=d%Fx= zlk47tt`g=EUr-kH2js*%a>%IZ_AUz1C5Wro;s3t=$^8rG-s1(;V!(>fWT@Ufsqz>K zJg4FP{8x92+by(%Ch*x|qlv`@`6pQDC%7otgzaL>WF-;pXJSfOx#P7pGu2lmJCYse zRpF>%L$hNBD8?S*cqZ)l#A69GUKUltr!v@0XZ<72tN5>Fp9Am{f8aW1o_$P%UT!vu z207yMUS$r=WVN)nIQgHP7JO-;ba@?Dm$Vd;;mdl7Cb!m+wat0CwAFmpNw-cH@yB?F z0Np;5?k*zC_v`2D6v#6b%bp$6G|k(Y)Z`J*p}wr`3HuWRn<7%*xjc?K>^+73^r?3? zLNkE}GDCMq^y5@){@*=iC>G7p_zoC`|FPH%%r0s=y5Q<+F60XsMtlJD^dT1)jz&gC zUu)nTDdlzohtwoAxGGqDFbaT0O>4N8YrjrnFf*6`$xnUMpsn0g6(?Jo!@R18!~3$g z#p?|-SV&QzonVIR>bT#aS9(q#R$-9(#^P(wiAeS@At`)%tEp)PO7{}zfWQ@T_pbsC z`+Z|fMvFb_#naI_D%p4V_nO?rRCOg$YqitXJAX@vTzJaVb}Az{Jv906sXR}tIr>1t z$q}BF|5q#L9n{3ShH(Kwq{tvuX;KaWMd?JOsq}sjq(~=k>AAFZ1-3|WFRpTGI0=7;1BG(r$>qKW{4|h`^!@#_=gtk)x_1DarQ(T> z2yzJ&*A5)@SXf_Fgy(>JRnWY-fwX-xcVQT*p!+7jMflRzAKvYo!n|Blhk;&OTS(Yi zT$DYBD94ezCnWO0#Yz#_`tgqcq7sMn<)XW5?Jb=;#h_Os*v=y}P>c=1Y(&a^BlF&U z>P)F=^lvTNqX$34TA_12E0Y5L(PIv-;0it|`MJMSmo{&ti}*RaU*MH;>6;CgQfskQ z`i7&ZbXtlGz#dXzUZ{zn;{_71@lU-Q&b|bUdty+rjA@|xBP}hRkeHYqgS>YZe{q5n zG|^g=RULLrl~`v4N(R^HSfH&nIUC3%G`s9-!J4AD7Qu-AvfsJ22a#U?JD-8Xez=~- z?v`Apjc{zZ27D&Xdz1tl{m76Tv^}(YoSD3tA25&aPGS?m+^sLVHk_=>H!rU&8?TN6 zfBj(ZX=3Gx0V#t*LWfQTD$!BR^ViUPI-`HlH=gw2=2KagDL7*)I}0}K@33-fg7M%C zMP)itRX{}UVZV%SM59`xYNGM@4yxG7r9Lx}FV7qnce&zHQW$(&g<-GkjJ*ObhzGG& z+8H*%jEzcRmF^u{fF?RPpp8xdCA1n=f*-f!4B*y0g1CbZmh3at8?QF{>&HoTzc^lD zUlf>&mz-?fT&dO+zqZtqx5XBWIpm3C6gF2nq^*e2tegGpvX?UmK3F@Qk#%>aye)pk zZzZ1NZhCL4Ty$n4&}R@s({x+Mji-b*!@SCwGLeQP0%X8dr(PTP*4(p~cASeo#^PS8 z@4CtpW|BW~w}#3P;&mvZ9TBB}I+dNj%|#PaO|&a(Oj^deUR`?xhaa3C$K#W`ZG{JI zm*YY4n&X8K2(3bNY(-eIkbYs08$@040L?Ef?`!Mog=v~Tf!{Ca@F)y=RHodyE z<25?nyVK?3P!%J~^0aMC{ZpNh#CzCME%qQ6e03p0w|KZeIl1T#(s zRPjKLvPRte-42N^epgqcpHJa?UTg8b=nCUv=)sXO40Q~K>{(V zKH+k^;b;1ZAxHYCN!Sagm4n1TgyQ{#1dQR5#S^SU#fL`?GuEScQ$aA0150rQIUZB{3UFOM52AkjSdi< zp(?^WXV4Rl`fo?U};_v#}vB@T;mX|pW!p_+WR`E5Rj8bzsih3+x(>;;JD)mM; z`*Vr@ZA*=CQmj^*ugUJ_2?>ILY{zP`|Ci$5eiO@9Vozq-g7adaP_GqbMzu5TFb-#lDLjcpQgH2xpwAd0uV60ISMfTmyet+^o zILlxT0DK2ouSy(7M@4h(WOj7|clN?bac9SQhf>b3FPHZz^}d=`yNtex`E9b3B+0+E zOZC||%OCYr+E}wP9bW^yR5DXpK5d=ELS5sCGKEE1OAvxZG$%zP@J`0?3`o1YoVk3( z!IX1&T?9^2C0`}?R*24M;J`Dn$cEX|%;((c+ThA$?ww)vIYvQOdh6-)@!TDTYfP|z zxXvdLJh;XNLqsU+w>)^*ge8b&lXTN`xELUVj8N(utZ8@!s#c> zcVfP0znf?J6{1rENXa=ChF(MTEFx{59E5P`$;X*AiiNgz_a658(uPih3)3i~$8B3y zxXfPSn-h9Ar_NxUg_y}yrQ%J3lt*mvp%b}cQ}=W{CB$)U1=|J*4|~YPi?)yH5p#-f zia6ScD*0{XcRn3%x(@T1V;Evj@@aS@b9yBU($5n-FC}bJR*i!-?P}COE_y6JJ9;F( z%YTl9vWm+2nGojry;tPN1`;M+JHGzCJzwBPOXaQEfJ|o9-TG4qL9?Y`IAycm@45)9 zML_U)x%+B8W~j@M@|%Y%1h5Ha+K%g8*36oft_yY$nFTT$PXC`6`+uNM_Tt=#c0o7X Tj}n0BzZ2%hHb%8Ky`KFGjSr~g literal 0 HcmV?d00001 diff --git a/docs/public_discovery.md b/docs/public_discovery.md index 5665b937..90246066 100644 --- a/docs/public_discovery.md +++ b/docs/public_discovery.md @@ -2,37 +2,79 @@ New in Bento v17. -Previously, the public data configuration given to Katsu was applied on all the metadata contained in the service. -This configuration declares which fields can be queried publicly for discovery purposes, which charts to display -and which censorship rules to apply on the results. +Previously, the public data configuration given to Katsu (`lib/katsu/config.json`) was applied on all the metadata +contained in the service. This configuration declares which fields can be queried publicly for discovery purposes, +which charts to display and which censorship rules to apply on the results. Katsu can hold multiple projects/datasets that may use different fields, require specific charts or custom `extra_properties` schemas at the project level. -Therefore, there is a need to tailor the discovery configuration at different levels. +Therefore, there is a need to tailor the discovery configuration at different levels to properly represent the +particularities in the information of a project or dataset. Bento v17 gives the ability to specify a scoped Discovery configuration at the following levels: - Dataset - Optional at dataset creation - For scoped queries on public endpoints targeting a project and dataset: - - Katsu will use the dataset's discovery configuration + - Katsu will use the dataset's discovery configuration, if one exists. - If no configuration is found, fallsback on the parent project's discovery - Project - Optional at project creation - For scoped queries on public endpoints targeting a project only: - - Katsu will use the project's discovery configuration + - Katsu will use the project's discovery configuration, if one exists. - If no configuration is found, fallsback on the node's config - Node - Optional during Katsu deployment - Uses the legacy `lib/katsu/config.json` file mount - For non-scoped queries on public endpoints: - - Katsu will use the node's discovery, if there is one. + - Katsu will use the node's discovery, if one exists. - If no node configuration is found, Katsu will respond with a 404 status. - For scoped queries on public endpoints: - Katsu will fallback on the node's discovery if the project and/or dataset in the scope don't have one - If no node configuration is found, Katsu will respond with a 404 status. -In previous versions of Bento, the bento_public web application could only show the aggregated data of all projects and datasets. -Now, bento_public users can select a project/dataset scope in order to only retrieve the data contained in it. -Given that projects/datasets use different fields or may have custom extra properties, depending on the study, you can now -declare the fields and charts of interest at the relevant level. +## Creating a discovery configuration file. + +Follow these steps in order to add public discovery for a given scope. + +1. Decide at which level the discovery configuration will be applied. +2. Create a copy of `etc/katsu.config.example.json`, use it as a base template +3. Modify the `fields` section so that it includes the fields of interest. +4. Modify the `search` section, include the fields from the previous section to make them searchable +5. Modify the `overview` section in order to specify which fields should be rendered as charts and how. +6. Set the desired censorship rules in the `rules` section. + +The resulting JSON file needs to respect the JSON-schema defined in Katsu [here](https://github.com/bento-platform/katsu/blob/develop/chord_metadata_service/discovery/schemas.py). + +## Using a discovery configuration file. + +With the valid discovery configuration file in hand, you can now add it to Katsu. + +### Apply the discovery configuration at the node level: + +This operation must be done by a Bento node administrator. +```bash +# Move the file to Katsu's config volume +cp ./lib/katsu/config.json + +# restart Katsu to load the config. +./bentoctl.bash restart katsu +``` + +### Apply the discovery configuration at the project or dataset level + +These operations must be performed in `bento_web` by an authenticated user +with the `edit:dataset`, `create:dataset`, `edit:project`, `create:project` permissions. + +Before creating/editing a project/dataset to specify a discovery configuration, +make sure that the JSON file is in Bento's dropbox. + +At project/dataset creation, leaving the discovery field empty is permitted. +Specify a discovery config by selecting the desired file from the drop-down options. + +![Specifying a discovery at project creation.](./img/discovery_proj_creation.png) + +During project/dataset editing, three radio buttons are shown, allowing the user to pick the existing value, +a new value from file, or none. The 'none' option is equivalent to leaving the field empty at creation. + +![Modifying the discovery when editing a project](./img/discovery_proj_edit.png) From 0b957e2b6d800d4d1cdfac6308346e20e4e3e8dc Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 16 Sep 2024 13:48:07 -0400 Subject: [PATCH 70/75] set beacon to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index bd9dcb3d..a7bb490b 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -405,7 +405,7 @@ BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon -BENTO_BEACON_VERSION=pr-107 +BENTO_BEACON_VERSION=edge BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_BEACON_EXTERNAL_PORT=5000 From 5f7f51108ab897ffe62f7865bcd042e5955002fd Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 16 Sep 2024 13:57:49 -0400 Subject: [PATCH 71/75] chore: bento web and public on edge version --- etc/bento.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index d0c69824..112ddfb8 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=5.0.1 +BENTOV2_WEB_VERSION=edge BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=0.19.1 +BENTO_PUBLIC_VERSION=edge BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 04a08f7bfbc93132b6e5f2596bb237c0800275e3 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 16 Sep 2024 14:58:47 -0400 Subject: [PATCH 72/75] address comments --- docs/deployment.md | 2 +- docs/public_discovery.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 41e869f5..53f682f9 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -102,7 +102,7 @@ If all went well, the `old-bento.example.com` domain should be redirected to `be ## Discovery configuration -Bento can serve censored data publicly if configured to do so, this allows anonymous users to take a glimpse into the +Bento can serve censored data publicly if configured to do so. This allows anonymous users to take a glimpse into the data hosted by a Bento node. When deploying a Bento instance, make sure that the discovery settings are configured properly at the necessary levels. diff --git a/docs/public_discovery.md b/docs/public_discovery.md index 90246066..562261e1 100644 --- a/docs/public_discovery.md +++ b/docs/public_discovery.md @@ -16,12 +16,12 @@ Bento v17 gives the ability to specify a scoped Discovery configuration at the f - Optional at dataset creation - For scoped queries on public endpoints targeting a project and dataset: - Katsu will use the dataset's discovery configuration, if one exists. - - If no configuration is found, fallsback on the parent project's discovery + - If no configuration is found, falls back to the parent project's discovery. - Project - Optional at project creation - For scoped queries on public endpoints targeting a project only: - Katsu will use the project's discovery configuration, if one exists. - - If no configuration is found, fallsback on the node's config + - If no configuration is found, falls back to the node's config. - Node - Optional during Katsu deployment - Uses the legacy `lib/katsu/config.json` file mount From 9b964d0db458803df9f7031d943c89c6ddf0133a Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 16 Sep 2024 15:57:54 -0400 Subject: [PATCH 73/75] docs: fix issues with issuer template values --- docs/installation.md | 2 +- docs/migrating_to_17.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 07f0c8e8..81ffe181 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -344,7 +344,7 @@ Run the following commands to set up authorization for the aggregation/Beacon cl ```bash # In the future, view:private_portal will need to be removed from this grant. bento_authz create grant \ - '{"iss": "", "client": "aggregation"}' \ + '{"iss": "", "client": "aggregation"}' \ '{"everything": true}' \ 'query:data' 'view:private_portal' ``` diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index f8195192..2f4e64bf 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -60,12 +60,12 @@ data by default, even if a discovery configuration has been set up. For anonymou # Configure aggregation/Beacon permissions # ---------------------------------------- # This assumes the aggregation/Beacon client ID is "aggregation". -# MUST be replaced with your actual issuer value. +# MUST be replaced with your actual issuer value. # - The query:data permission gives access to Katsu endpoints which are properly authz-enabled. # - The view:private_portal permission gives access to Katsu and Gohan endpoints where the proxy still manages access. # This permission will be removed in an uncoming version. bento_authz create grant \ - '{"iss": "", "client": "aggregation"}' \ + '{"iss": "", "client": "aggregation"}' \ '{"everything": true}' \ 'query:data' 'view:private_portal' From 3a1f559e2c8039e037009645edf130c915842087 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 16 Sep 2024 15:59:43 -0400 Subject: [PATCH 74/75] docs: reminder to put client secret(s) in local.env --- docs/migrating_to_17.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index 2f4e64bf..e1025079 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -41,6 +41,8 @@ To create the client secrets for aggregation/Beacon and Grafana (if the latter i ./bentoctl.bash init-auth ``` +**Reminder:** Make sure to put the client secret(s) generated by `init-auth` into your `local.env` file! + Aggregation/Beacon data access authorization will not work until an authorization service grant is configured; see step 4 below. From b2bb0b4f3deeb6c7259cd52b7e5cfd876b4a11bd Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 16 Sep 2024 16:04:32 -0400 Subject: [PATCH 75/75] docs: note granular discovery + fix typo --- docs/migrating_to_17.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/migrating_to_17.md b/docs/migrating_to_17.md index e1025079..77774981 100644 --- a/docs/migrating_to_17.md +++ b/docs/migrating_to_17.md @@ -8,7 +8,9 @@ Key points: * Data that used to be completely public by default (i.e., censored counts) now requires a permission (`query:project_level_counts` and/or `query:dataset_level_counts`), and thus a grant in the authorization service. - * Beacon now requires a client ID/secret and an authorization service grant to access uncesored data. + * Beacon now requires a client ID/secret and an authorization service grant to access uncensored data. +* Katsu discovery is now more granular, and can be configured to the project or dataset level, in addition to the + instance level. See the [Public data discovery configuration](./public_discovery.md) document for more information. * ...