diff --git a/infrastructure/deployment/deploy.sh b/infrastructure/deployment/deploy.sh index 5d8725915..ba3eda77d 100755 --- a/infrastructure/deployment/deploy.sh +++ b/infrastructure/deployment/deploy.sh @@ -386,6 +386,15 @@ echo echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios." echo +echo 'Setting up elastalert indices' + +while true; do + if configured_ssh "/opt/opencrvs/infrastructure/elasticsearch/setup-elastalert-indices.sh"; then + break + fi + sleep 5 +done + echo "Setting up Kibana config & alerts" while true; do diff --git a/infrastructure/docker-compose.deploy.yml b/infrastructure/docker-compose.deploy.yml index 4d8a8473f..3af57d9e6 100644 --- a/infrastructure/docker-compose.deploy.yml +++ b/infrastructure/docker-compose.deploy.yml @@ -60,7 +60,7 @@ services: - overlay_net filebeat: - image: docker.elastic.co/beats/filebeat:7.17.0 + image: docker.elastic.co/beats/filebeat:8.14.3 user: root networks: - overlay_net @@ -85,7 +85,7 @@ services: - 'traefik.enable=false' metricbeat: - image: docker.elastic.co/beats/metricbeat:7.17.13 + image: docker.elastic.co/beats/metricbeat:8.14.3 user: root cap_add: - SYS_PTRACE @@ -128,7 +128,7 @@ services: [ 'curl', '-u', - 'elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD}', + 'kibana_system:${KIBANA_SYSTEM_PASSWORD}', '-X', 'POST', 'http://kibana:5601/api/saved_objects/_import?overwrite=true', @@ -156,7 +156,7 @@ services: gelf-address: 'udp://127.0.0.1:12201' tag: 'setup-kibana-config' kibana: - image: docker.elastic.co/kibana/kibana:7.17.0 + image: docker.elastic.co/kibana/kibana:8.14.3 restart: always deploy: labels: @@ -173,8 +173,8 @@ services: networks: - overlay_net environment: - - ELASTICSEARCH_USERNAME=elastic - - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_SUPERUSER_PASSWORD} + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_SYSTEM_PASSWORD} configs: - source: kibana.{{ts}} target: /usr/share/kibana/config/kibana.yml @@ -282,7 +282,6 @@ services: - path.repo=/data/backups/elasticsearch - cluster.name=docker-cluster - network.host=0.0.0.0 - - discovery.zen.minimum_master_nodes=1 - discovery.type=single-node - xpack.security.enabled=true - xpack.security.authc.api_key.enabled=true @@ -365,6 +364,7 @@ services: - APM_ELASTIC_PASSWORD=${ROTATING_APM_ELASTIC_PASSWORD} - SEARCH_ELASTIC_USERNAME=search-user - SEARCH_ELASTIC_PASSWORD=${ROTATING_SEARCH_ELASTIC_PASSWORD} + - KIBANA_SYSTEM_PASSWORD=${KIBANA_SYSTEM_PASSWORD} - KIBANA_USERNAME=${KIBANA_USERNAME} - KIBANA_PASSWORD=${KIBANA_PASSWORD} volumes: @@ -384,7 +384,7 @@ services: gelf-address: 'udp://127.0.0.1:12201' tag: 'setup-elasticsearch-users' elastalert: - image: jertel/elastalert2:2.3.0 + image: jertel/elastalert2:2.19.0 restart: unless-stopped environment: - ES_USERNAME=elastic @@ -408,7 +408,7 @@ services: tag: 'elastalert' logstash: - image: logstash:7.17.0 + image: logstash:8.14.3 command: logstash -f /etc/logstash/logstash.conf --verbose ports: - '12201:12201' @@ -431,7 +431,7 @@ services: - 'traefik.enable=false' replicas: 1 apm-server: - image: docker.elastic.co/apm/apm-server:7.15.2 + image: docker.elastic.co/apm/apm-server:7.17.22 cap_add: ['CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID'] cap_drop: ['ALL'] restart: always diff --git a/infrastructure/elasticsearch/roles/search_user.json b/infrastructure/elasticsearch/roles/search_user.json index b7be198b5..da60cdeb8 100644 --- a/infrastructure/elasticsearch/roles/search_user.json +++ b/infrastructure/elasticsearch/roles/search_user.json @@ -1,8 +1,17 @@ { + "cluster": ["manage"], "indices": [ { - "names": ["ocrvs"], - "privileges": ["write", "create", "create_index", "delete", "delete_index", "read"] + "names": ["ocrvs", "ocrvs-*"], + "privileges": [ + "write", + "create", + "create_index", + "delete", + "delete_index", + "read", + "manage" + ] } ] } diff --git a/infrastructure/elasticsearch/setup-elastalert-indices.sh b/infrastructure/elasticsearch/setup-elastalert-indices.sh new file mode 100755 index 000000000..699ec03de --- /dev/null +++ b/infrastructure/elasticsearch/setup-elastalert-indices.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. + +# Upgrading from 7 to 8 requires deleting elastalert indices. https://elastalert2.readthedocs.io/en/latest/recipes/faq.html#does-elastalert-2-support-elasticsearch-8 + +set -e + +docker_command="docker run --rm --network=opencrvs_overlay_net curlimages/curl" + +echo 'Waiting for availability of Elasticsearch' +ping_status_code=$($docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -o /dev/null -w '%{http_code}' "http://elasticsearch:9200") + +if [ "$ping_status_code" -ne 200 ]; then + echo "Elasticsearch is not ready. API returned status code: $ping_status_code" + exit 1 +fi + + + +echo 'Scaling down Elastalert' + +docker service scale opencrvs_elastalert=0 + +echo 'Deleting Elastalert indices' +indices='elastalert_status,elastalert_status_error,elastalert_status_past,elastalert_status_silence,elastalert_status_status' + +delete_status_code=$($docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -o /dev/null -w '%{http_code}' "http://elasticsearch:9200/${indices}" -X DELETE) + +if [ "$delete_status_code" -ne 200 ]; then + echo "Could not delete indices. API returned status code: $delete_status_code" + exit 1 +fi + +echo 'Scaling up Elastalert' +docker service scale opencrvs_elastalert=1 + diff --git a/infrastructure/elasticsearch/setup-helpers.sh b/infrastructure/elasticsearch/setup-helpers.sh index c6a25e874..1b3380f0f 100755 --- a/infrastructure/elasticsearch/setup-helpers.sh +++ b/infrastructure/elasticsearch/setup-helpers.sh @@ -230,36 +230,3 @@ function ensure_settings { return $result } - - -function create_elastic_index { - local index_name=$1 - local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' - "http://${elasticsearch_host}:9200/${index_name}" - '-X' 'PUT' - '-H' 'Content-Type: application/json' - ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local output - - output="$(curl "${args[@]}")" - - echo "${output}" - - if [[ "${output: -3}" -eq 200 || $output == *"resource_already_exists"* ]]; then - result=0 - fi - - if ((result)); then - echo -e "\n${output::-3}\n" - fi - - return $result -} diff --git a/infrastructure/elasticsearch/setup-settings.sh b/infrastructure/elasticsearch/setup-settings.sh index eea61f90a..260e13a31 100644 --- a/infrastructure/elasticsearch/setup-settings.sh +++ b/infrastructure/elasticsearch/setup-settings.sh @@ -19,8 +19,5 @@ echo "-------- $(date) --------" log 'Waiting for availability of Elasticsearch' wait_for_elasticsearch -log "Creating index for Elasticsearch. Index: ocrvs" -create_elastic_index "ocrvs" - log "Updating replicas for Elasticsearch" ensure_settings "{\"index\":{\"number_of_replicas\":0}}" diff --git a/infrastructure/elasticsearch/setup-users.sh b/infrastructure/elasticsearch/setup-users.sh index 366090da1..30d2f5923 100755 --- a/infrastructure/elasticsearch/setup-users.sh +++ b/infrastructure/elasticsearch/setup-users.sh @@ -24,6 +24,7 @@ users_passwords=( [$SEARCH_ELASTIC_USERNAME]="${SEARCH_ELASTIC_PASSWORD:-}" [beats_system]="${METRICBEAT_ELASTIC_PASSWORD:-}" [apm_system]="${APM_ELASTIC_PASSWORD:-}" + [kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}" [$KIBANA_USERNAME]="${KIBANA_PASSWORD:-}" ) diff --git a/infrastructure/environments/setup-environment.ts b/infrastructure/environments/setup-environment.ts index 83a0642bf..aa5d3dd41 100644 --- a/infrastructure/environments/setup-environment.ts +++ b/infrastructure/environments/setup-environment.ts @@ -653,6 +653,13 @@ const derivedVariables = [ type: 'disabled', scope: 'ENVIRONMENT' }, + { + name: 'KIBANA_SYSTEM_PASSWORD', + valueLabel: 'KIBANA_SYSTEM_PASSWORD', + valueType: 'SECRET', + type: 'disabled', + scope: 'ENVIRONMENT' + }, { name: 'MINIO_ROOT_USER', valueLabel: 'MINIO_ROOT_USER', @@ -1103,6 +1110,23 @@ const SPECIAL_NON_APPLICATION_ENVIRONMENTS = ['jump', 'backup'] ), scope: 'ENVIRONMENT' as const }, + { + name: 'KIBANA_SYSTEM_PASSWORD', + type: 'SECRET' as const, + didExist: findExistingValue( + 'KIBANA_SYSTEM_PASSWORD', + 'SECRET', + 'ENVIRONMENT', + existingValues + ), + value: findExistingOrDefine( + 'KIBANA_SYSTEM_PASSWORD', + 'SECRET', + 'ENVIRONMENT', + generateLongPassword() + ), + scope: 'ENVIRONMENT' as const + }, { name: 'MINIO_ROOT_USER', type: 'SECRET' as const, diff --git a/infrastructure/monitoring/beats/metricbeat.yml b/infrastructure/monitoring/beats/metricbeat.yml index cb4875457..78bd940a7 100644 --- a/infrastructure/monitoring/beats/metricbeat.yml +++ b/infrastructure/monitoring/beats/metricbeat.yml @@ -85,7 +85,7 @@ setup.kibana: password: ${KIBANA_PASSWORD} #============================== Xpack Monitoring =============================== -xpack.monitoring: +monitoring: enabled: true elasticsearch: username: ${BEATS_USERNAME} diff --git a/infrastructure/monitoring/filebeat/filebeat.yml b/infrastructure/monitoring/filebeat/filebeat.yml index d2ac41bcb..8f75ec4b9 100644 --- a/infrastructure/monitoring/filebeat/filebeat.yml +++ b/infrastructure/monitoring/filebeat/filebeat.yml @@ -60,7 +60,7 @@ setup.kibana: password: ${ELASTICSEARCH_PASSWORD} #============================== Xpack Monitoring =============================== -xpack.monitoring: +monitoring: enabled: true elasticsearch: diff --git a/infrastructure/monitoring/kibana/kibana.yml b/infrastructure/monitoring/kibana/kibana.yml index 90404ecab..a97e87780 100644 --- a/infrastructure/monitoring/kibana/kibana.yml +++ b/infrastructure/monitoring/kibana/kibana.yml @@ -53,7 +53,6 @@ monitoring.ui.container.elasticsearch.enabled: true xpack.encryptedSavedObjects.encryptionKey: '{{KIBANA_ENCRYPTION_KEY}}' xpack.reporting.encryptionKey: '{{KIBANA_ENCRYPTION_KEY}}' xpack.actions.preconfiguredAlertHistoryEsIndex: true -xpack.infra.sources.default.logAlias: 'logs-*,filebeat-*,kibana_sample_data_logs*,logstash*' # If your Elasticsearch is protected with basic authentication, these settings provide # the username and password that the Kibana server uses to perform maintenance on the Kibana # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which diff --git a/infrastructure/monitoring/kibana/setup-config.sh b/infrastructure/monitoring/kibana/setup-config.sh index a2f5cca78..b94cab21f 100755 --- a/infrastructure/monitoring/kibana/setup-config.sh +++ b/infrastructure/monitoring/kibana/setup-config.sh @@ -34,4 +34,4 @@ $docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWOR $docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "$kibana_alerting_api_url" | docker run --rm -i --network=opencrvs_overlay_net ghcr.io/jqlang/jq -r '.data[].id' | while read -r id; do $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_disable" $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_enable" -done +done \ No newline at end of file diff --git a/infrastructure/run-migrations.sh b/infrastructure/run-migrations.sh index d68ee546b..fbe00d8b0 100755 --- a/infrastructure/run-migrations.sh +++ b/infrastructure/run-migrations.sh @@ -25,13 +25,5 @@ elasticsearch_host() { fi } -create_elastic_index () { - local index_name=$1 - echo "Creating ElasticSearch Index: ${index_name}" - docker run --rm --network=opencrvs_overlay_net appropriate/curl curl -XPUT "http://$(elasticsearch_host)/$index_name" -v -} - -create_elastic_index "ocrvs" - # run migration by restarting migration service docker service update --force --update-parallelism 1 --update-delay 30s opencrvs_migration