From 62fc1b00721b13169e8af377eaf65a689c767363 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 15 Oct 2020 15:22:05 +0200 Subject: [PATCH] Update to v8.0.0 List of changes impacting docker-elk: - [logstash]: The output to Elasticsearch is handled as a data stream. Starting with v8.0.0, the `elasticsearch` output for Logstash sends log data to a data stream instead of `logstash-*` indices by default. The name of the default data stream is `logs-generic-default`. docker-elk remains unopinionated and simply uses Elastic's defaults like it always has, so users who prefer to retain the old behaviour need to explicitly opt-out of data streams in their Logstash pipelines. Refs: - https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html - https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams - [logstash]: The (legacy) monitoring data collection is now disabled. This feature was deprecated since v7.9.0, and removed in v8.0.0. Ref: https://www.elastic.co/guide/en/logstash/current/monitoring-internal-collection-legacy.html - [kibana]: An index pattern for `logs-*` indices is automatically created. It used to be required to manually create an index pattern for indices managed by Logstash, even when using the default Logstash indices. This is no longer the case since the output data is now being handled as a data stream, and Kibana automatically creates index patterns for these. - [elasticsearch]: The command line tool `elasticsearch-setup-passwords` was deprecated in favour of a new `elasticsearch-reset-password` tool. Passwords for built-in users must now be generated one by one. Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-passwords.html - [enterprise-search]: Kibana is now the new management interface, and the only one available moving forward. The old standalone Enterprise Search interface was removed in v8.0.0. Ref: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html --- .env | 2 +- .../scripts/elasticsearch-setup-passwords.exp | 32 ++++++--- .github/workflows/scripts/run-tests-core.sh | 26 ++----- .../workflows/scripts/run-tests-logspout.sh | 2 +- .github/workflows/update.yml | 8 +-- README.md | 72 ++++++++----------- docker-stack.yml | 6 +- extensions/enterprise-search/README.md | 13 ++++ .../config/enterprise-search.yml | 3 +- logstash/config/logstash.yml | 7 -- logstash/pipeline/logstash.conf | 1 - 11 files changed, 79 insertions(+), 93 deletions(-) diff --git a/.env b/.env index e40c94c1c7..4279128d06 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.17.0 +ELK_VERSION=8.0.0 diff --git a/.github/workflows/scripts/elasticsearch-setup-passwords.exp b/.github/workflows/scripts/elasticsearch-setup-passwords.exp index 3061f92b36..d2f9a36020 100755 --- a/.github/workflows/scripts/elasticsearch-setup-passwords.exp +++ b/.github/workflows/scripts/elasticsearch-setup-passwords.exp @@ -1,7 +1,7 @@ #!/usr/bin/expect -f # List of expected users with dummy password -set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)" +set users {"elastic" "kibana_system" "logstash_system" "beats_system" "apm_system" "remote_monitoring_user"} set password "testpasswd" # Find elasticsearch container id @@ -12,17 +12,27 @@ if { [string match "swarm" $MODE] } { set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch] } -set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b -u http://localhost:9200" +foreach user $users { + set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i" -spawn {*}$cmd + spawn {*}$cmd -expect { - -re "(E|Ree)nter password for \\\[$user\\\]: " { - send "$password\r" - exp_continue + expect { + -re "(E|Re-e)nter password for \\\[$user\\\]: " { + send "$password\r" + exp_continue + } + timeout { + puts "\ntimed out waiting for input" + exit 4 + } + eof } - eof -} -lassign [wait] pid spawnid os_error_flag value -exit $value + lassign [wait] pid spawnid os_error_flag value + + if {$value != 0} { + if {$os_error_flag == 0} { puts "exit status: $value" } else { puts "errno: $value" } + exit $value + } +} diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index 45a25052a0..51f9664713 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -24,25 +24,6 @@ poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty" log 'Waiting for readiness of Kibana' poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" -u 'kibana_system:testpasswd' -log 'Creating Logstash index pattern in Kibana' -source .env -curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \ - -s -w '\n' \ - -H 'Content-Type: application/json' \ - -H "kbn-version: ${ELK_VERSION}" \ - -u elastic:testpasswd \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' - -log 'Searching index pattern via Kibana API' -response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" -echo "$response" -declare -i count -count="$(jq -rn --argjson data "${response}" '$data.total')" -if (( count != 1 )); then - echo "Expected 1 index pattern, got ${count}" - exit 1 -fi - log 'Sending message to Logstash TCP input' declare -i was_retried=0 @@ -62,13 +43,14 @@ if ((was_retried)); then echo >&2 fi -sleep 3 -curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \ +sleep 5 +curl -X POST "http://${ip_es}:9200/logs-generic-default/_refresh" -u elastic:testpasswd \ -s -w '\n' log 'Searching message in Elasticsearch' -response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" +response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" echo "$response" +declare -i count count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" if (( count != 1 )); then echo "Expected 1 document, got ${count}" diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh index caf62bcbe3..ea1748b110 100755 --- a/.github/workflows/scripts/run-tests-logspout.sh +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -39,7 +39,7 @@ declare -i was_retried=0 # retry for max 60s (30*2s) for _ in $(seq 1 30); do - response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" + response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" set +u # prevent "unbound variable" if assigned value is not an integer count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index a22cfc6e87..6c7ac2954f 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -12,13 +12,13 @@ jobs: strategy: matrix: release: + - 8.x - 7.x - - 6.x include: - - release: 7.x + - release: 8.x branch: main - - release: 6.x - branch: release-6.x + - release: 7.x + branch: release-7.x steps: - uses: actions/setup-node@v2 diff --git a/README.md b/README.md index d91ab8712a..1860a1175a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.17.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -24,7 +24,7 @@ Based on the official Docker images from Elastic: Other available stack variants: -* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch. +* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch * [`searchguard`](https://github.com/deviantony/docker-elk/tree/searchguard): Search Guard support --- @@ -54,7 +54,6 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [Initial setup](#initial-setup) * [Setting up user authentication](#setting-up-user-authentication) * [Injecting data](#injecting-data) - * [Default Kibana index pattern creation](#default-kibana-index-pattern-creation) 1. [Configuration](#configuration) * [How to configure Elasticsearch](#how-to-configure-elasticsearch) * [How to configure Kibana](#how-to-configure-kibana) @@ -114,7 +113,7 @@ instructions from the [documentation][mac-filesharing] to add more locations. ### Version selection This repository tries to stay aligned with the latest version of the Elastic stack. The `main` branch tracks the current -major version (7.x). +major version (8.x). To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If you are upgrading an existing stack, please carefully read the note in the next section. @@ -124,8 +123,9 @@ performing a stack upgrade.** Older major versions are also supported on separate branches: -* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series -* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-Of-Life) +* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series +* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life) +* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life) ### Bringing up the stack @@ -168,11 +168,31 @@ users][builtin-users] instead for increased security. 1. Initialize passwords for built-in users + The commands below generate random passwords for all 6 built-in users. Take note of them. + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user elastic + ``` + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system + ``` + ```console - $ docker-compose exec -T elasticsearch bin/elasticsearch-setup-passwords auto --batch + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user logstash_system ``` - Passwords for all 6 built-in users will be randomly generated. Take note of them. + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user beats_system + ``` + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user apm_system + ``` + + ```console + $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user remote_monitoring_user + ``` 1. Unset the bootstrap password (_optional_) @@ -181,9 +201,8 @@ users][builtin-users] instead for increased security. 1. Replace usernames and passwords in configuration files - Use the `kibana_system` user (`kibana` for releases <7.8.0) inside the Kibana configuration file - (`kibana/config/kibana.yml`) and the `logstash_system` user inside the Logstash configuration file - (`logstash/config/logstash.yml`) in place of the existing `elastic` user. + Use the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) in place of the + existing `elastic` user. Replace the password for the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`). @@ -225,37 +244,6 @@ $ cat /path/to/logfile.log | nc -c localhost 5000 You can also load the sample data provided by your Kibana installation. -### Default Kibana index pattern creation - -When Kibana launches for the first time, it is not configured with any index pattern. - -#### Via the Kibana web UI - -*:information_source: You need to inject data into Logstash before being able to configure a Logstash index pattern via -the Kibana web UI.* - -Navigate to the _Discover_ view of Kibana from the left sidebar. You will be prompted to create an index pattern. Enter -`logstash-*` to match Logstash indices then, on the next page, select `@timestamp` as the time filter field. Finally, -click _Create index pattern_ and return to the _Discover_ view to inspect your log entries. - -Refer to [Connect Kibana with Elasticsearch][connect-kibana] and [Creating an index pattern][index-pattern] for detailed -instructions about the index pattern configuration. - -#### On the command line - -Create an index pattern via the Kibana API: - -```console -$ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ - -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.17.0' \ - -u elastic: \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' -``` - -The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the -first time. - ## Configuration *:information_source: Configuration is not dynamically reloaded, you will need to restart individual components after diff --git a/docker-stack.yml b/docker-stack.yml index 611f14d95a..f7eaad8f27 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.17.0 + image: docker.elastic.co/logstash/logstash:8.0.0 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.17.0 + image: docker.elastic.co/kibana/kibana:8.0.0 ports: - "5601:5601" configs: diff --git a/extensions/enterprise-search/README.md b/extensions/enterprise-search/README.md index d055c8852f..71fd789dec 100644 --- a/extensions/enterprise-search/README.md +++ b/extensions/enterprise-search/README.md @@ -57,6 +57,17 @@ add the following setting: xpack.security.authc.api_key.enabled: true ``` +### Configure the Enterprise Search host in Kibana + +Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search. + +To enable the management experience for Enterprise Search, modify the Kibana configuration file in +[`kibana/config/kibana.yml`][config-kbn] and add the following setting: + +```yaml +enterpriseSearch.host: http://enterprise-search:3002 +``` + ### Start the server To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command @@ -129,6 +140,8 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock [enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html [enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html [enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html +[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html [es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings [config-es]: ../../elasticsearch/config/elasticsearch.yml +[config-kbn]: ../../kibana/config/kibana.yml diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index 891b510a53..eb94457a72 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -15,8 +15,9 @@ secret_management.encryption_keys: # IP address Enterprise Search listens on ent_search.listen_host: 0.0.0.0 -# URL at which users reach Enterprise Search +# URL at which users reach Enterprise Search / Kibana ent_search.external_url: http://localhost:3002 +kibana.host: http://localhost:5601 # Elasticsearch URL and credentials elasticsearch.host: http://elasticsearch:9200 diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index a48c35ff58..47722ea7f3 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -3,10 +3,3 @@ ## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml # http.host: "0.0.0.0" -xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] - -## X-Pack security credentials -# -xpack.monitoring.enabled: true -xpack.monitoring.elasticsearch.username: elastic -xpack.monitoring.elasticsearch.password: changeme diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf index 7d5918ba88..40ca75775c 100644 --- a/logstash/pipeline/logstash.conf +++ b/logstash/pipeline/logstash.conf @@ -15,6 +15,5 @@ output { hosts => "elasticsearch:9200" user => "elastic" password => "changeme" - ecs_compatibility => disabled } }