diff --git a/Makefile b/Makefile index 851cd26..d2e73dd 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,14 @@ .DEFAULT_GOAL:=help -COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.tools.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml +COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml COMPOSE_MONITORING := -f docker-compose.yml -f docker-compose.monitor.yml COMPOSE_LOGGING := -f docker-compose.yml -f docker-compose.logs.yml -COMPOSE_TOOLS := -f docker-compose.yml -f docker-compose.tools.yml COMPOSE_NODES := -f docker-compose.yml -f docker-compose.nodes.yml ELK_SERVICES := elasticsearch logstash kibana apm-server ELK_LOG_COLLECTION := filebeat ELK_MONITORING := elasticsearch-exporter logstash-exporter filebeat-cluster-logs -ELK_TOOLS := rubban ELK_NODES := elasticsearch-1 elasticsearch-2 -ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING} ${ELK_TOOLS} +ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING} ELK_ALL_SERVICES := ${ELK_MAIN_SERVICES} ${ELK_NODES} ${ELK_LOG_COLLECTION} compose_v2_not_supported = $(shell command docker compose 2> /dev/null) @@ -21,7 +19,7 @@ else endif # -------------------------- -.PHONY: setup keystore certs all elk monitoring tools build down stop restart rm logs +.PHONY: setup keystore certs all elk monitoring build down stop restart rm logs keystore: ## Setup Elasticsearch Keystore, by initializing passwords, and add credentials defined in `keystore.sh`. $(DOCKER_COMPOSE_COMMAND) -f docker-compose.setup.yml run --rm keystore @@ -46,12 +44,9 @@ up: monitoring: ## Start ELK Monitoring. $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_MONITORING} up -d --build ${ELK_MONITORING} -collect-docker-logs: ## Start Filebeat that collects all Host Docker Logs and ship it to ELK +collect-docker-logs: ## Start Filebeat that collects all Host Docker Logs and ship it to ELK $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_LOGGING} up -d --build ${ELK_LOG_COLLECTION} -tools: ## Start ELK Tools (ElastAlert, Curator). - $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_TOOLS} up -d --build ${ELK_TOOLS} - nodes: ## Start Two Extra Elasticsearch Nodes $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_NODES} up -d --build ${ELK_NODES} diff --git a/README.md b/README.md index 06b2b93..66da61d 100644 --- a/README.md +++ b/README.md @@ -45,18 +45,18 @@ Stack Version: [8.10.2](https://www.elastic.co/blog/whats-new-elastic-8-10-0) - Security Enabled By Default. - Configured to Enable: - Logging & Metrics Ingestion + - Option to collect logs of all Docker Containers running on the host. via `make collect-docker-logs`. - APM - Alerting - Machine Learning - - SIEM + - Anomaly Detection + - SIEM (Security information and event management). - Enabling Trial License - Use Docker-Compose and `.env` to configure your entire stack parameters. - Persist Elasticsearch's Keystore and SSL Certifications. - Self-Monitoring Metrics Enabled. - Prometheus Exporters for Stack Metrics. -- Collect Docker Host Logs to ELK via `make collect-docker-logs`. - Embedded Container Healthchecks for Stack Images. -- [Rubban](https://github.com/sherifabdlnaby/rubban) for Kibana curating tasks. #### More points And comparing Elastdocker and the popular [deviantony/docker-elk](https://github.com/deviantony/docker-elk) @@ -85,7 +85,7 @@ Elastdocker differs from `deviantony/docker-elk` in the following points. - Configuring the Self-Monitoring and the Filebeat agent that ship ELK logs to ELK itself. (as a step to shipping it to a monitoring cluster in the future). -- Configured tools and Prometheus Exporters. +- Configured Prometheus Exporters. - The Makefile that simplifies everything into some simple commands. @@ -135,10 +135,6 @@ Elastdocker differs from `deviantony/docker-elk` in the following points. ```shell $ make monitoring ``` -#### To Start Tools -```shell -$ make tools -``` #### To Ship Docker Container Logs to ELK ```shell $ make collect-docker-logs diff --git a/docker-compose.tools.yml b/docker-compose.tools.yml deleted file mode 100644 index ba842c8..0000000 --- a/docker-compose.tools.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3.5' - -services: - rubban: - image: sherifabdlnaby/rubban:latest - restart: unless-stopped - environment: - RUBBAN_KIBANA_HOST: "https://${KIBANA_HOST}:${KIBANA_PORT}" - RUBBAN_KIBANA_USER: ${ELASTIC_USERNAME} - RUBBAN_KIBANA_PASSWORD: ${ELASTIC_PASSWORD} - RUBBAN_REFRESHINDEXPATTERN_ENABLED: 'true' - RUBBAN_REFRESHINDEXPATTERN_SCHEDULE: '*/5 * * * *' - RUBBAN_REFRESHINDEXPATTERN_PATTERNS: '*' - RUBBAN_AUTOINDEXPATTERN_ENABLED: 'true' - RUBBAN_AUTOINDEXPATTERN_SCHEDULE: '*/5 * * * *' - RUBBAN_AUTOINDEXPATTERN_GENERALPATTERNS: '[{"pattern":"filebeat?","timeFieldName":"@timestamp"},{"pattern":"logstash?","timeFieldName":"@timestamp"}]' \ No newline at end of file