Remove Secrets and use Docker Volumes

.env

@@ -1,2 +1,2 @@
-TAG=6.5.2
-ELASTIC_VERSION=6.5.2
+TAG=6.5.4
+ELASTIC_VERSION=6.5.4

.gitattributes

@@ -0,0 +1,2 @@
+* text=auto
+*.sh text eol=lf

Makefile

@@ -27,8 +27,5 @@ $(TARGETS:%=%-checkout):
 	(cd stack/$(@:%-checkout=%) && git fetch && git reset --hard && git checkout origin/$(GIT_BRANCH))
 
 $(TARGETS:%=%-clean):
-	rm -rf stack/$(@:%-clean=%) && find . -name "*.keystore" -exec rm -f {} \; && \
-		docker-compose -f docker-compose.setup.yml -f docker-compose.yml down --remove-orphans && \
-		docker-compose -f setup.yml down --remove-orphans && \
-		docker volume rm stack-docker_es_data
+	docker-compose down --remove-orphans -v
 

README.md

@@ -7,11 +7,20 @@ Elastic Stack, all running on a single machine under Docker.
   * Windows and Mac users get Compose installed automatically
 with Docker for Windows/Mac.
 
+  * Ensure that docker-compose version >= 1.21.0,
+    * Compose introduced a bug wrt project names stripping out hyphens and underscores that was fixed in 1.21.0
+
   * Linux users can read the [install instructions](https://docs.docker.com/compose/install/#install-compose) or can install via pip:
 ```
 pip install docker-compose
 ```
 
+### Windows
+Please note that this repo has scripting that's written in `bash`. What this means is the containers that are linux based
+are expecting linux style line endings. If you clone this repo and you have windows style line endings, then please
+read [The case of Windows line-ending in bash-script](https://techblog.dorogin.com/case-of-windows-line-ending-in-bash-script-7236f056abe). We've included the `.gitattributes` file in this project to help alliviate any issues. But you might need to make
+git global config settings.
+
 * Windows Users must set the following 2 ENV vars:
   * `COMPOSE_CONVERT_WINDOWS_PATHS=1`
   * `PWD=/path/to/checkout/for/stack-docker`
@@ -43,23 +52,38 @@ By default, the amount of Virtual Memory [is not enough](https://www.elastic.co/
 First we need to:
 
 1. set default password
-2. create keystores to store passwords
-3. install dashboards, index patterns, etc.. for beats and apm
+2. Create SSL certs
+3. create keystores to store passwords
+4. install dashboards, index patterns, etc.. for beats and apm
 
 This is accomplished using the setup.yml file:
 ```
-docker-compose -f setup.yml up
+docker-compose -f setup/setup.yml run --rm setup
 ```
 
 Please take note after the setup completes it will output the password
 that is used for the `elastic` login.
 
+*Please note* that this repository makes the assumption that you will be cloneing the repository into a directory
+named `stack-docker`. If you choose to name the directory something else upon cloneing or after, you will have
+to specify the `-p stack-docker` flag when running `docker-compose` commands (aside from the setup command).
+
+From the docker-compose help:
+```
+-p, --project-name NAME     Specify an alternate project name
+                              (default: directory name)
+```
+
 Now we can launch the stack with `docker-compose up -d` to create a demonstration Elastic Stack with
 Elasticsearch, Kibana, Logstash, Auditbeat, Metricbeat, Filebeat, Packetbeat,
 and Heartbeat.
 
 Point a browser at [`http://localhost:5601`](http://localhost:5601) to see the results.
 > *NOTE*: Elasticsearch is now setup with self-signed certs.
+> This means anytime you want to interact with elasticsearch by using other tools/clients you must use
+> https, and if you want to get the `ca.crt` you can get it by running
+> `docker exec -it elasticsearch cat /usr/share/elasticsearch/config/certs/ssl/ca/ca.crt`
+
 
 Log in with `elastic` and what ever your auto generated elastic password is from the
 setup.

config/apm-server/apm-server.yml

@@ -1,3 +1,5 @@
+keystore.path: "/usr/share/apm-server/config/apm-server.keystore"
+
 apm-server.frontend.enabled: true
 apm-server.host: "0.0.0.0:8200"
 
@@ -7,12 +9,12 @@ output.elasticsearch:
   username: elastic
   # Read PW from apm-server.keystore
   password: "${ELASTIC_PASSWORD}"
-  ssl.certificate_authorities: ["/usr/share/apm-server/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 setup.kibana:
   host: "http://kibana:5601"
   username: elastic
   password: "${ELASTIC_PASSWORD}"
   protocol: "http"
   ssl.enabled: false
-  ssl.certificate_authorities: ["/usr/share/apm-server/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]

config/auditbeat/auditbeat.yml

@@ -1,3 +1,5 @@
+keystore.path: "/usr/share/auditbeat/config/auditbeat.keystore"
+
 auditbeat.modules:
 
 - module: auditd
@@ -19,14 +21,14 @@ output.elasticsearch:
   username: elastic
   # Read PW from auditbeat.keystore
   password: "${ELASTIC_PASSWORD}"
-  ssl.certificate_authorities: ["/usr/share/auditbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 setup.kibana:
   host: "http://kibana:5601"
   username: elastic
   password: "${ELASTIC_PASSWORD}"
   protocol: "http"
   ssl.enabled: false
-  ssl.certificate_authorities: ["/usr/share/auditbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 xpack.monitoring.enabled: true

config/elasticsearch/elasticsearch.yml

@@ -10,11 +10,11 @@ xpack.license.self_generated.type: trial
 xpack.security.enabled: true
 xpack.security.http.ssl.enabled: true
 xpack.security.http.ssl.verification_mode: certificate
-xpack.security.http.ssl.key:  certs/elasticsearch/elasticsearch.key
-xpack.security.http.ssl.certificate: certs/elasticsearch/elasticsearch.crt
-xpack.security.http.ssl.certificate_authorities: [ "certs/ca/ca.crt" ]
+xpack.security.http.ssl.key:  /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/certs/ssl/ca/ca.crt" ]
 
 xpack.security.transport.ssl.enabled: true
-xpack.security.transport.ssl.key:  certs/elasticsearch/elasticsearch.key
-xpack.security.transport.ssl.certificate: certs/elasticsearch/elasticsearch.crt
-xpack.security.transport.ssl.certificate_authorities: [ "certs/ca/ca.crt" ]
+xpack.security.transport.ssl.key:  /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/certs/ssl/ca/ca.crt" ]

config/filebeat/filebeat.yml

@@ -1,3 +1,5 @@
+keystore.path: "/usr/share/filebeat/config/filebeat.keystore"
+
 filebeat.config:
   prospectors:
     path: ${path.config}/prospectors.d/*.yml
@@ -15,14 +17,14 @@ output.elasticsearch:
   username: elastic
   # Read PW from filebeat.keystore
   password: "${ELASTIC_PASSWORD}"
-  ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 setup.kibana:
   host: "http://kibana:5601"
   username: elastic
   password: "${ELASTIC_PASSWORD}"
   protocol: "http"
   ssl.enabled: false
-  ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 xpack.monitoring.enabled: true

config/heartbeat/heartbeat.yml

@@ -1,11 +1,15 @@
+keystore.path: "/usr/share/heartbeat/config/heartbeat.keystore"
+
 heartbeat.monitors:
 - type: http
   schedule: '@every 5s'
   urls:
     - https://elasticsearch:9200
     - http://kibana:5601
   ssl:
-    certificate_authorities: ["/usr/share/heartbeat/certs/ca/ca.crt"]
+    certificate_authorities: ["/certs/ssl/ca/ca.crt"]
+  username: elastic
+  password: "${ELASTIC_PASSWORD}"
 
 - type: icmp
   schedule: '@every 5s'
@@ -22,14 +26,14 @@ output.elasticsearch:
   username: elastic
   # Read PW from heartbeat.keystore
   password: "${ELASTIC_PASSWORD}"
-  ssl.certificate_authorities: ["/usr/share/heartbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 setup.kibana:
   host: "http://kibana:5601"
   username: elastic
   password: "${ELASTIC_PASSWORD}"
   protocol: "http"
   ssl.enabled: false
-  ssl.certificate_authorities: ["/usr/share/heartbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 xpack.monitoring.enabled: true

config/kibana/kibana.yml

@@ -5,8 +5,8 @@ server.host: "0"
 elasticsearch.url: https://elasticsearch:9200
 # elasticsearch.password is stored in `kibana.keystore`
 elasticsearch.username: kibana
-elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca/ca.crt"]
+elasticsearch.ssl.certificateAuthorities: ["/certs/ssl/ca/ca.crt"]
 server.ssl.enabled: false
-#server.ssl.certificate: /usr/share/kibana/config/certs/kibana/kibana.crt
-#server.ssl.key: /usr/share/kibana/config/certs/kibana/kibana.key
+#server.ssl.certificate: /certs/ssl/docker-cluster/kibana/kibana.crt
+#server.ssl.key: /certs/ssl/docker-cluster/kibana/kibana.key
 xpack.monitoring.ui.container.elasticsearch.enabled: true

config/logstash/logstash.yml

@@ -3,5 +3,4 @@ http.host: 0.0.0.0
 xpack.monitoring.elasticsearch.password: ${ELASTIC_PASSWORD}
 xpack.monitoring.elasticsearch.url: https://elasticsearch:9200
 xpack.monitoring.elasticsearch.username: logstash_system
-xpack.monitoring.elasticsearch.ssl.ca: /usr/share/logstash/config/certs/ca/ca.crt
-
+xpack.monitoring.elasticsearch.ssl.ca: /certs/ssl/ca/ca.crt

config/logstash/pipeline/logstash.conf

@@ -11,6 +11,6 @@ output {
     user     => 'elastic'
     password => "${ELASTIC_PASSWORD}"  # read password from logstash.keystore
     ssl => true
-    cacert => '/usr/share/logstash/config/certs/ca/ca.crt'
+    cacert => '/certs/ssl/ca/ca.crt'
   }
 }

config/logstash/pipelines.yml

@@ -0,0 +1,2 @@
+- pipeline.id: pipeline_1
+  path.config: '/usr/share/logstash/pipeline/logstash.conf'

config/metricbeat/metricbeat.yml

@@ -1,3 +1,5 @@
+keystore.path: "/usr/share/metricbeat/config/metricbeat.keystore"
+
 metricbeat.config.modules:
   path: ${path.config}/modules.d/*.yml
   reload.enabled: false
@@ -11,14 +13,14 @@ output.elasticsearch:
   username: elastic
   # Read PW from metricbeat.keystore
   password: "${ELASTIC_PASSWORD}"
-  ssl.certificate_authorities: ["/usr/share/metricbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 setup.kibana:
   host: "http://kibana:5601"
   username: elastic
   password: "${ELASTIC_PASSWORD}"
   protocol: "http"
   ssl.enabled: false
-  ssl.certificate_authorities: ["/usr/share/metricbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 xpack.monitoring.enabled: true

config/packetbeat/packetbeat.yml

@@ -1,3 +1,5 @@
+keystore.path: "/usr/share/packetbeat/config/packetbeat.keystore"
+
 packetbeat.interfaces.device: any
 
 packetbeat.flows:
@@ -38,19 +40,19 @@ processors:
 - add_cloud_metadata:
 
 output.elasticsearch:
-  hosts: ['elasticsearch:9200']
+  hosts: ['localhost:9200'] # locahost cause we are using network mode = host
   protocol: "https"
   username: elastic
   # Read PW from packetbeat.keystore
   password: "${ELASTIC_PASSWORD}"
-  ssl.certificate_authorities: ["/usr/share/packetbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 setup.kibana:
-  host: "http://kibana:5601"
+  host: "http://localhost:5601" # locahost cause we are using network mode = host
   username: elastic
   password: "${ELASTIC_PASSWORD}"
   protocol: "http"
   ssl.enabled: false
-  ssl.certificate_authorities: ["/usr/share/packetbeat/certs/ca/ca.crt"]
+  ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"]
 
 xpack.monitoring.enabled: true