Add Filebeat extension

.github/workflows/ci.yml

@@ -51,6 +51,7 @@ jobs:
             -f extensions/enterprise-search/enterprise-search-compose.yml \
             -f extensions/apm-server/apm-server-compose.yml \
             -f extensions/metricbeat/metricbeat-compose.yml \
+            -f extensions/filebeat/filebeat-compose.yml \
             build
 
       ########################################################
@@ -70,6 +71,7 @@ jobs:
           sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
           sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml
           sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml
+          sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml
 
           # Run Elasticsearch and wait for its availability
 
@@ -211,6 +213,23 @@ jobs:
           # next steps don't need Metricbeat
           docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat
 
+      #
+      # Filebeat
+      #
+
+      - name: Execute Filebeat test suite
+        run: |
+          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up -d filebeat
+          .github/workflows/scripts/run-tests-filebeat.sh
+
+      - name: 'debug: Display state and logs (Filebeat)'
+        if: always()
+        run: |
+          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml ps
+          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml logs filebeat
+          # next steps don't need Filebeat
+          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml stop filebeat
+
         ##############
         #            #
         # Tear down. #
@@ -226,6 +245,7 @@ jobs:
           -f extensions/enterprise-search/enterprise-search-compose.yml
           -f extensions/apm-server/apm-server-compose.yml
           -f extensions/metricbeat/metricbeat-compose.yml
+          -f extensions/filebeat/filebeat-compose.yml
           down -v
 
   test-swarm:

.github/workflows/scripts/run-tests-filebeat.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+cid_es="$(container_id elasticsearch)"
+cid_mb="$(container_id filebeat)"
+
+ip_es="$(service_ip elasticsearch)"
+ip_mb="$(service_ip filebeat)"
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+
+log 'Waiting for readiness of Filebeat'
+poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
+
+# We expect to find log entries for the 'elasticsearch' Compose service using
+# the following query:
+#
+#   agent.type:"filebeat"
+#   AND input.type:"container"
+#   AND container.name:"docker-elk-elasticsearch-1"
+#
+log 'Searching documents generated by Filebeat'
+
+declare response
+declare -i count
+
+declare -i was_retried=0
+
+# retry for max 60s (30*2s)
+for _ in $(seq 1 30); do
+	response="$(curl "http://${ip_es}:9200/filebeat-*/_search?q=agent.type:%22filebeat%22%20AND%20input.type:%22container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)"
+
+	set +u  # prevent "unbound variable" if assigned value is not an integer
+	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
+	set -u
+
+	if (( count > 0 )); then
+		break
+	fi
+
+	was_retried=1
+	echo -n 'x' >&2
+	sleep 2
+done
+if ((was_retried)); then
+	# flush stderr, important in non-interactive environments (CI)
+	echo >&2
+fi
+
+echo "$response"
+if (( count == 0 )); then
+	echo 'Expected at least 1 document'
+	exit 1
+fi

extensions/filebeat/Dockerfile

@@ -0,0 +1,3 @@
+ARG ELK_VERSION
+
+FROM docker.elastic.co/beats/filebeat:${ELK_VERSION}

extensions/filebeat/README.md

@@ -0,0 +1,36 @@
+# Filebeat
+
+Filebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers,
+Filebeat monitors the log files or locations that you specify, collects log events, and forwards them either to
+Elasticsearch or Logstash for indexing.
+
+## Usage
+
+To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line
+argument referencing the `filebeat-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up
+```
+
+## Configuring Filebeat
+
+The Filebeat configuration is stored in [`config/filebeat.yml`](./config/filebeat.yml). You can modify this file with
+the help of the [Configuration reference][filebeat-config].
+
+Any change to the Filebeat configuration requires a restart of the Filebeat container:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat
+```
+
+Please refer to the following documentation page for more details about how to configure Filebeat inside a Docker
+container: [Run Filebeat on Docker][filebeat-docker].
+
+## See also
+
+[Filebeat documentation][filebeat-doc]
+
+[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
+[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
+[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html

extensions/filebeat/config/filebeat.yml

@@ -0,0 +1,30 @@
+## Filebeat configuration
+## https://github.com/elastic/beats/blob/master/deploy/docker/filebeat.docker.yml
+#
+
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+filebeat.autodiscover:
+  providers:
+    # The Docker autodiscover provider automatically retrieves logs from Docker
+    # containers as they start and stop.
+    - type: docker
+      hints.enabled: true
+
+processors:
+  - add_cloud_metadata: ~
+
+output.elasticsearch:
+  hosts: ['http://elasticsearch:9200']
+  username: elastic
+  password: changeme
+
+## HTTP endpoint for health checking
+## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html
+#
+
+http.enabled: true
+http.host: 0.0.0.0

extensions/filebeat/filebeat-compose.yml

@@ -0,0 +1,35 @@
+version: '3.2'
+
+services:
+  filebeat:
+    build:
+      context: extensions/filebeat/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading
+    # 'docker.sock' and the host's filesystem.
+    user: root
+    command:
+      # Log to stderr.
+      - -e
+      # Disable config file permissions checks. Allows mounting
+      # 'config/filebeat.yml' even if it's not owned by root.
+      # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
+      - --strict.perms=false
+    volumes:
+      - type: bind
+        source: ./extensions/filebeat/config/filebeat.yml
+        target: /usr/share/filebeat/filebeat.yml
+        read_only: true
+      - type: bind
+        source: /var/lib/docker/containers
+        target: /var/lib/docker/containers
+        read_only: true
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+        read_only: true
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch

extensions/metricbeat/metricbeat-compose.yml

@@ -10,14 +10,14 @@ services:
     # 'docker.sock' and the host's filesystem.
     user: root
     command:
-        # Log to stderr.
+      # Log to stderr.
       - -e
-        # Disable config file permissions checks. Allows mounting
-        # 'config/metricbeat.yml' even if it's not owned by root.
-        # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
+      # Disable config file permissions checks. Allows mounting
+      # 'config/metricbeat.yml' even if it's not owned by root.
+      # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
       - --strict.perms=false
-        # Mount point of the host’s filesystem. Required to monitor the host
-        # from within a container.
+      # Mount point of the host’s filesystem. Required to monitor the host
+      # from within a container.
       - --system.hostfs=/hostfs
     volumes:
       - type: bind