From 1815972838cef8c838eedffc6fb74672f6026abb Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Sat, 4 Dec 2021 00:41:45 +0100 Subject: [PATCH] Add Filebeat extension --- .github/workflows/ci.yml | 20 ++++++ .../workflows/scripts/run-tests-filebeat.sh | 61 +++++++++++++++++++ extensions/filebeat/Dockerfile | 3 + extensions/filebeat/README.md | 36 +++++++++++ extensions/filebeat/config/filebeat.yml | 30 +++++++++ extensions/filebeat/filebeat-compose.yml | 35 +++++++++++ extensions/metricbeat/metricbeat-compose.yml | 12 ++-- 7 files changed, 191 insertions(+), 6 deletions(-) create mode 100755 .github/workflows/scripts/run-tests-filebeat.sh create mode 100644 extensions/filebeat/Dockerfile create mode 100644 extensions/filebeat/README.md create mode 100644 extensions/filebeat/config/filebeat.yml create mode 100644 extensions/filebeat/filebeat-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a558020..6d81b58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,7 @@ jobs: -f extensions/enterprise-search/enterprise-search-compose.yml \ -f extensions/apm-server/apm-server-compose.yml \ -f extensions/metricbeat/metricbeat-compose.yml \ + -f extensions/filebeat/filebeat-compose.yml \ build ######################################################## @@ -70,6 +71,7 @@ jobs: sed -i -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml + sed -i 's/\(password:\) changeme/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml # Run Elasticsearch and wait for its availability @@ -211,6 +213,23 @@ jobs: # next steps don't need Metricbeat docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml stop metricbeat + # + # Filebeat + # + + - name: Execute Filebeat test suite + run: | + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up -d filebeat + .github/workflows/scripts/run-tests-filebeat.sh + + - name: 'debug: Display state and logs (Filebeat)' + if: always() + run: | + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml ps + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml logs filebeat + # next steps don't need Filebeat + docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml stop filebeat + ############## # # # Tear down. # @@ -226,6 +245,7 @@ jobs: -f extensions/enterprise-search/enterprise-search-compose.yml -f extensions/apm-server/apm-server-compose.yml -f extensions/metricbeat/metricbeat-compose.yml + -f extensions/filebeat/filebeat-compose.yml down -v test-swarm: diff --git a/.github/workflows/scripts/run-tests-filebeat.sh b/.github/workflows/scripts/run-tests-filebeat.sh new file mode 100755 index 0000000..567c59d --- /dev/null +++ b/.github/workflows/scripts/run-tests-filebeat.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + + +source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" + + +cid_es="$(container_id elasticsearch)" +cid_mb="$(container_id filebeat)" + +ip_es="$(service_ip elasticsearch)" +ip_mb="$(service_ip filebeat)" + +log 'Waiting for readiness of Elasticsearch' +poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' + +log 'Waiting for readiness of Filebeat' +poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" + +# We expect to find log entries for the 'elasticsearch' Compose service using +# the following query: +# +# agent.type:"filebeat" +# AND input.type:"container" +# AND container.name:"docker-elk-elasticsearch-1" +# +log 'Searching documents generated by Filebeat' + +declare response +declare -i count + +declare -i was_retried=0 + +# retry for max 60s (30*2s) +for _ in $(seq 1 30); do + response="$(curl "http://${ip_es}:9200/filebeat-*/_search?q=agent.type:%22filebeat%22%20AND%20input.type:%22container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)" + + set +u # prevent "unbound variable" if assigned value is not an integer + count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" + set -u + + if (( count > 0 )); then + break + fi + + was_retried=1 + echo -n 'x' >&2 + sleep 2 +done +if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 +fi + +echo "$response" +if (( count == 0 )); then + echo 'Expected at least 1 document' + exit 1 +fi diff --git a/extensions/filebeat/Dockerfile b/extensions/filebeat/Dockerfile new file mode 100644 index 0000000..b70aa10 --- /dev/null +++ b/extensions/filebeat/Dockerfile @@ -0,0 +1,3 @@ +ARG ELK_VERSION + +FROM docker.elastic.co/beats/filebeat:${ELK_VERSION} diff --git a/extensions/filebeat/README.md b/extensions/filebeat/README.md new file mode 100644 index 0000000..43c5d67 --- /dev/null +++ b/extensions/filebeat/README.md @@ -0,0 +1,36 @@ +# Filebeat + +Filebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers, +Filebeat monitors the log files or locations that you specify, collects log events, and forwards them either to +Elasticsearch or Logstash for indexing. + +## Usage + +To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line +argument referencing the `filebeat-compose.yml` file: + +```console +$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up +``` + +## Configuring Filebeat + +The Filebeat configuration is stored in [`config/filebeat.yml`](./config/filebeat.yml). You can modify this file with +the help of the [Configuration reference][filebeat-config]. + +Any change to the Filebeat configuration requires a restart of the Filebeat container: + +```console +$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat +``` + +Please refer to the following documentation page for more details about how to configure Filebeat inside a Docker +container: [Run Filebeat on Docker][filebeat-docker]. + +## See also + +[Filebeat documentation][filebeat-doc] + +[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html +[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html +[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html diff --git a/extensions/filebeat/config/filebeat.yml b/extensions/filebeat/config/filebeat.yml new file mode 100644 index 0000000..347d070 --- /dev/null +++ b/extensions/filebeat/config/filebeat.yml @@ -0,0 +1,30 @@ +## Filebeat configuration +## https://github.com/elastic/beats/blob/master/deploy/docker/filebeat.docker.yml +# + +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + # The Docker autodiscover provider automatically retrieves logs from Docker + # containers as they start and stop. + - type: docker + hints.enabled: true + +processors: + - add_cloud_metadata: ~ + +output.elasticsearch: + hosts: ['http://elasticsearch:9200'] + username: elastic + password: changeme + +## HTTP endpoint for health checking +## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html +# + +http.enabled: true +http.host: 0.0.0.0 diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml new file mode 100644 index 0000000..40b1150 --- /dev/null +++ b/extensions/filebeat/filebeat-compose.yml @@ -0,0 +1,35 @@ +version: '3.2' + +services: + filebeat: + build: + context: extensions/filebeat/ + args: + ELK_VERSION: $ELK_VERSION + # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading + # 'docker.sock' and the host's filesystem. + user: root + command: + # Log to stderr. + - -e + # Disable config file permissions checks. Allows mounting + # 'config/filebeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + - --strict.perms=false + volumes: + - type: bind + source: ./extensions/filebeat/config/filebeat.yml + target: /usr/share/filebeat/filebeat.yml + read_only: true + - type: bind + source: /var/lib/docker/containers + target: /var/lib/docker/containers + read_only: true + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: true + networks: + - elk + depends_on: + - elasticsearch diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index c8f9032..81ceb35 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -10,14 +10,14 @@ services: # 'docker.sock' and the host's filesystem. user: root command: - # Log to stderr. + # Log to stderr. - -e - # Disable config file permissions checks. Allows mounting - # 'config/metricbeat.yml' even if it's not owned by root. - # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + # Disable config file permissions checks. Allows mounting + # 'config/metricbeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html - --strict.perms=false - # Mount point of the host’s filesystem. Required to monitor the host - # from within a container. + # Mount point of the host’s filesystem. Required to monitor the host + # from within a container. - --system.hostfs=/hostfs volumes: - type: bind