diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b68403e..9ff0107 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,8 +85,6 @@ jobs: docker-compose logs elasticsearch docker-compose logs logstash docker-compose logs kibana - # next steps don't need Logstash - docker-compose stop logstash ############################## # # @@ -94,6 +92,38 @@ jobs: # # ############################## + # + # Logspout + # + + - name: Execute Logspout test suite + run: | + + # Set mandatory Logstash settings + + sed -i '$ a input { udp { port => 5000 codec => json } }' logstash/pipeline/logstash.conf + + # Run Logspout and execute tests. + # Logstash will be restarted as a result of building the Logspout + # image, so changes above will automatically take effect. + + docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up -d logspout + .github/workflows/scripts/run-tests-logspout.sh + + # Revert changes to Logstash configuration + + sed -i '/input { udp { port => 5000 codec => json } }/d' logstash/pipeline/logstash.conf + + - name: 'debug: Display state and logs (Logspout)' + if: always() + run: | + docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps + docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout + # next steps don't need Logspout + docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml stop logspout + # next steps don't need Logstash + docker-compose stop logstash + # # Enterprise Search # @@ -155,6 +185,7 @@ jobs: run: >- docker-compose -f docker-compose.yml + -f extensions/logspout/logspout-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml -f extensions/apm-server/apm-server-compose.yml down -v diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh new file mode 100755 index 0000000..f2e9dde --- /dev/null +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + + +source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" + + +cid_es="$(container_id elasticsearch)" +cid_ls="$(container_id logspout)" + +ip_es="$(service_ip elasticsearch)" +ip_ls="$(service_ip logspout)" + +log 'Waiting for readiness of Elasticsearch' +poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' + +log 'Waiting for readiness of Logspout' +poll_ready "$cid_ls" "http://${ip_ls}/health" + +# When Logspout starts, it prints the following log line: +# 2021/01/07 16:14:52 # logspout v3.2.13-custom by gliderlabs +# +# which we expect to find by querying: +# docker.image:"docker-elk_logspout" AND message:"logspout gliderlabs"~3 +# +log 'Searching a log entry forwarded by Logspout' + +declare response +declare -i count + +# retry for max 60s (30*2s) +for _ in $(seq 1 30); do + response="$(curl "http://${ip_es}:9200/_count?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" + count="$(jq -rn --argjson data "${response}" '$data.count')" + if [[ $count -gt 0 ]]; then + break + fi + + echo -n 'x' >&2 + sleep 2 +done +echo -e '\n' >&2 + +echo "$response" +# Logspout may restart if Logstash isn't ready yet, so we tolerate multiple +# results +if [[ $count -lt 1 ]]; then + echo "Expected at least 1 document, got ${count}" + exit 1 +fi diff --git a/extensions/logspout/modules.go b/extensions/logspout/modules.go index a674bee..f1a2258 100644 --- a/extensions/logspout/modules.go +++ b/extensions/logspout/modules.go @@ -3,7 +3,8 @@ package main // installs the Logstash adapter for Logspout, and required dependencies // https://github.com/looplab/logspout-logstash import ( - _ "github.com/looplab/logspout-logstash" - _ "github.com/gliderlabs/logspout/transports/udp" + _ "github.com/gliderlabs/logspout/healthcheck" _ "github.com/gliderlabs/logspout/transports/tcp" + _ "github.com/gliderlabs/logspout/transports/udp" + _ "github.com/looplab/logspout-logstash" )