Travis: improve robustness of readiness checks

.travis.yml

@@ -18,9 +18,6 @@ before_script:
   # Build images
   - docker-compose build
 
-  # Pull buildpack image (curl 7.52.0+)
-  - docker pull buildpack-deps:artful-curl
-
   # Use built-in users
   - sed -i 's/\(elasticsearch.username:\) elastic/\1 kibana/g' kibana/config/kibana.yml
   - sed -i 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' logstash/config/logstash.yml
@@ -31,7 +28,6 @@ script:
   - sleep 20
   - .travis/elasticsearch-setup-passwords.exp
   - docker-compose up -d
-  - sleep 90
   - .travis/run-tests.sh
   - docker-compose ps
   - docker-compose logs elasticsearch
@@ -44,12 +40,11 @@ script:
   - docker stack deploy -c ./docker-stack.yml elk
   - docker service scale elk_kibana=0 --detach=false
   - docker service scale elk_logstash=0 --detach=false
-  - sleep 60
+  - sleep 30
   - .travis/elasticsearch-setup-passwords.exp swarm
   - docker service scale elk_kibana=1 --detach=false
   - docker service scale elk_logstash=1 --detach=false
-  - sleep 90
+  - .travis/run-tests.sh swarm
-  - .travis/run-tests.sh
   - docker stack services elk
   - docker service logs elk_elasticsearch
   - docker service logs elk_kibana

.travis/run-tests.sh

@@ -3,36 +3,73 @@
 set -eu
 set -o pipefail
 
-shopt -s expand_aliases
-alias curl="docker run --rm --net=host buildpack-deps:artful-curl curl -s -w '\n'"
-
 function log {
 	echo -e "\n[+] $1\n"
 }
 
+function poll_ready {
+	local svc=$1
+	local url=$2
+
+	local -a args=( '-s' '-D-' '-w' '%{http_code}' "$url" )
+	if [ "$#" -ge 3 ]; then
+		args+=( '-u' "$3" )
+	fi
+
+	local label
+	if [ "$MODE" == "swarm" ]; then
+		label="com.docker.swarm.service.name=elk_${svc}"
+	else
+		label="com.docker.compose.service=${svc}"
+	fi
+
+	local -i result=1
+	local cid
+	local output
+
+	# retry for max 90s (18*5s)
+	for _ in $(seq 1 18); do
+		cid="$(docker ps -q -f label="$label")"
+		if [ -z "${cid:-}" ]; then
+			echo "Container exited"
+			return 1
+		fi
+
+		set +e
+		output="$(curl "${args[@]}")"
+		set -e
+		if [ "${output: -3}" -eq 200 ]; then
+			result=0
+			break
+		fi
+
+		echo -n '.'
+		sleep 5
+	done
+
+	echo -e "\n${output::-3}"
+
+	return $result
+}
+
+declare MODE=""
+if [ "$#" -ge 1 ]; then
+	MODE=$1
+fi
+
 log 'Waiting for Elasticsearch readiness'
-curl -D- 'http://localhost:9200/' \
+poll_ready elasticsearch 'http://localhost:9200/' 'elastic:changeme'
-	--retry 10 \
-	--retry-delay 5 \
-	--retry-connrefused \
-	-u elastic:changeme
 
 log 'Waiting for Kibana readiness'
-curl -D- 'http://localhost:5601/api/status' \
+poll_ready kibana 'http://localhost:5601/api/status' 'kibana:changeme'
-	--retry 10 \
-	--retry-delay 5 \
-	--retry-connrefused \
-	-u kibana:changeme
 
 log 'Waiting for Logstash readiness'
-curl -D- 'http://localhost:9600/_node/pipelines/main?pretty' \
+poll_ready logstash 'http://localhost:9600/_node/pipelines/main?pretty'
-	--retry 10 \
-	--retry-delay 5 \
-	--retry-connrefused
 
 log 'Creating Logstash index pattern in Kibana'
 source .env
 curl -X POST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \
+	-s -w '\n' \
 	-H 'Content-Type: application/json' \
 	-H "kbn-version: ${ELK_VERSION}" \
 	-u elastic:changeme \
@@ -40,7 +77,7 @@ curl -X POST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \
 
 log 'Searching index pattern via Kibana API'
 response="$(curl 'http://localhost:5601/api/saved_objects/_find?type=index-pattern' -u elastic:changeme)"
-echo $response
+echo "$response"
 count="$(jq -rn --argjson data "${response}" '$data.total')"
 if [[ $count -ne 1 ]]; then
 	echo "Expected 1 index pattern, got ${count}"
@@ -51,11 +88,12 @@ log 'Sending message to Logstash TCP input'
 echo 'dockerelk' | nc localhost 5000
 
 sleep 1
-curl -X POST 'http://localhost:9200/_refresh' -u elastic:changeme
+curl -X POST 'http://localhost:9200/_refresh' -u elastic:changeme \
+	-s -w '\n'
 
 log 'Searching message in Elasticsearch'
 response="$(curl 'http://localhost:9200/_count?q=message:dockerelk&pretty' -u elastic:changeme)"
-echo $response
+echo "$response"
 count="$(jq -rn --argjson data "${response}" '$data.count')"
 if [[ $count -ne 1 ]]; then
 	echo "Expected 1 document, got ${count}"