fix(setup): Ensure built-in users exist before proceeding

Fixes #786

setup/entrypoint.sh

@@ -3,7 +3,7 @@
 set -eu
 set -o pipefail
 
-source "$(dirname "${BASH_SOURCE[0]}")/helpers.sh"
+source "${BASH_SOURCE[0]%/*}"/helpers.sh
 
 
 # --------------------------------------------------------
@@ -33,7 +33,7 @@ roles_files=(
 
 echo "-------- $(date) --------"
 
-state_file="$(dirname "${BASH_SOURCE[0]}")/state/.done"
+state_file="${BASH_SOURCE[0]%/*}"/state/.done
 if [[ -e "$state_file" ]]; then
 	log "State file exists at '${state_file}', skipping setup"
 	exit 0
@@ -65,11 +65,22 @@ fi
 
 sublog 'Elasticsearch is running'
 
+log 'Waiting for initialization of built-in users'
+
+wait_for_builtin_users || exit_code=$?
+
+if ((exit_code)); then
+	suberr 'Timed out waiting for condition'
+	exit $exit_code
+fi
+
+sublog 'Built-in users were initialized'
+
 for role in "${!roles_files[@]}"; do
 	log "Role '$role'"
 
 	declare body_file
-	body_file="$(dirname "${BASH_SOURCE[0]}")/roles/${roles_files[$role]:-}"
+	body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
 	if [[ ! -f "${body_file:-}" ]]; then
 		sublog "No role body found at '${body_file}', skipping"
 		continue
@@ -94,7 +105,7 @@ for user in "${!users_passwords[@]}"; do
 		set_user_password "$user" "${users_passwords[$user]}"
 	else
 		if [[ -z "${users_roles[$user]:-}" ]]; then
-			err '  No role defined, skipping creation'
+			suberr '  No role defined, skipping creation'
 			continue
 		fi
 
@@ -103,5 +114,5 @@ for user in "${!users_passwords[@]}"; do
 	fi
 done
 
-mkdir -p "$(dirname "${state_file}")"
+mkdir -p "${state_file%/*}"
 touch "$state_file"

setup/helpers.sh

@@ -57,6 +57,53 @@ function wait_for_elasticsearch {
 	return $result
 }
 
+# Poll the Elasticsearch users API until it returns users.
+function wait_for_builtin_users {
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" )
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+
+	local line
+	local -i exit_code
+	local -i num_users
+
+	# retry for max 30s (30*1s)
+	for _ in $(seq 1 30); do
+		num_users=0
+
+		# read exits with a non-zero code if the last read input doesn't end
+		# with a newline character. The printf without newline that follows the
+		# curl command ensures that the final input not only contains curl's
+		# exit code, but causes read to fail so we can capture the return value.
+		# Ref. https://unix.stackexchange.com/a/176703/152409
+		while IFS= read -r line || ! exit_code="$line"; do
+			if [[ "$line" =~ _reserved.+true ]]; then
+				(( num_users++ ))
+			fi
+		done < <(curl "${args[@]}"; printf '%s' "$?")
+
+		if ((exit_code)); then
+			result=$exit_code
+		fi
+
+		# we expect more than just the 'elastic' user in the result
+		if (( num_users > 1 )); then
+			result=0
+			break
+		fi
+
+		sleep 1
+	done
+
+	return $result
+}
+
 # Verify that the given Elasticsearch user exists.
 function check_user_exists {
 	local username=$1