ci: Mask wait messages, collect debug data once

.github/workflows/ci.yml

@@ -66,17 +66,6 @@ jobs:
       - name: Execute core test suite
         run: .github/workflows/scripts/run-tests-core.sh
 
-      - name: 'debug: Display state and logs (core)'
-        # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idif
-        # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#job-status-check-functions
-        if: always()
-        run: |
-          docker compose ps
-          docker compose logs setup
-          docker compose logs elasticsearch
-          docker compose logs logstash
-          docker compose logs kibana
-
       ##############################
       #                            #
       # Test supported extensions. #
@@ -108,11 +97,6 @@ jobs:
           sed -i '/input { udp { port => 50000 codec => json } }/d' logstash/pipeline/logstash.conf
           docker compose restart logstash
 
-      - name: 'debug: Display state and logs (Logspout)'
-        if: always()
-        run: |
-          docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps
-          docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout
           # next steps don't need Logstash
           docker compose stop logstash
 
@@ -125,13 +109,6 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml up --remove-orphans -d fleet-server apm-server
           .github/workflows/scripts/run-tests-fleet.sh
 
-      - name: 'debug: Display state and logs (Fleet)'
-        if: always()
-        run: |
-          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml ps
-          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml logs fleet-server
-          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml logs apm-server
-
       #
       # Metricbeat
       #
@@ -141,12 +118,6 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up --remove-orphans -d metricbeat
           .github/workflows/scripts/run-tests-metricbeat.sh
 
-      - name: 'debug: Display state and logs (Metricbeat)'
-        if: always()
-        run: |
-          docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps
-          docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat
-
       #
       # Filebeat
       #
@@ -156,12 +127,6 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up --remove-orphans -d filebeat
           .github/workflows/scripts/run-tests-filebeat.sh
 
-      - name: 'debug: Display state and logs (Filebeat)'
-        if: always()
-        run: |
-          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml ps
-          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml logs filebeat
-
       #
       # Heartbeat
       #
@@ -171,12 +136,6 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat
           .github/workflows/scripts/run-tests-heartbeat.sh
 
-      - name: 'debug: Display state and logs (Heartbeat)'
-        if: always()
-        run: |
-          docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml ps
-          docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml logs heartbeat
-
       #
       # Enterprise Search
       #
@@ -204,11 +163,42 @@ jobs:
           sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
           docker compose restart elasticsearch
 
-      - name: 'debug: Display state and logs (Enterprise Search)'
+      - name: Collect troubleshooting data
+        id: debug-data
         if: always()
         run: |
-          docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml ps
+          declare debug_data_dir="$(mktemp -d)"
-          docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search
+
+          docker compose \
+          -f docker-compose.yml \
+          -f extensions/logspout/logspout-compose.yml \
+          -f extensions/fleet/fleet-compose.yml \
+          -f extensions/fleet/agent-apmserver-compose.yml \
+          -f extensions/metricbeat/metricbeat-compose.yml \
+          -f extensions/filebeat/filebeat-compose.yml \
+          -f extensions/heartbeat/heartbeat-compose.yml \
+          -f extensions/enterprise-search/enterprise-search-compose.yml \
+          ps >"$debug_data_dir"/docker_ps.log
+
+          docker compose \
+          -f docker-compose.yml \
+          -f extensions/logspout/logspout-compose.yml \
+          -f extensions/fleet/fleet-compose.yml \
+          -f extensions/fleet/agent-apmserver-compose.yml \
+          -f extensions/metricbeat/metricbeat-compose.yml \
+          -f extensions/filebeat/filebeat-compose.yml \
+          -f extensions/heartbeat/heartbeat-compose.yml \
+          -f extensions/enterprise-search/enterprise-search-compose.yml \
+          logs >"$debug_data_dir"/docker_logs.log
+
+          echo "path=${debug_data_dir}" >>"$GITHUB_OUTPUT"
+
+      - name: Upload collected troubleshooting data
+        if: always() && steps.debug-data.path
+        uses: actions/upload-artifact@v3
+        with:
+          name: debug-data
+          path: ${{ steps.debug-data.outputs.path }}/*.*
 
         ##############
         #            #

.github/workflows/scripts/lib/testing.sh

@@ -10,6 +10,17 @@ function err {
 	echo -e "\n[x] $1\n" >&2
 }
 
+# Start an expandable group in the GitHub Action log.
+# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
+function grouplog {
+	echo "::group::$1"
+}
+
+# End the current expandable group in the GitHub Action log.
+function endgroup {
+	echo '::endgroup::'
+}
+
 # Return the ID of the container running the given service.
 function container_id {
 	local svc=$1

.github/workflows/scripts/run-tests-core.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -15,14 +15,17 @@ ip_es="$(service_ip elasticsearch)"
 ip_ls="$(service_ip logstash)"
 ip_kb="$(service_ip kibana)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Logstash'
+grouplog 'Wait for readiness of Logstash'
 poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
+endgroup
 
-log 'Waiting for readiness of Kibana'
+grouplog 'Wait for readiness of Kibana'
 poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" -u 'kibana_system:testpasswd'
+endgroup
 
 log 'Sending message to Logstash TCP input'
 

.github/workflows/scripts/run-tests-enterprise-search.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -13,11 +13,13 @@ cid_en="$(container_id enterprise-search)"
 ip_es="$(service_ip elasticsearch)"
 ip_en="$(service_ip enterprise-search)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Enterprise Search'
+grouplog 'Wait for readiness of Enterprise Search'
 poll_ready "$cid_en" "http://${ip_en}:3002/api/ent/v1/internal/health" -u 'elastic:testpasswd'
+endgroup
 
 log 'Ensuring that App Search API keys were created in Elasticsearch'
 response="$(curl "http://${ip_es}:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?q=*:*&pretty" -s -u elastic:testpasswd)"

.github/workflows/scripts/run-tests-filebeat.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -13,11 +13,13 @@ cid_mb="$(container_id filebeat)"
 ip_es="$(service_ip elasticsearch)"
 ip_mb="$(service_ip filebeat)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Filebeat'
+grouplog 'Wait for readiness of Filebeat'
 poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
+endgroup
 
 # We expect to find log entries for the 'elasticsearch' Compose service using
 # the following query:

.github/workflows/scripts/run-tests-fleet.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -15,14 +15,17 @@ ip_es="$(service_ip elasticsearch)"
 ip_fl="$(service_ip fleet-server)"
 ip_apm="$(service_ip apm-server)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Fleet Server'
+grouplog 'Wait for readiness of Fleet Server'
 poll_ready "$cid_fl" "http://${ip_fl}:8220/api/status"
+endgroup
 
-log 'Waiting for readiness of APM Server'
+grouplog 'Wait for readiness of APM Server'
 poll_ready "$cid_apm" "http://${ip_apm}:8200/"
+endgroup
 
 # We expect to find metrics entries using the following query:
 #

.github/workflows/scripts/run-tests-heartbeat.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -13,11 +13,13 @@ cid_mb="$(container_id heartbeat)"
 ip_es="$(service_ip elasticsearch)"
 ip_mb="$(service_ip heartbeat)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Heartbeat'
+grouplog 'Wait for readiness of Heartbeat'
 poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
+endgroup
 
 # We expect to find heartbeat entries for the 'elasticsearch' HTTP service
 # using the following query:

.github/workflows/scripts/run-tests-logspout.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -15,14 +15,17 @@ ip_es="$(service_ip elasticsearch)"
 ip_ls="$(service_ip logstash)"
 ip_lsp="$(service_ip logspout)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Logstash'
+grouplog 'Wait for readiness of Logstash'
 poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
+endgroup
 
-log 'Waiting for readiness of Logspout'
+grouplog 'Wait for readiness of Logspout'
 poll_ready "$cid_lsp" "http://${ip_lsp}/health"
+endgroup
 
 # When Logspout starts, it prints the following log line:
 #   2021/01/07 16:14:52 # logspout v3.2.13-custom by gliderlabs

.github/workflows/scripts/run-tests-metricbeat.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
 
 
 cid_es="$(container_id elasticsearch)"
@@ -13,11 +13,13 @@ cid_mb="$(container_id metricbeat)"
 ip_es="$(service_ip elasticsearch)"
 ip_mb="$(service_ip metricbeat)"
 
-log 'Waiting for readiness of Elasticsearch'
+grouplog 'Wait for readiness of Elasticsearch'
 poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+endgroup
 
-log 'Waiting for readiness of Metricbeat'
+grouplog 'Wait for readiness of Metricbeat'
 poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
+endgroup
 
 # We expect to find monitoring entries for the 'elasticsearch' Compose service
 # using the following query:

.github/workflows/spam-issue-close.yml

@@ -24,32 +24,19 @@ jobs:
           stale-issue-label: bot:close
           close-issue-label: insufficient information
           close-issue-message: >-
-            To avoid frequent and unnecessary back and forth in issue comments to ask for information that could have
+            This description omits all, or critical parts of the information requested by maintainers to be able to
-            been provided right away in the issue description, maintainers created a clear issue template with all the
+            reproduce the issue:
-            information they need to be able to reproduce common issues, and ask everyone to follow it.
 
 
-            **It appears that this issue description omits all, or critical parts the requested information.**
+             - the **complete** log history of your Elastic components, including `setup`.
+             - any change(s) performed to the docker-elk configuration.
+             - details about the runtime environment, for both Docker and Compose.
 
 
-            Maintainers of this project wish they had the superpower to read minds. Alas, they are mere mortals who
+            Therefore, this issue will now be **closed**. Please open a new issue and fill in the template. It saves
-            cannot understand the context in which this problem occurred without information such as:
+            everyone's efforts, and allows maintainers to provide you with a solution in as few round trips as possible.
 
-
+            Thank you for your understanding. :pray:
-             - your docker-elk configuration
-             - details about your runtime environment
-             - the complete log history of your Elastic components
-
-
-            Therefore, this issue will now be **closed**. Thank you for your understanding. :pray:
-
-
-            ---
-
-
-            Next time, please be respectful of maintainers' time by providing the requested information right away. It
-            saves everyone's efforts, and allows them to provide you with a solution with as few round trips as
-            possible.
 
       # Due to eventual consistency, listing closed issues immediately after a
       # close does not always yield the expected results. A sleep is a simple