From 9afd2c160767ab2086438773ad1b9d4b644855ed Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 17 Feb 2023 10:52:32 +0100 Subject: [PATCH] ci: Mask wait messages, collect debug data once --- .github/workflows/ci.yml | 78 ++++++++----------- .github/workflows/scripts/lib/testing.sh | 11 +++ .github/workflows/scripts/run-tests-core.sh | 11 ++- .../scripts/run-tests-enterprise-search.sh | 8 +- .../workflows/scripts/run-tests-filebeat.sh | 8 +- .github/workflows/scripts/run-tests-fleet.sh | 11 ++- .../workflows/scripts/run-tests-heartbeat.sh | 8 +- .../workflows/scripts/run-tests-logspout.sh | 11 ++- .../workflows/scripts/run-tests-metricbeat.sh | 8 +- .github/workflows/spam-issue-close.yml | 29 ++----- 10 files changed, 94 insertions(+), 89 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e9cbb1..4a4052a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,17 +66,6 @@ jobs: - name: Execute core test suite run: .github/workflows/scripts/run-tests-core.sh - - name: 'debug: Display state and logs (core)' - # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idif - # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#job-status-check-functions - if: always() - run: | - docker compose ps - docker compose logs setup - docker compose logs elasticsearch - docker compose logs logstash - docker compose logs kibana - ############################## # # # Test supported extensions. # @@ -108,11 +97,6 @@ jobs: sed -i '/input { udp { port => 50000 codec => json } }/d' logstash/pipeline/logstash.conf docker compose restart logstash - - name: 'debug: Display state and logs (Logspout)' - if: always() - run: | - docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps - docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout # next steps don't need Logstash docker compose stop logstash @@ -125,13 +109,6 @@ jobs: docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml up --remove-orphans -d fleet-server apm-server .github/workflows/scripts/run-tests-fleet.sh - - name: 'debug: Display state and logs (Fleet)' - if: always() - run: | - docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml ps - docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml logs fleet-server - docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml logs apm-server - # # Metricbeat # @@ -141,12 +118,6 @@ jobs: docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up --remove-orphans -d metricbeat .github/workflows/scripts/run-tests-metricbeat.sh - - name: 'debug: Display state and logs (Metricbeat)' - if: always() - run: | - docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml ps - docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml logs metricbeat - # # Filebeat # @@ -156,12 +127,6 @@ jobs: docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up --remove-orphans -d filebeat .github/workflows/scripts/run-tests-filebeat.sh - - name: 'debug: Display state and logs (Filebeat)' - if: always() - run: | - docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml ps - docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml logs filebeat - # # Heartbeat # @@ -171,12 +136,6 @@ jobs: docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat .github/workflows/scripts/run-tests-heartbeat.sh - - name: 'debug: Display state and logs (Heartbeat)' - if: always() - run: | - docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml ps - docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml logs heartbeat - # # Enterprise Search # @@ -204,11 +163,42 @@ jobs: sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml docker compose restart elasticsearch - - name: 'debug: Display state and logs (Enterprise Search)' + - name: Collect troubleshooting data + id: debug-data if: always() run: | - docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml ps - docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search + declare debug_data_dir="$(mktemp -d)" + + docker compose \ + -f docker-compose.yml \ + -f extensions/logspout/logspout-compose.yml \ + -f extensions/fleet/fleet-compose.yml \ + -f extensions/fleet/agent-apmserver-compose.yml \ + -f extensions/metricbeat/metricbeat-compose.yml \ + -f extensions/filebeat/filebeat-compose.yml \ + -f extensions/heartbeat/heartbeat-compose.yml \ + -f extensions/enterprise-search/enterprise-search-compose.yml \ + ps >"$debug_data_dir"/docker_ps.log + + docker compose \ + -f docker-compose.yml \ + -f extensions/logspout/logspout-compose.yml \ + -f extensions/fleet/fleet-compose.yml \ + -f extensions/fleet/agent-apmserver-compose.yml \ + -f extensions/metricbeat/metricbeat-compose.yml \ + -f extensions/filebeat/filebeat-compose.yml \ + -f extensions/heartbeat/heartbeat-compose.yml \ + -f extensions/enterprise-search/enterprise-search-compose.yml \ + logs >"$debug_data_dir"/docker_logs.log + + echo "path=${debug_data_dir}" >>"$GITHUB_OUTPUT" + + - name: Upload collected troubleshooting data + if: always() && steps.debug-data.path + uses: actions/upload-artifact@v3 + with: + name: debug-data + path: ${{ steps.debug-data.outputs.path }}/*.* ############## # # diff --git a/.github/workflows/scripts/lib/testing.sh b/.github/workflows/scripts/lib/testing.sh index 1c77954..2d80992 100755 --- a/.github/workflows/scripts/lib/testing.sh +++ b/.github/workflows/scripts/lib/testing.sh @@ -10,6 +10,17 @@ function err { echo -e "\n[x] $1\n" >&2 } +# Start an expandable group in the GitHub Action log. +# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines +function grouplog { + echo "::group::$1" +} + +# End the current expandable group in the GitHub Action log. +function endgroup { + echo '::endgroup::' +} + # Return the ID of the container running the given service. function container_id { local svc=$1 diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index b2df747..849e52d 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -15,14 +15,17 @@ ip_es="$(service_ip elasticsearch)" ip_ls="$(service_ip logstash)" ip_kb="$(service_ip kibana)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Logstash' +grouplog 'Wait for readiness of Logstash' poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty" +endgroup -log 'Waiting for readiness of Kibana' +grouplog 'Wait for readiness of Kibana' poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" -u 'kibana_system:testpasswd' +endgroup log 'Sending message to Logstash TCP input' diff --git a/.github/workflows/scripts/run-tests-enterprise-search.sh b/.github/workflows/scripts/run-tests-enterprise-search.sh index 3a58afd..92158b3 100755 --- a/.github/workflows/scripts/run-tests-enterprise-search.sh +++ b/.github/workflows/scripts/run-tests-enterprise-search.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -13,11 +13,13 @@ cid_en="$(container_id enterprise-search)" ip_es="$(service_ip elasticsearch)" ip_en="$(service_ip enterprise-search)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Enterprise Search' +grouplog 'Wait for readiness of Enterprise Search' poll_ready "$cid_en" "http://${ip_en}:3002/api/ent/v1/internal/health" -u 'elastic:testpasswd' +endgroup log 'Ensuring that App Search API keys were created in Elasticsearch' response="$(curl "http://${ip_es}:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?q=*:*&pretty" -s -u elastic:testpasswd)" diff --git a/.github/workflows/scripts/run-tests-filebeat.sh b/.github/workflows/scripts/run-tests-filebeat.sh index 567c59d..0fe56de 100755 --- a/.github/workflows/scripts/run-tests-filebeat.sh +++ b/.github/workflows/scripts/run-tests-filebeat.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -13,11 +13,13 @@ cid_mb="$(container_id filebeat)" ip_es="$(service_ip elasticsearch)" ip_mb="$(service_ip filebeat)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Filebeat' +grouplog 'Wait for readiness of Filebeat' poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" +endgroup # We expect to find log entries for the 'elasticsearch' Compose service using # the following query: diff --git a/.github/workflows/scripts/run-tests-fleet.sh b/.github/workflows/scripts/run-tests-fleet.sh index 5d1c2a5..aed7d58 100755 --- a/.github/workflows/scripts/run-tests-fleet.sh +++ b/.github/workflows/scripts/run-tests-fleet.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -15,14 +15,17 @@ ip_es="$(service_ip elasticsearch)" ip_fl="$(service_ip fleet-server)" ip_apm="$(service_ip apm-server)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Fleet Server' +grouplog 'Wait for readiness of Fleet Server' poll_ready "$cid_fl" "http://${ip_fl}:8220/api/status" +endgroup -log 'Waiting for readiness of APM Server' +grouplog 'Wait for readiness of APM Server' poll_ready "$cid_apm" "http://${ip_apm}:8200/" +endgroup # We expect to find metrics entries using the following query: # diff --git a/.github/workflows/scripts/run-tests-heartbeat.sh b/.github/workflows/scripts/run-tests-heartbeat.sh index 882a977..7b44a54 100755 --- a/.github/workflows/scripts/run-tests-heartbeat.sh +++ b/.github/workflows/scripts/run-tests-heartbeat.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -13,11 +13,13 @@ cid_mb="$(container_id heartbeat)" ip_es="$(service_ip elasticsearch)" ip_mb="$(service_ip heartbeat)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Heartbeat' +grouplog 'Wait for readiness of Heartbeat' poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" +endgroup # We expect to find heartbeat entries for the 'elasticsearch' HTTP service # using the following query: diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh index 4bf0047..72c15c8 100755 --- a/.github/workflows/scripts/run-tests-logspout.sh +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -15,14 +15,17 @@ ip_es="$(service_ip elasticsearch)" ip_ls="$(service_ip logstash)" ip_lsp="$(service_ip logspout)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Logstash' +grouplog 'Wait for readiness of Logstash' poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty" +endgroup -log 'Waiting for readiness of Logspout' +grouplog 'Wait for readiness of Logspout' poll_ready "$cid_lsp" "http://${ip_lsp}/health" +endgroup # When Logspout starts, it prints the following log line: # 2021/01/07 16:14:52 # logspout v3.2.13-custom by gliderlabs diff --git a/.github/workflows/scripts/run-tests-metricbeat.sh b/.github/workflows/scripts/run-tests-metricbeat.sh index b0e9856..2cc9abe 100755 --- a/.github/workflows/scripts/run-tests-metricbeat.sh +++ b/.github/workflows/scripts/run-tests-metricbeat.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" +source "${BASH_SOURCE[0]%/*}"/lib/testing.sh cid_es="$(container_id elasticsearch)" @@ -13,11 +13,13 @@ cid_mb="$(container_id metricbeat)" ip_es="$(service_ip elasticsearch)" ip_mb="$(service_ip metricbeat)" -log 'Waiting for readiness of Elasticsearch' +grouplog 'Wait for readiness of Elasticsearch' poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' +endgroup -log 'Waiting for readiness of Metricbeat' +grouplog 'Wait for readiness of Metricbeat' poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty" +endgroup # We expect to find monitoring entries for the 'elasticsearch' Compose service # using the following query: diff --git a/.github/workflows/spam-issue-close.yml b/.github/workflows/spam-issue-close.yml index cb07bd8..caad7c2 100644 --- a/.github/workflows/spam-issue-close.yml +++ b/.github/workflows/spam-issue-close.yml @@ -24,32 +24,19 @@ jobs: stale-issue-label: bot:close close-issue-label: insufficient information close-issue-message: >- - To avoid frequent and unnecessary back and forth in issue comments to ask for information that could have - been provided right away in the issue description, maintainers created a clear issue template with all the - information they need to be able to reproduce common issues, and ask everyone to follow it. + This description omits all, or critical parts of the information requested by maintainers to be able to + reproduce the issue: - **It appears that this issue description omits all, or critical parts the requested information.** + - the **complete** log history of your Elastic components, including `setup`. + - any change(s) performed to the docker-elk configuration. + - details about the runtime environment, for both Docker and Compose. - Maintainers of this project wish they had the superpower to read minds. Alas, they are mere mortals who - cannot understand the context in which this problem occurred without information such as: + Therefore, this issue will now be **closed**. Please open a new issue and fill in the template. It saves + everyone's efforts, and allows maintainers to provide you with a solution in as few round trips as possible. - - - your docker-elk configuration - - details about your runtime environment - - the complete log history of your Elastic components - - - Therefore, this issue will now be **closed**. Thank you for your understanding. :pray: - - - --- - - - Next time, please be respectful of maintainers' time by providing the requested information right away. It - saves everyone's efforts, and allows them to provide you with a solution with as few round trips as - possible. + Thank you for your understanding. :pray: # Due to eventual consistency, listing closed issues immediately after a # close does not always yield the expected results. A sleep is a simple