From 30cb8cb484a9b588350314c2aa63b1d25cc542f6 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Thu, 6 Oct 2022 11:58:02 +0200 Subject: [PATCH] feat: Add Fleet extension, remove standalone APM Server (#760) The preferred way to run APM Server is via the APM integration of the Elastic Agent. Ref. https://www.elastic.co/guide/en/apm/guide/8.4/legacy-apm-overview.html --- .github/workflows/ci.yml | 97 +++++++++---------- .../workflows/scripts/run-tests-apm-server.sh | 14 --- .github/workflows/scripts/run-tests-fleet.sh | 63 ++++++++++++ docker-compose.yml | 2 + extensions/apm-server/Dockerfile | 3 - extensions/apm-server/README.md | 56 ----------- extensions/apm-server/apm-server-compose.yml | 22 ----- extensions/apm-server/config/apm-server.yml | 8 -- .../{apm-server => fleet}/.dockerignore | 0 extensions/fleet/Dockerfile | 3 + extensions/fleet/README.md | 63 ++++++++++++ extensions/fleet/fleet-compose.yml | 26 +++++ kibana/config/kibana.yml | 34 ++++++- 13 files changed, 237 insertions(+), 154 deletions(-) delete mode 100755 .github/workflows/scripts/run-tests-apm-server.sh create mode 100755 .github/workflows/scripts/run-tests-fleet.sh delete mode 100644 extensions/apm-server/Dockerfile delete mode 100644 extensions/apm-server/README.md delete mode 100644 extensions/apm-server/apm-server-compose.yml delete mode 100644 extensions/apm-server/config/apm-server.yml rename extensions/{apm-server => fleet}/.dockerignore (100%) create mode 100644 extensions/fleet/Dockerfile create mode 100644 extensions/fleet/README.md create mode 100644 extensions/fleet/fleet-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26a387a..1bc5a83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,20 +23,17 @@ jobs: # # ##################################################### - - name: Prepare environment - run: | - - # Pre-build container images - - docker compose \ - -f docker-compose.yml \ - -f extensions/logspout/logspout-compose.yml \ - -f extensions/enterprise-search/enterprise-search-compose.yml \ - -f extensions/apm-server/apm-server-compose.yml \ - -f extensions/metricbeat/metricbeat-compose.yml \ - -f extensions/filebeat/filebeat-compose.yml \ - -f extensions/heartbeat/heartbeat-compose.yml \ - build + - name: Pre-build container images + run: >- + docker compose + -f docker-compose.yml + -f extensions/logspout/logspout-compose.yml + -f extensions/fleet/fleet-compose.yml + -f extensions/metricbeat/metricbeat-compose.yml + -f extensions/filebeat/filebeat-compose.yml + -f extensions/heartbeat/heartbeat-compose.yml + -f extensions/enterprise-search/enterprise-search-compose.yml + build ######################################################## # # @@ -113,33 +110,6 @@ jobs: # next steps don't need Logstash docker compose stop logstash - # - # Enterprise Search - # - - - name: Execute Enterprise Search test suite - run: | - - # Set mandatory Elasticsearch settings - - sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml - - # Restart Elasticsearch for changes to take effect - - docker compose restart elasticsearch - - # Run Enterprise Search and execute tests - - sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml - - docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search - .github/workflows/scripts/run-tests-enterprise-search.sh - - # Revert changes to Elasticsearch configuration - - sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml - docker compose restart elasticsearch - - name: 'debug: Display state and logs (Enterprise Search)' if: always() run: | @@ -147,19 +117,19 @@ jobs: docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search # - # APM Server + # Fleet # - - name: Execute APM Server test suite + - name: Execute Fleet test suite run: | - docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up --remove-orphans -d apm-server - .github/workflows/scripts/run-tests-apm-server.sh + docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml up --remove-orphans -d fleet-server + .github/workflows/scripts/run-tests-fleet.sh - - name: 'debug: Display state and logs (APM Server)' + - name: 'debug: Display state and logs (Fleet)' if: always() run: | - docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml ps - docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml logs apm-server + docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml ps + docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml logs fleet-server # # Metricbeat @@ -206,6 +176,33 @@ jobs: docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml ps docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml logs heartbeat + # + # Enterprise Search + # + + - name: Execute Enterprise Search test suite + run: | + + # Set mandatory Elasticsearch settings + + sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml + + # Restart Elasticsearch for changes to take effect + + docker compose restart elasticsearch + + # Run Enterprise Search and execute tests + + sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml + + docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search + .github/workflows/scripts/run-tests-enterprise-search.sh + + # Revert changes to Elasticsearch configuration + + sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml + docker compose restart elasticsearch + ############## # # # Tear down. # @@ -218,9 +215,9 @@ jobs: docker compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml - -f extensions/enterprise-search/enterprise-search-compose.yml - -f extensions/apm-server/apm-server-compose.yml + -f extensions/fleet/fleet-compose.yml -f extensions/metricbeat/metricbeat-compose.yml -f extensions/filebeat/filebeat-compose.yml -f extensions/heartbeat/heartbeat-compose.yml + -f extensions/enterprise-search/enterprise-search-compose.yml down -v diff --git a/.github/workflows/scripts/run-tests-apm-server.sh b/.github/workflows/scripts/run-tests-apm-server.sh deleted file mode 100755 index f3d135d..0000000 --- a/.github/workflows/scripts/run-tests-apm-server.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - - -source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" - - -cid="$(container_id apm-server)" -ip="$(service_ip apm-server)" - -log 'Waiting for readiness of APM Server' -poll_ready "$cid" "http://${ip}:8200/" diff --git a/.github/workflows/scripts/run-tests-fleet.sh b/.github/workflows/scripts/run-tests-fleet.sh new file mode 100755 index 0000000..44928f2 --- /dev/null +++ b/.github/workflows/scripts/run-tests-fleet.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + + +source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" + + +cid_es="$(container_id elasticsearch)" +ip_es="$(service_ip elasticsearch)" + +log 'Waiting for readiness of Elasticsearch' +poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd' + +# Fleet-managed Elastic Agent does not expose a liveness endpoint. +# Wait for the existence of the metrics index instead. +log 'Waiting for creation of metrics index in Elasticsearch' +poll_ready "$cid_es" "http://${ip_es}:9200/metrics-system.cpu-default" -u 'elastic:testpasswd' + +# We expect to find metrics entries using the following query: +# +# agent.name:"fleet-server" +# AND agent.type:"metricbeat" +# AND event.module:"system" +# AND event.dataset:"system.cpu" +# AND metricset.name:"cpu" +# +log 'Searching a document generated by Fleet Server' + +declare response +declare -i count + +declare -i was_retried=0 + +# retry for max 60s (30*2s) +for _ in $(seq 1 30); do + response="$(curl "http://${ip_es}:9200/metrics-system.cpu-default/_search?q=agent.name:%22fleet-server%22%20AND%20agent.type:%22metricbeat%22%20AND%20event.module:%22system%22%20AND%20event.dataset:%22system.cpu%22%20AND%20metricset.name:%22cpu%22&pretty" -s -u elastic:testpasswd)" + + set +u # prevent "unbound variable" if assigned value is not an integer + count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" + set -u + + if (( count > 0 )); then + break + fi + + was_retried=1 + echo -n 'x' >&2 + sleep 2 +done +if ((was_retried)); then + # flush stderr, important in non-interactive environments (CI) + echo >&2 +fi + +echo "$response" +# Elastic Agent buffers metrics until Elasticsearch becomes ready, so we +# tolerate multiple results +if (( count == 0 )); then + echo 'Expected at least 1 document' + exit 1 +fi diff --git a/docker-compose.yml b/docker-compose.yml index 6cc590b..e3174ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,8 @@ services: - "5601:5601" environment: KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} + # Fleet plugin + KIBANA_FLEET_SETUP: '1' networks: - elk depends_on: diff --git a/extensions/apm-server/Dockerfile b/extensions/apm-server/Dockerfile deleted file mode 100644 index 9f03f56..0000000 --- a/extensions/apm-server/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -ARG ELASTIC_VERSION - -FROM docker.elastic.co/apm/apm-server:${ELASTIC_VERSION} diff --git a/extensions/apm-server/README.md b/extensions/apm-server/README.md deleted file mode 100644 index ef4f34e..0000000 --- a/extensions/apm-server/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# APM Server extension - -The APM Server receives data from APM agents and transforms them into Elasticsearch documents that can be visualised in -Kibana. - -## Usage - -To include APM Server in the stack, run Docker Compose from the root of the repository with an additional command line -argument referencing the `apm-server-compose.yml` file: - -```console -$ docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up -``` - -Meanwhile, you can navigate to the **APM** application in Kibana and follow the setup instructions to get started. - -## Connecting an agent to APM Server - -The most basic configuration to send traces to APM server is to specify the `SERVICE_NAME` and `SERVICE_URL`. Here is an -example Python Flask configuration: - -```python -import elasticapm -from elasticapm.contrib.flask import ElasticAPM - -from flask import Flask - -app = Flask(__name__) -app.config['ELASTIC_APM'] = { - # Set required service name. Allowed characters: - # a-z, A-Z, 0-9, -, _, and space - 'SERVICE_NAME': 'PYTHON_FLASK_TEST_APP', - - # Set custom APM Server URL (default: http://localhost:8200) - 'SERVER_URL': 'http://apm-server:8200', - - 'DEBUG': True, -} -``` - -Configuration settings for each supported language are available in the APM documentation: [APM Agents][apm-agents]. - -## Checking connectivity and importing default APM dashboards - -1. On the Kibana home page, click `Add APM` under the _Observability_ panel. -1. Click `Check APM Server status` to confirm the server is up and running. -1. Click `Check agent status` to verify your agent has registered properly. -1. Click `Load Kibana objects` to create an index pattern for APM. -1. Click `Launch APM` to be taken to the APM dashboard. - -## See also - -[Running APM Server on Docker][apm-docker] - -[apm-agents]: https://www.elastic.co/guide/en/apm/guide/current/components.html -[apm-docker]: https://www.elastic.co/guide/en/apm/guide/current/running-on-docker.html diff --git a/extensions/apm-server/apm-server-compose.yml b/extensions/apm-server/apm-server-compose.yml deleted file mode 100644 index 9d5d2b1..0000000 --- a/extensions/apm-server/apm-server-compose.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '3.7' - -services: - apm-server: - build: - context: extensions/apm-server/ - args: - ELASTIC_VERSION: ${ELASTIC_VERSION} - command: - # Disable strict permission checking on 'apm-server.yml' configuration file - # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html - - --strict.perms=false - volumes: - - ./extensions/apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro,Z - ports: - - '8200:8200' - environment: - ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} - networks: - - elk - depends_on: - - elasticsearch diff --git a/extensions/apm-server/config/apm-server.yml b/extensions/apm-server/config/apm-server.yml deleted file mode 100644 index 71e2ea9..0000000 --- a/extensions/apm-server/config/apm-server.yml +++ /dev/null @@ -1,8 +0,0 @@ -apm-server: - host: 0.0.0.0:8200 - -output: - elasticsearch: - hosts: ['http://elasticsearch:9200'] - username: elastic - password: ${ELASTIC_PASSWORD} diff --git a/extensions/apm-server/.dockerignore b/extensions/fleet/.dockerignore similarity index 100% rename from extensions/apm-server/.dockerignore rename to extensions/fleet/.dockerignore diff --git a/extensions/fleet/Dockerfile b/extensions/fleet/Dockerfile new file mode 100644 index 0000000..43fa698 --- /dev/null +++ b/extensions/fleet/Dockerfile @@ -0,0 +1,3 @@ +ARG ELASTIC_VERSION + +FROM docker.elastic.co/beats/elastic-agent:${ELASTIC_VERSION} diff --git a/extensions/fleet/README.md b/extensions/fleet/README.md new file mode 100644 index 0000000..fa3f42d --- /dev/null +++ b/extensions/fleet/README.md @@ -0,0 +1,63 @@ +# Fleet Server + +> **Warning** +> This extension currently exists for preview purposes and should be considered **EXPERIMENTAL**. Expect regular changes +> to the default Fleet settings, both in the Elastic Agent and Kibana. +> +> See [Known Issues](#known-issues) for a list of issues that need to be addressed before this extension can be +> considered functional. + +Fleet provides central management capabilities for [Elastic Agents][fleet-doc] via an API and web UI served by Kibana, +with Elasticsearch acting as the communication layer. +Fleet Server is the central component which allows connecting Elastic Agents to the Fleet. + +## Usage + +> **Note** +> Elastic Agent does not retry failed connections to Kibana upon the initial enrollment phase. Therefore, Kibana must be +> fully started before this extension can be run. + +To include Fleet Server in the stack, run Docker Compose from the root of the repository with an additional command line +argument referencing the `fleet-compose.yml` file: + +```console +$ docker-compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml up +``` + +## Configuring Fleet Server + +Fleet Server — like any Elastic Agent — is configured via [Agent Policies][fleet-pol] which can be either managed +through the Fleet management UI in Kibana, or statically pre-configured inside the Kibana configuration file. + +To ease the enrollment of Fleet Server in this extension, docker-elk comes with a pre-configured Agent Policy for Fleet +Server defined inside [`kibana/config/kibana.yml`][config-kbn]. + +Please refer to the following documentation page for more details about configuring Fleet Server through the Fleet +management UI: [Fleet UI Settings][fleet-cfg]. + +## Known Issues + +- Every re-creation of the `fleet-server` container creates a duplicate agent in Fleet's central management. +- Logs and metrics are only collected within the Fleet Server's container. Ultimately, we want to emulate the behaviour + of the existing Metricsbeat and Filebeat extensions, and collect logs and metrics from all ELK containers + out-of-the-box. Unfortunately, this kind of use-case isn't (yet) well supported by Fleet, and most advanced + configurations currently require running Elastic Agents in [standalone mode][fleet-standalone]. + (Relevant resource: [Migrate from Beats to Elastic Agent][fleet-beats]) +- The Elastic Agent auto-enrolls using the `elastic` super-user. With this approach, you do not need to generate a + service token — either using the Fleet management UI or [CLI utility][es-svc-token] — prior to starting this + extension. However convenient that is, this approach _does not follow security best practices_, and we recommend + generating a service token for Fleet Server instead. + +## See also + +[Fleet and Elastic Agent Guide][fleet-doc] + +[fleet-doc]: https://www.elastic.co/guide/en/fleet/current/fleet-overview.html +[fleet-pol]: https://www.elastic.co/guide/en/fleet/current/agent-policy.html +[fleet-cfg]: https://www.elastic.co/guide/en/fleet/current/fleet-settings.html + +[config-kbn]: ../../kibana/config/kibana.yml + +[fleet-standalone]: https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html +[fleet-beats]: https://www.elastic.co/guide/en/fleet/current/migrate-beats-to-agent.html +[es-svc-token]: https://www.elastic.co/guide/en/elasticsearch/reference/current/service-tokens-command.html diff --git a/extensions/fleet/fleet-compose.yml b/extensions/fleet/fleet-compose.yml new file mode 100644 index 0000000..bb4080e --- /dev/null +++ b/extensions/fleet/fleet-compose.yml @@ -0,0 +1,26 @@ +version: '3.7' + +services: + fleet-server: + build: + context: extensions/fleet/ + args: + ELASTIC_VERSION: ${ELASTIC_VERSION} + environment: + FLEET_SERVER_ENABLE: '1' + FLEET_SERVER_INSECURE_HTTP: 'true' + FLEET_SERVER_POLICY_ID: fleet-server-policy + # Enrollment. + # (a) Auto-enroll using basic authentication + KIBANA_FLEET_USERNAME: elastic + KIBANA_FLEET_PASSWORD: ${ELASTIC_PASSWORD:-} + # (b) Enroll using a pre-generated service token + #FLEET_SERVER_SERVICE_TOKEN: + ports: + - '8220:8220' + hostname: fleet-server + networks: + - elk + depends_on: + - elasticsearch + - kibana diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 9eb2099..63378d9 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -4,10 +4,42 @@ # server.name: kibana server.host: 0.0.0.0 -elasticsearch.hosts: [ "http://elasticsearch:9200" ] +elasticsearch.hosts: [ 'http://elasticsearch:9200' ] monitoring.ui.container.elasticsearch.enabled: true ## X-Pack security credentials # elasticsearch.username: kibana_system elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD} + +## Fleet +## https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html +# +xpack.fleet.agents.fleet_server.hosts: [ 'http://fleet:8220' ] +xpack.fleet.agents.elasticsearch.hosts: [ 'http://elasticsearch:9200' ] + +xpack.fleet.packages: + - name: fleet_server + version: latest + - name: system + version: latest + - name: elastic_agent + version: latest + +xpack.fleet.agentPolicies: + - name: Fleet Server policy + id: fleet-server-policy + description: Fleet Server policy + monitoring_enabled: + - logs + - metrics + package_policies: + - name: fleet_server-1 + package: + name: fleet_server + - name: system-1 + package: + name: system + - name: elastic_agent-1 + package: + name: elastic_agent