feat: Add Fleet extension, remove standalone APM Server (#760)

The preferred way to run APM Server is via the APM integration of the
Elastic Agent.

Ref. https://www.elastic.co/guide/en/apm/guide/8.4/legacy-apm-overview.html

.github/workflows/ci.yml

@@ -23,20 +23,17 @@ jobs:
       #                                                   #
       #####################################################
 
-      - name: Prepare environment
+      - name: Pre-build container images
-        run: |
+        run: >-
-
+          docker compose
-          # Pre-build container images
+          -f docker-compose.yml
-
+          -f extensions/logspout/logspout-compose.yml
-          docker compose \
+          -f extensions/fleet/fleet-compose.yml
-            -f docker-compose.yml \
+          -f extensions/metricbeat/metricbeat-compose.yml
-            -f extensions/logspout/logspout-compose.yml \
+          -f extensions/filebeat/filebeat-compose.yml
-            -f extensions/enterprise-search/enterprise-search-compose.yml \
+          -f extensions/heartbeat/heartbeat-compose.yml
-            -f extensions/apm-server/apm-server-compose.yml \
+          -f extensions/enterprise-search/enterprise-search-compose.yml
-            -f extensions/metricbeat/metricbeat-compose.yml \
+          build
-            -f extensions/filebeat/filebeat-compose.yml \
-            -f extensions/heartbeat/heartbeat-compose.yml \
-            build
 
       ########################################################
       #                                                      #
@@ -113,33 +110,6 @@ jobs:
           # next steps don't need Logstash
           docker compose stop logstash
 
-      #
-      # Enterprise Search
-      #
-
-      - name: Execute Enterprise Search test suite
-        run: |
-
-          # Set mandatory Elasticsearch settings
-
-          sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml
-
-          # Restart Elasticsearch for changes to take effect
-
-          docker compose restart elasticsearch
-
-          # Run Enterprise Search and execute tests
-
-          sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
-
-          docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search
-          .github/workflows/scripts/run-tests-enterprise-search.sh
-
-          # Revert changes to Elasticsearch configuration
-
-          sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
-          docker compose restart elasticsearch
-
       - name: 'debug: Display state and logs (Enterprise Search)'
         if: always()
         run: |
@@ -147,19 +117,19 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml logs enterprise-search
 
       #
-      # APM Server
+      # Fleet
       #
 
-      - name: Execute APM Server test suite
+      - name: Execute Fleet test suite
         run: |
-          docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up --remove-orphans -d apm-server
+          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml up --remove-orphans -d fleet-server
-          .github/workflows/scripts/run-tests-apm-server.sh
+          .github/workflows/scripts/run-tests-fleet.sh
 
-      - name: 'debug: Display state and logs (APM Server)'
+      - name: 'debug: Display state and logs (Fleet)'
         if: always()
         run: |
-          docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml ps
+          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml ps
-          docker compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml logs apm-server
+          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml logs fleet-server
 
       #
       # Metricbeat
@@ -206,6 +176,33 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml ps
           docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml logs heartbeat
 
+      #
+      # Enterprise Search
+      #
+
+      - name: Execute Enterprise Search test suite
+        run: |
+
+          # Set mandatory Elasticsearch settings
+
+          sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml
+
+          # Restart Elasticsearch for changes to take effect
+
+          docker compose restart elasticsearch
+
+          # Run Enterprise Search and execute tests
+
+          sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
+
+          docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search
+          .github/workflows/scripts/run-tests-enterprise-search.sh
+
+          # Revert changes to Elasticsearch configuration
+
+          sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
+          docker compose restart elasticsearch
+
         ##############
         #            #
         # Tear down. #
@@ -218,9 +215,9 @@ jobs:
           docker compose
           -f docker-compose.yml
           -f extensions/logspout/logspout-compose.yml
-          -f extensions/enterprise-search/enterprise-search-compose.yml
+          -f extensions/fleet/fleet-compose.yml
-          -f extensions/apm-server/apm-server-compose.yml
           -f extensions/metricbeat/metricbeat-compose.yml
           -f extensions/filebeat/filebeat-compose.yml
           -f extensions/heartbeat/heartbeat-compose.yml
+          -f extensions/enterprise-search/enterprise-search-compose.yml
           down -v

.github/workflows/scripts/run-tests-apm-server.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-set -o pipefail
-
-
-source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
-
-
-cid="$(container_id apm-server)"
-ip="$(service_ip apm-server)"
-
-log 'Waiting for readiness of APM Server'
-poll_ready "$cid" "http://${ip}:8200/"

.github/workflows/scripts/run-tests-fleet.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+cid_es="$(container_id elasticsearch)"
+ip_es="$(service_ip elasticsearch)"
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+
+# Fleet-managed Elastic Agent does not expose a liveness endpoint.
+# Wait for the existence of the metrics index instead.
+log 'Waiting for creation of metrics index in Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/metrics-system.cpu-default" -u 'elastic:testpasswd'
+
+# We expect to find metrics entries using the following query:
+#
+#   agent.name:"fleet-server"
+#   AND agent.type:"metricbeat"
+#   AND event.module:"system"
+#   AND event.dataset:"system.cpu"
+#   AND metricset.name:"cpu"
+#
+log 'Searching a document generated by Fleet Server'
+
+declare response
+declare -i count
+
+declare -i was_retried=0
+
+# retry for max 60s (30*2s)
+for _ in $(seq 1 30); do
+	response="$(curl "http://${ip_es}:9200/metrics-system.cpu-default/_search?q=agent.name:%22fleet-server%22%20AND%20agent.type:%22metricbeat%22%20AND%20event.module:%22system%22%20AND%20event.dataset:%22system.cpu%22%20AND%20metricset.name:%22cpu%22&pretty" -s -u elastic:testpasswd)"
+
+	set +u  # prevent "unbound variable" if assigned value is not an integer
+	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
+	set -u
+
+	if (( count > 0 )); then
+		break
+	fi
+
+	was_retried=1
+	echo -n 'x' >&2
+	sleep 2
+done
+if ((was_retried)); then
+	# flush stderr, important in non-interactive environments (CI)
+	echo >&2
+fi
+
+echo "$response"
+# Elastic Agent buffers metrics until Elasticsearch becomes ready, so we
+# tolerate multiple results
+if (( count == 0 )); then
+	echo 'Expected at least 1 document'
+	exit 1
+fi

docker-compose.yml

@@ -81,6 +81,8 @@ services:
       - "5601:5601"
     environment:
       KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+      # Fleet plugin
+      KIBANA_FLEET_SETUP: '1'
     networks:
       - elk
     depends_on:

extensions/apm-server/Dockerfile

@@ -1,3 +0,0 @@
-ARG ELASTIC_VERSION
-
-FROM docker.elastic.co/apm/apm-server:${ELASTIC_VERSION}

extensions/apm-server/README.md

@@ -1,56 +0,0 @@
-# APM Server extension
-
-The APM Server receives data from APM agents and transforms them into Elasticsearch documents that can be visualised in
-Kibana.
-
-## Usage
-
-To include APM Server in the stack, run Docker Compose from the root of the repository with an additional command line
-argument referencing the `apm-server-compose.yml` file:
-
-```console
-$ docker-compose -f docker-compose.yml -f extensions/apm-server/apm-server-compose.yml up
-```
-
-Meanwhile, you can navigate to the **APM** application in Kibana and follow the setup instructions to get started.
-
-## Connecting an agent to APM Server
-
-The most basic configuration to send traces to APM server is to specify the `SERVICE_NAME` and `SERVICE_URL`. Here is an
-example Python Flask configuration:
-
-```python
-import elasticapm
-from elasticapm.contrib.flask import ElasticAPM
-
-from flask import Flask
-
-app = Flask(__name__)
-app.config['ELASTIC_APM'] = {
-    # Set required service name. Allowed characters:
-    # a-z, A-Z, 0-9, -, _, and space
-    'SERVICE_NAME': 'PYTHON_FLASK_TEST_APP',
-
-    # Set custom APM Server URL (default: http://localhost:8200)
-    'SERVER_URL': 'http://apm-server:8200',
-
-    'DEBUG': True,
-}
-```
-
-Configuration settings for each supported language are available in the APM documentation: [APM Agents][apm-agents].
-
-## Checking connectivity and importing default APM dashboards
-
-1. On the Kibana home page, click `Add APM` under the _Observability_ panel.
-1. Click `Check APM Server status` to confirm the server is up and running.
-1. Click `Check agent status` to verify your agent has registered properly.
-1. Click `Load Kibana objects` to create an index pattern for APM.
-1. Click `Launch APM` to be taken to the APM dashboard.
-
-## See also
-
-[Running APM Server on Docker][apm-docker]
-
-[apm-agents]: https://www.elastic.co/guide/en/apm/guide/current/components.html
-[apm-docker]: https://www.elastic.co/guide/en/apm/guide/current/running-on-docker.html

extensions/apm-server/apm-server-compose.yml

@@ -1,22 +0,0 @@
-version: '3.7'
-
-services:
-  apm-server:
-    build:
-      context: extensions/apm-server/
-      args:
-        ELASTIC_VERSION: ${ELASTIC_VERSION}
-    command:
-        # Disable strict permission checking on 'apm-server.yml' configuration file
-        # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
-      - --strict.perms=false
-    volumes:
-      - ./extensions/apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro,Z
-    ports:
-      - '8200:8200'
-    environment:
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
-    networks:
-      - elk
-    depends_on:
-      - elasticsearch

extensions/apm-server/config/apm-server.yml

@@ -1,8 +0,0 @@
-apm-server:
-  host: 0.0.0.0:8200
-
-output:
-  elasticsearch:
-    hosts: ['http://elasticsearch:9200']
-    username: elastic
-    password: ${ELASTIC_PASSWORD}

extensions/fleet/Dockerfile

@@ -0,0 +1,3 @@
+ARG ELASTIC_VERSION
+
+FROM docker.elastic.co/beats/elastic-agent:${ELASTIC_VERSION}

extensions/fleet/README.md

@@ -0,0 +1,63 @@
+# Fleet Server
+
+> **Warning**  
+> This extension currently exists for preview purposes and should be considered **EXPERIMENTAL**. Expect regular changes
+> to the default Fleet settings, both in the Elastic Agent and Kibana.
+>
+> See [Known Issues](#known-issues) for a list of issues that need to be addressed before this extension can be
+> considered functional.
+
+Fleet provides central management capabilities for [Elastic Agents][fleet-doc] via an API and web UI served by Kibana,
+with Elasticsearch acting as the communication layer.
+Fleet Server is the central component which allows connecting Elastic Agents to the Fleet.
+
+## Usage
+
+> **Note**  
+> Elastic Agent does not retry failed connections to Kibana upon the initial enrollment phase. Therefore, Kibana must be
+> fully started before this extension can be run.
+
+To include Fleet Server in the stack, run Docker Compose from the root of the repository with an additional command line
+argument referencing the `fleet-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml up
+```
+
+## Configuring Fleet Server
+
+Fleet Server — like any Elastic Agent — is configured via [Agent Policies][fleet-pol] which can be either managed
+through the Fleet management UI in Kibana, or statically pre-configured inside the Kibana configuration file.
+
+To ease the enrollment of Fleet Server in this extension, docker-elk comes with a pre-configured Agent Policy for Fleet
+Server defined inside [`kibana/config/kibana.yml`][config-kbn].
+
+Please refer to the following documentation page for more details about configuring Fleet Server through the Fleet
+management UI: [Fleet UI Settings][fleet-cfg].
+
+## Known Issues
+
+- Every re-creation of the `fleet-server` container creates a duplicate agent in Fleet's central management.
+- Logs and metrics are only collected within the Fleet Server's container. Ultimately, we want to emulate the behaviour
+  of the existing Metricsbeat and Filebeat extensions, and collect logs and metrics from all ELK containers
+  out-of-the-box. Unfortunately, this kind of use-case isn't (yet) well supported by Fleet, and most advanced
+  configurations currently require running Elastic Agents in [standalone mode][fleet-standalone].
+  (Relevant resource: [Migrate from Beats to Elastic Agent][fleet-beats])
+- The Elastic Agent auto-enrolls using the `elastic` super-user. With this approach, you do not need to generate a
+  service token — either using the Fleet management UI or [CLI utility][es-svc-token] — prior to starting this
+  extension. However convenient that is, this approach _does not follow security best practices_, and we recommend
+  generating a service token for Fleet Server instead.
+
+## See also
+
+[Fleet and Elastic Agent Guide][fleet-doc]
+
+[fleet-doc]: https://www.elastic.co/guide/en/fleet/current/fleet-overview.html
+[fleet-pol]: https://www.elastic.co/guide/en/fleet/current/agent-policy.html
+[fleet-cfg]: https://www.elastic.co/guide/en/fleet/current/fleet-settings.html
+
+[config-kbn]: ../../kibana/config/kibana.yml
+
+[fleet-standalone]: https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html
+[fleet-beats]: https://www.elastic.co/guide/en/fleet/current/migrate-beats-to-agent.html
+[es-svc-token]: https://www.elastic.co/guide/en/elasticsearch/reference/current/service-tokens-command.html

extensions/fleet/fleet-compose.yml

@@ -0,0 +1,26 @@
+version: '3.7'
+
+services:
+  fleet-server:
+    build:
+      context: extensions/fleet/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    environment:
+      FLEET_SERVER_ENABLE: '1'
+      FLEET_SERVER_INSECURE_HTTP: 'true'
+      FLEET_SERVER_POLICY_ID: fleet-server-policy
+      # Enrollment.
+      # (a) Auto-enroll using basic authentication
+      KIBANA_FLEET_USERNAME: elastic
+      KIBANA_FLEET_PASSWORD: ${ELASTIC_PASSWORD:-}
+      # (b) Enroll using a pre-generated service token
+      #FLEET_SERVER_SERVICE_TOKEN: <service_token>
+    ports:
+      - '8220:8220'
+    hostname: fleet-server
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+      - kibana

kibana/config/kibana.yml

@@ -4,10 +4,42 @@
 #
 server.name: kibana
 server.host: 0.0.0.0
-elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+elasticsearch.hosts: [ 'http://elasticsearch:9200' ]
 monitoring.ui.container.elasticsearch.enabled: true
 
 ## X-Pack security credentials
 #
 elasticsearch.username: kibana_system
 elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}
+
+## Fleet
+## https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html
+#
+xpack.fleet.agents.fleet_server.hosts: [ 'http://fleet:8220' ]
+xpack.fleet.agents.elasticsearch.hosts: [ 'http://elasticsearch:9200' ]
+
+xpack.fleet.packages:
+  - name: fleet_server
+    version: latest
+  - name: system
+    version: latest
+  - name: elastic_agent
+    version: latest
+
+xpack.fleet.agentPolicies:
+  - name: Fleet Server policy
+    id: fleet-server-policy
+    description: Fleet Server policy
+    monitoring_enabled:
+      - logs
+      - metrics
+    package_policies:
+      - name: fleet_server-1
+        package:
+          name: fleet_server
+      - name: system-1
+        package:
+          name: system
+      - name: elastic_agent-1
+        package:
+          name: elastic_agent