feat(ext/beats): Use unprivileged publish users

.env

@@ -21,14 +21,22 @@ LOGSTASH_INTERNAL_PASSWORD='changeme'
 # https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
 KIBANA_SYSTEM_PASSWORD='changeme'
 
-# User 'beats_system' (built-in)
+# Users 'metricbeat_internal', 'filebeat_internal' and 'heartbeat_internal' (custom)
 #
-# The user the Beats use when storing monitoring information in Elasticsearch.
+# The users Beats use to connect and send data to Elasticsearch.
-# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+# https://www.elastic.co/guide/en/beats/metricbeat/current/feature-roles.html
-BEATS_SYSTEM_PASSWORD=''
+METRICBEAT_INTERNAL_PASSWORD=''
+FILEBEAT_INTERNAL_PASSWORD=''
+HEARTBEAT_INTERNAL_PASSWORD=''
 
 # User 'monitoring_internal' (custom)
 #
 # The user Metricbeat uses to collect monitoring data from stack components.
 # https://www.elastic.co/guide/en/elasticsearch/reference/current/how-monitoring-works.html
 MONITORING_INTERNAL_PASSWORD=''
+
+# User 'beats_system' (built-in)
+#
+# The user the Beats use when storing monitoring information in Elasticsearch.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+BEATS_SYSTEM_PASSWORD=''

.github/workflows/ci.yml

@@ -48,8 +48,11 @@ jobs:
           sed -i -e 's/\(ELASTIC_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
                  -e 's/\(LOGSTASH_INTERNAL_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
                  -e 's/\(KIBANA_SYSTEM_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
-                 -e 's/\(BEATS_SYSTEM_PASSWORD=\)'\'\''/\1testpasswd/g' \
+                 -e 's/\(METRICBEAT_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g' \
-                 -e 's/\(MONITORING_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g' .env
+                 -e 's/\(FILEBEAT_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g' \
+                 -e 's/\(HEARTBEAT_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g' \
+                 -e 's/\(MONITORING_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g' \
+                 -e 's/\(BEATS_SYSTEM_PASSWORD=\)'\'\''/\1testpasswd/g' .env
 
       ##########################################################
       #                                                        #

docker-compose.yml

@@ -24,8 +24,11 @@ services:
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
       LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
       KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
-      BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
+      METRICBEAT_INTERNAL_PASSWORD: ${METRICBEAT_INTERNAL_PASSWORD:-}
+      FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}
+      HEARTBEAT_INTERNAL_PASSWORD: ${HEARTBEAT_INTERNAL_PASSWORD:-}
       MONITORING_INTERNAL_PASSWORD: ${MONITORING_INTERNAL_PASSWORD:-}
+      BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
     networks:
       - elk
     depends_on:

extensions/filebeat/README.md

@@ -6,9 +6,9 @@ Elasticsearch or Logstash for indexing.
 
 ## Usage
 
-**This extension requires the `beats_system` user to be created and initialized with a password.** In case you haven't
+**This extension requires the `filebeat_internal` and `beats_system` users to be created and initialized with a
-done that during the initial startup of the stack, please refer to [How to re-execute the setup][setup] to run the setup
+password.** In case you haven't done that during the initial startup of the stack, please refer to [How to re-execute
-container again and initialize this user.
+the setup][setup] to run the setup container again and initialize these users.
 
 To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line
 argument referencing the `filebeat-compose.yml` file:

extensions/filebeat/config/filebeat.yml

@@ -27,8 +27,8 @@ monitoring:
 
 output.elasticsearch:
   hosts: [ http://elasticsearch:9200 ]
-  username: elastic
+  username: filebeat_internal
-  password: ${ELASTIC_PASSWORD}
+  password: ${FILEBEAT_INTERNAL_PASSWORD}
 
 ## HTTP endpoint for health checking
 ## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html

extensions/filebeat/filebeat-compose.yml

@@ -27,7 +27,7 @@ services:
         target: /var/run/docker.sock
         read_only: true
     environment:
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}
       BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
     networks:
       - elk

extensions/heartbeat/README.md

@@ -5,9 +5,9 @@ available.
 
 ## Usage
 
-**This extension requires the `beats_system` user to be created and initialized with a password.** In case you haven't
+**This extension requires the `heartbeat_internal` and `beats_system` users to be created and initialized with a
-done that during the initial startup of the stack, please refer to [How to re-execute the setup][setup] to run the setup
+password.** In case you haven't done that during the initial startup of the stack, please refer to [How to re-execute
-container again and initialize this user.
+the setup][setup] to run the setup container again and initialize these users.
 
 To include Heartbeat in the stack, run Docker Compose from the root of the repository with an additional command line
 argument referencing the `heartbeat-compose.yml` file:

extensions/heartbeat/config/heartbeat.yml

@@ -9,8 +9,8 @@ heartbeat.monitors:
   schedule: '@every 5s'
   urls:
     - http://elasticsearch:9200
-  username: elastic
+  username: heartbeat_internal
-  password: ${ELASTIC_PASSWORD}
+  password: ${HEARTBEAT_INTERNAL_PASSWORD}
 
 - type: icmp
   schedule: '@every 5s'
@@ -28,8 +28,8 @@ monitoring:
 
 output.elasticsearch:
   hosts: [ http://elasticsearch:9200 ]
-  username: elastic
+  username: heartbeat_internal
-  password: ${ELASTIC_PASSWORD}
+  password: ${HEARTBEAT_INTERNAL_PASSWORD}
 
 ## HTTP endpoint for health checking
 ## https://www.elastic.co/guide/en/beats/heartbeat/current/http-endpoint.html

extensions/heartbeat/heartbeat-compose.yml

@@ -16,7 +16,7 @@ services:
     volumes:
       - ./extensions/heartbeat/config/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro,Z
     environment:
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      HEARTBEAT_INTERNAL_PASSWORD: ${HEARTBEAT_INTERNAL_PASSWORD:-}
       BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
     networks:
       - elk

extensions/metricbeat/README.md

@@ -6,9 +6,9 @@ and ships them to the output that you specify, such as Elasticsearch or Logstash
 
 ## Usage
 
-**This extension requires the `beats_system` and `monitoring_internal` users to be created and initialized with a
+**This extension requires the `metricbeat_internal`, `monitoring_internal` and `beats_system` users to be created and
-password.** In case you haven't done that during the initial startup of the stack, please refer to [How to re-execute
+initialized with a password.** In case you haven't done that during the initial startup of the stack, please refer to
-the setup][setup] to run the setup container again and initialize these users.
+[How to re-execute the setup][setup] to run the setup container again and initialize these users.
 
 To include Metricbeat in the stack, run Docker Compose from the root of the repository with an additional command line
 argument referencing the `metricbeat-compose.yml` file:

extensions/metricbeat/config/metricbeat.yml

@@ -60,8 +60,8 @@ monitoring:
 
 output.elasticsearch:
   hosts: [ http://elasticsearch:9200 ]
-  username: elastic
+  username: metricbeat_internal
-  password: ${ELASTIC_PASSWORD}
+  password: ${METRICBEAT_INTERNAL_PASSWORD}
 
 ## HTTP endpoint for health checking
 ## https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html

extensions/metricbeat/metricbeat-compose.yml

@@ -38,9 +38,9 @@ services:
         target: /var/run/docker.sock
         read_only: true
     environment:
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      METRICBEAT_INTERNAL_PASSWORD: ${METRICBEAT_INTERNAL_PASSWORD:-}
-      BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
       MONITORING_INTERNAL_PASSWORD: ${MONITORING_INTERNAL_PASSWORD:-}
+      BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
     networks:
       - elk
     depends_on:

setup/entrypoint.sh

@@ -12,14 +12,20 @@ source "${BASH_SOURCE[0]%/*}"/helpers.sh
 declare -A users_passwords
 users_passwords=(
 	[logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}"
-	[monitoring_internal]="${MONITORING_INTERNAL_PASSWORD:-}"
 	[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
+	[metricbeat_internal]="${METRICBEAT_INTERNAL_PASSWORD:-}"
+	[filebeat_internal]="${FILEBEAT_INTERNAL_PASSWORD:-}"
+	[heartbeat_internal]="${HEARTBEAT_INTERNAL_PASSWORD:-}"
+	[monitoring_internal]="${MONITORING_INTERNAL_PASSWORD:-}"
 	[beats_system]="${BEATS_SYSTEM_PASSWORD=:-}"
 )
 
 declare -A users_roles
 users_roles=(
 	[logstash_internal]='logstash_writer'
+	[metricbeat_internal]='metricbeat_writer'
+	[filebeat_internal]='filebeat_writer'
+	[heartbeat_internal]='heartbeat_writer'
 	[monitoring_internal]='remote_monitoring_collector'
 )
 
@@ -29,6 +35,9 @@ users_roles=(
 declare -A roles_files
 roles_files=(
 	[logstash_writer]='logstash_writer.json'
+	[metricbeat_writer]='metricbeat_writer.json'
+	[filebeat_writer]='filebeat_writer.json'
+	[heartbeat_writer]='heartbeat_writer.json'
 )
 
 # --------------------------------------------------------

setup/roles/filebeat_writer.json

@@ -0,0 +1,19 @@
+{
+  "cluster": [
+    "manage_ilm",
+    "manage_index_templates",
+    "monitor",
+    "read_pipeline"
+  ],
+  "indices": [
+    {
+      "names": [
+        "filebeat-*"
+      ],
+      "privileges": [
+        "create_doc",
+        "manage"
+      ]
+    }
+  ]
+}

setup/roles/heartbeat_writer.json

@@ -0,0 +1,18 @@
+{
+  "cluster": [
+    "manage_ilm",
+    "manage_index_templates",
+    "monitor"
+  ],
+  "indices": [
+    {
+      "names": [
+        "heartbeat-*"
+      ],
+      "privileges": [
+        "create_doc",
+        "manage"
+      ]
+    }
+  ]
+}

setup/roles/metricbeat_writer.json

@@ -0,0 +1,19 @@
+{
+  "cluster": [
+    "manage_ilm",
+    "manage_index_templates",
+    "monitor"
+  ],
+  "indices": [
+    {
+      "names": [
+        ".monitoring-*-mb",
+        "metricbeat-*"
+      ],
+      "privileges": [
+        "create_doc",
+        "manage"
+      ]
+    }
+  ]
+}