From a519ed63eac456164858fee43220216c499eb16c Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Thu, 28 Apr 2016 20:40:04 +1200 Subject: [PATCH 01/12] X-Pack (alpha) support. --- README.md | 14 ++++++++------ docker-compose.yml | 5 ++--- elasticsearch/Dockerfile | 12 ++++++++++++ elasticsearch/entrypoint.sh | 19 +++++++++++++++++++ kibana/Dockerfile | 6 +++--- kibana/config/kibana.yml | 7 +++++++ logstash/config/logstash.conf | 2 ++ 7 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 elasticsearch/Dockerfile create mode 100644 elasticsearch/entrypoint.sh diff --git a/README.md b/README.md index eb44394..800b54c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ [![Join the chat at https://gitter.im/deviantony/fig-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/fig-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +**WARNING: Experimental support of the X-Pack version of the Elastic stack.** + +It is *NOT* recommended to use this in production. + Run the latest version of the ELK (Elasticseach, Logstash, Kibana) stack with Docker and Docker-compose. It will give you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticseach and the visualization power of Kibana. @@ -50,18 +54,16 @@ Now that the stack is running, you'll want to inject logs in it. The shipped log $ nc localhost 5000 < /path/to/logfile.log ``` -And then access Kibana UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser. +And then access Kibana UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser and use the following credentials to login: + +* user: *elastic* +* password: *pass-elastic* *NOTE*: You'll need to inject data into logstash before being able to create a logstash index in Kibana. Then all you should have to do is to hit the create button. See: https://www.elastic.co/guide/en/kibana/current/setup.html#connect -You can also access: -* Sense: [http://localhost:5601/app/sense](http://localhost:5601/app/sense) - -*NOTE*: In order to use Sense, you'll need to query the IP address associated to your *network device* instead of localhost. - By default, the stack exposes the following ports: * 5000: Logstash TCP input. * 9200: Elasticsearch HTTP diff --git a/docker-compose.yml b/docker-compose.yml index 863a109..5352411 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,10 @@ elasticsearch: - image: elasticsearch:latest - command: elasticsearch -Des.network.host=0.0.0.0 + build: elasticsearch/ ports: - "9200:9200" - "9300:9300" logstash: - image: logstash:latest + image: logstash:5 command: logstash -f /etc/logstash/conf.d/logstash.conf volumes: - ./logstash/config:/etc/logstash/conf.d diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile new file mode 100644 index 0000000..12eb078 --- /dev/null +++ b/elasticsearch/Dockerfile @@ -0,0 +1,12 @@ +FROM elasticsearch:5 + +ENV ES_JAVA_OPTS="-Des.path.conf=/etc/elasticsearch" + +RUN elasticsearch-plugin install x-pack + +RUN apt-get update && apt-get install -y netcat + +COPY entrypoint.sh /tmp/entrypoint.sh +RUN chmod +x /tmp/entrypoint.sh + +CMD ["/tmp/entrypoint.sh"] diff --git a/elasticsearch/entrypoint.sh b/elasticsearch/entrypoint.sh new file mode 100644 index 0000000..94edd8e --- /dev/null +++ b/elasticsearch/entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +echo "Starting Elasticsearch" +gosu elasticsearch elasticsearch -E es.network.host=0.0.0.0 -E es.discovery.zen.minimum_master_nodes=1 & + +echo "Waiting for Elasticsearch to boot..." +while true; do + nc -q 1 localhost 9200 2>/dev/null && break +done + +echo "Elasticsearch ready. Creating x-pack users..." + +/usr/share/elasticsearch/bin/x-pack/users useradd elastic -r admin -p 'pass-elastic' +/usr/share/elasticsearch/bin/x-pack/users useradd kibana -r kibana4_server -p 'pass-kibana' +/usr/share/elasticsearch/bin/x-pack/users useradd logstash -r logstash -p 'pass-logstash' + +while true; do sleep 1000; done + +exit 0 diff --git a/kibana/Dockerfile b/kibana/Dockerfile index fe594e1..a6760b8 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,10 +1,10 @@ -FROM kibana:latest +FROM kibana:5 -RUN apt-get update && apt-get install -y netcat +RUN apt-get update && apt-get install -y netcat bzip2 COPY entrypoint.sh /tmp/entrypoint.sh RUN chmod +x /tmp/entrypoint.sh -RUN kibana plugin --install elastic/sense +RUN kibana-plugin install x-pack CMD ["/tmp/entrypoint.sh"] diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 820c776..816deac 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -74,3 +74,10 @@ bundled_plugin_ids: - plugins/table_vis/index - plugins/vis_types/index - plugins/visualize/index + + +xpack.security.encryptionKey: "my-secret-key" +elasticsearch.username: "kibana" +elasticsearch.password: "pass-kibana" +xpack.security.skipSslCheck: true +xpack.security.useUnsafeSessions: true diff --git a/logstash/config/logstash.conf b/logstash/config/logstash.conf index 5718b33..20e7081 100644 --- a/logstash/config/logstash.conf +++ b/logstash/config/logstash.conf @@ -9,5 +9,7 @@ input { output { elasticsearch { hosts => "elasticsearch:9200" + user => "logstash" + password => "pass-logstash" } } From 54cd6679120723548b7bda3ca82fa5dcc99a1a51 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Sun, 15 May 2016 16:55:20 +1200 Subject: [PATCH 02/12] X-Pack (alpha-2) support --- README.md | 16 ++++++++++++---- docker-compose.yml | 2 ++ elasticsearch/Dockerfile | 9 ++------- elasticsearch/entrypoint.sh | 19 +------------------ kibana/config/kibana.yml | 2 +- logstash/config/logstash.conf | 4 ++-- 6 files changed, 20 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 800b54c..c59cb45 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Join the chat at https://gitter.im/deviantony/fig-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/fig-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -**WARNING: Experimental support of the X-Pack version of the Elastic stack.** +**WARNING: Experimental support of the X-Pack (alpha-2) version of the Elastic stack.** It is *NOT* recommended to use this in production. @@ -29,10 +29,18 @@ Based on the official images: On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux into Permissive mode in order for docker-elk to start properly. For example on Redhat and CentOS, the following will apply the proper context: -````bash +```bash .-root@centos ~ --$ chcon -R system_u:object_r:admin_home_t:s0 fig-elk/ -```` +-$ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ +``` + +## Increase max_map_count on your host + +You need to increase `max_map_count` on your Docker host: + +```bash +$ sudo sysctl -w vm.max_map_count=262144 +``` # Usage diff --git a/docker-compose.yml b/docker-compose.yml index 5352411..3eea1c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ elasticsearch: ports: - "9200:9200" - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" logstash: image: logstash:5 command: logstash -f /etc/logstash/conf.d/logstash.conf diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 12eb078..a18ac90 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -2,11 +2,6 @@ FROM elasticsearch:5 ENV ES_JAVA_OPTS="-Des.path.conf=/etc/elasticsearch" -RUN elasticsearch-plugin install x-pack +RUN elasticsearch-plugin install --batch x-pack -RUN apt-get update && apt-get install -y netcat - -COPY entrypoint.sh /tmp/entrypoint.sh -RUN chmod +x /tmp/entrypoint.sh - -CMD ["/tmp/entrypoint.sh"] +CMD ["-E", "es.network.host=0.0.0.0", "-E", "es.discovery.zen.minimum_master_nodes=1"] diff --git a/elasticsearch/entrypoint.sh b/elasticsearch/entrypoint.sh index 94edd8e..b936b66 100644 --- a/elasticsearch/entrypoint.sh +++ b/elasticsearch/entrypoint.sh @@ -1,19 +1,2 @@ -#!/usr/bin/env bash - echo "Starting Elasticsearch" -gosu elasticsearch elasticsearch -E es.network.host=0.0.0.0 -E es.discovery.zen.minimum_master_nodes=1 & - -echo "Waiting for Elasticsearch to boot..." -while true; do - nc -q 1 localhost 9200 2>/dev/null && break -done - -echo "Elasticsearch ready. Creating x-pack users..." - -/usr/share/elasticsearch/bin/x-pack/users useradd elastic -r admin -p 'pass-elastic' -/usr/share/elasticsearch/bin/x-pack/users useradd kibana -r kibana4_server -p 'pass-kibana' -/usr/share/elasticsearch/bin/x-pack/users useradd logstash -r logstash -p 'pass-logstash' - -while true; do sleep 1000; done - -exit 0 +exec gosu elasticsearch elasticsearch -E es.network.host=0.0.0.0 -E es.discovery.zen.minimum_master_nodes=1 diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 816deac..ec72eec 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -78,6 +78,6 @@ bundled_plugin_ids: xpack.security.encryptionKey: "my-secret-key" elasticsearch.username: "kibana" -elasticsearch.password: "pass-kibana" +elasticsearch.password: "changeme" xpack.security.skipSslCheck: true xpack.security.useUnsafeSessions: true diff --git a/logstash/config/logstash.conf b/logstash/config/logstash.conf index 20e7081..2cc24b4 100644 --- a/logstash/config/logstash.conf +++ b/logstash/config/logstash.conf @@ -9,7 +9,7 @@ input { output { elasticsearch { hosts => "elasticsearch:9200" - user => "logstash" - password => "pass-logstash" + user => "elastic" + password => "changeme" } } From ae55dc0ab2882a804fe3d5dc1283395f902c314f Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Sun, 15 May 2016 16:57:59 +1200 Subject: [PATCH 03/12] Remove useless entrypoint for Elasticsearch container --- elasticsearch/entrypoint.sh | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 elasticsearch/entrypoint.sh diff --git a/elasticsearch/entrypoint.sh b/elasticsearch/entrypoint.sh deleted file mode 100644 index b936b66..0000000 --- a/elasticsearch/entrypoint.sh +++ /dev/null @@ -1,2 +0,0 @@ -echo "Starting Elasticsearch" -exec gosu elasticsearch elasticsearch -E es.network.host=0.0.0.0 -E es.discovery.zen.minimum_master_nodes=1 From fc9c2783a7a20e6f08ad5edd5595c738d71c2e6f Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Sun, 15 May 2016 17:00:37 +1200 Subject: [PATCH 04/12] update credentials for Kibana --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c59cb45..5a994b2 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ $ nc localhost 5000 < /path/to/logfile.log And then access Kibana UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser and use the following credentials to login: * user: *elastic* -* password: *pass-elastic* +* password: *changeme* *NOTE*: You'll need to inject data into logstash before being able to create a logstash index in Kibana. Then all you should have to do is to hit the create button. From a7b51a2d0fcfb44687f61c6d67ef79eea1870e22 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Wed, 15 Jun 2016 10:11:46 +1200 Subject: [PATCH 05/12] X-Pack (alpha-3) support --- README.md | 2 +- docker-compose.yml | 2 +- elasticsearch/Dockerfile | 2 +- kibana/config/kibana.yml | 137 +++++++++++++++++++++------------------ 4 files changed, 76 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 5a994b2..a841c5f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Join the chat at https://gitter.im/deviantony/fig-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/fig-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -**WARNING: Experimental support of the X-Pack (alpha-2) version of the Elastic stack.** +**WARNING: Experimental support of the X-Pack (alpha-3) version of the Elastic stack.** It is *NOT* recommended to use this in production. diff --git a/docker-compose.yml b/docker-compose.yml index 3eea1c1..9d795c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ elasticsearch: ES_JAVA_OPTS: "-Xms1g -Xmx1g" logstash: image: logstash:5 - command: logstash -f /etc/logstash/conf.d/logstash.conf + command: /usr/share/logstash/bin/logstash --path.settings /etc/logstash/ volumes: - ./logstash/config:/etc/logstash/conf.d ports: diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index a18ac90..3cc27b2 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -4,4 +4,4 @@ ENV ES_JAVA_OPTS="-Des.path.conf=/etc/elasticsearch" RUN elasticsearch-plugin install --batch x-pack -CMD ["-E", "es.network.host=0.0.0.0", "-E", "es.discovery.zen.minimum_master_nodes=1"] +CMD ["-E", "network.host=0.0.0.0", "-E", "discovery.zen.minimum_master_nodes=1"] diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index ec72eec..97bfeb6 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -1,83 +1,92 @@ -# Kibana is served by a back end server. This controls which port to use. -port: 5601 +# Kibana is served by a back end server. This setting specifies the port to use. +server.port: 5601 -# The host to bind the server to. -host: "0.0.0.0" +# This setting specifies the IP address of the back end server. +server.host: "0.0.0.0" -# The Elasticsearch instance to use for all your queries. -elasticsearch_url: "http://elasticsearch:9200" +# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This setting +# cannot end in a slash. +# server.basePath: "" -# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, -# then the host you use to connect to *this* Kibana instance will be sent. -elasticsearch_preserve_host: true +# The maximum payload size in bytes for incoming server requests. +# server.maxPayloadBytes: 1048576 -# Kibana uses an index in Elasticsearch to store saved searches, visualizations -# and dashboards. It will create a new index if it doesn't already exist. -kibana_index: ".kibana" +# The Kibana server's name. This is used for display purposes. +# server.name: "your-hostname" -# If your Elasticsearch is protected with basic auth, this is the user credentials -# used by the Kibana server to perform maintence on the kibana_index at statup. Your Kibana -# users will still need to authenticate with Elasticsearch (which is proxied thorugh -# the Kibana server) -# kibana_elasticsearch_username: user -# kibana_elasticsearch_password: pass +# The URL of the Elasticsearch instance to use for all your queries. +elasticsearch.url: "http://elasticsearch:9200" -# If your Elasticsearch requires client certificate and key -# kibana_elasticsearch_client_crt: /path/to/your/client.crt -# kibana_elasticsearch_client_key: /path/to/your/client.key +# When this setting’s value is true Kibana uses the hostname specified in the server.host +# setting. When the value of this setting is false, Kibana uses the hostname of the host +# that connects to this Kibana instance. +# elasticsearch.preserveHost: true -# If you need to provide a CA certificate for your Elasticsarech instance, put -# the path of the pem file here. -# ca: /path/to/your/CA.pem +# Kibana uses an index in Elasticsearch to store saved searches, visualizations and +# dashboards. Kibana creates a new index if the index doesn’t already exist. +# kibana.index: ".kibana" # The default application to load. -default_app_id: "discover" +# kibana.defaultAppId: "discover" -# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to -# request_timeout setting -# ping_timeout: 1500 +# If your Elasticsearch is protected with basic authentication, these settings provide +# the username and password that the Kibana server uses to perform maintenance on the Kibana +# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which +# is proxied through the Kibana server. +# elasticsearch.username: "user" +# elasticsearch.password: "pass" -# Time in milliseconds to wait for responses from the back end or elasticsearch. -# This must be > 0 -request_timeout: 300000 +# Paths to the PEM-format SSL certificate and SSL key files, respectively. These +# files enable SSL for outgoing requests from the Kibana server to the browser. +# server.ssl.cert: /path/to/your/server.crt +# server.ssl.key: /path/to/your/server.key -# Time in milliseconds for Elasticsearch to wait for responses from shards. -# Set to 0 to disable. -shard_timeout: 0 +# Optional settings that provide the paths to the PEM-format SSL certificate and key files. +# These files validate that your Elasticsearch backend uses the same key files. +# elasticsearch.ssl.cert: /path/to/your/client.crt +# elasticsearch.ssl.key: /path/to/your/client.key -# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying -# startup_timeout: 5000 +# Optional setting that enables you to specify a path to the PEM file for the certificate +# authority for your Elasticsearch instance. +# elasticsearch.ssl.ca: /path/to/your/CA.pem -# Set to false to have a complete disregard for the validity of the SSL -# certificate. -verify_ssl: true +# To disregard the validity of SSL certificates, change this setting’s value to false. +# elasticsearch.ssl.verify: true -# SSL for outgoing requests from the Kibana Server (PEM formatted) -# ssl_key_file: /path/to/your/server.key -# ssl_cert_file: /path/to/your/server.crt +# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of +# the elasticsearch.requestTimeout setting. +# elasticsearch.pingTimeout: 1500 -# Set the path to where you would like the process id file to be created. -# pid_file: /var/run/kibana.pid +# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value +# must be a positive integer. +# elasticsearch.requestTimeout: 30000 -# If you would like to send the log output to a file you can set the path below. -# This will also turn off the STDOUT log output. -# log_file: ./kibana.log -# Plugins that are included in the build, and no longer found in the plugins/ folder -bundled_plugin_ids: - - plugins/dashboard/index - - plugins/discover/index - - plugins/doc/index - - plugins/kibana/index - - plugins/markdown_vis/index - - plugins/metric_vis/index - - plugins/settings/index - - plugins/table_vis/index - - plugins/vis_types/index - - plugins/visualize/index +# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side +# headers, set this value to [] (an empty list). +# elasticsearch.requestHeadersWhitelist: [ authorization ] +# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. +# elasticsearch.shardTimeout: 0 -xpack.security.encryptionKey: "my-secret-key" -elasticsearch.username: "kibana" -elasticsearch.password: "changeme" -xpack.security.skipSslCheck: true -xpack.security.useUnsafeSessions: true +# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. +# elasticsearch.startupTimeout: 5000 + +# Specifies the path where Kibana creates the process ID file. +# pid.file: /var/run/kibana.pid + +# Enables you specify a file where Kibana stores log output. +# logging.dest: stdout + +# Set the value of this setting to true to suppress all logging output. +# logging.silent: false + +# Set the value of this setting to true to suppress all logging output other than error messages. +# logging.quiet: false + +# Set the value of this setting to true to log all events, including system usage information +# and all requests. +# logging.verbose: false + +# Set the interval in milliseconds to sample system and process performance +# metrics. Minimum is 100ms. Defaults to 10000. +# ops.interval: 10000 From 3cf96eb92578e3c8f3e7165deb0b74d76068c932 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Thu, 16 Jun 2016 10:18:04 +1200 Subject: [PATCH 06/12] update logstash command --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9d795c4..06e0f7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ elasticsearch: ES_JAVA_OPTS: "-Xms1g -Xmx1g" logstash: image: logstash:5 - command: /usr/share/logstash/bin/logstash --path.settings /etc/logstash/ + command: --path.settings /etc/logstash/ volumes: - ./logstash/config:/etc/logstash/conf.d ports: From b14907d33a8f059d94fd5e64a4ca889f0e25630d Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Wed, 6 Jul 2016 10:40:23 +1200 Subject: [PATCH 07/12] X-Pack (alpha-4) support --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 06e0f7b..15d70a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ elasticsearch: ES_JAVA_OPTS: "-Xms1g -Xmx1g" logstash: image: logstash:5 - command: --path.settings /etc/logstash/ + command: -f /etc/logstash/conf.d/ volumes: - ./logstash/config:/etc/logstash/conf.d ports: From 890c3959e6f78363e987309ab8c31581b73b8cd6 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Thu, 7 Jul 2016 10:08:50 +1200 Subject: [PATCH 08/12] allow logstash plugin management --- README.md | 30 +++++++++++++++++++++--------- docker-compose.yml | 2 +- logstash/Dockerfile | 4 ++++ logstash/config/logstash.conf | 2 +- 4 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 logstash/Dockerfile diff --git a/README.md b/README.md index a841c5f..4fd4fa9 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,7 @@ And then access Kibana UI by hitting [http://localhost:5601](http://localhost:56 * user: *elastic* * password: *changeme* -*NOTE*: You'll need to inject data into logstash before being able to create a logstash index in Kibana. Then all you should have to do is to -hit the create button. +*NOTE*: You'll need to inject data into logstash before being able to create a logstash index in Kibana. Then all you should have to do is to hit the create button. See: https://www.elastic.co/guide/en/kibana/current/setup.html#connect @@ -105,8 +104,8 @@ If you want to override the default configuration, add the *LS_HEAP_SIZE* enviro ```yml logstash: - image: logstash:latest - command: logstash -f /etc/logstash/conf.d/logstash.conf + build: logstash/ + command: -f /etc/logstash/conf.d/ volumes: - ./logstash/config:/etc/logstash/conf.d ports: @@ -117,6 +116,13 @@ logstash: - LS_HEAP_SIZE=2048m ``` +## How can I add Logstash plugins? ## + +To add plugins to logstash you have to: + +1. Add a RUN statement to the `logstash/Dockerfile` (ex. `RUN logstash-plugin install logstash-filter-json`) +2. Add the associated plugin code configuration to the `logstash/config/logstash.conf` file + ## How can I enable a remote JMX connection to Logstash? As for the Java heap memory, another environment variable allows to specify JAVA_OPTS used by Logstash. You'll need to specify the appropriate options to enable JMX and map the JMX port on the docker host. @@ -125,13 +131,12 @@ Update the container in the `docker-compose.yml` to add the *LS_JAVA_OPTS* envir ```yml logstash: - image: logstash:latest - command: logstash -f /etc/logstash/conf.d/logstash.conf + build: logstash/ + command: -f /etc/logstash/conf.d/ volumes: - ./logstash/config:/etc/logstash/conf.d ports: - "5000:5000" - - "18080:18080" links: - elasticsearch environment: @@ -149,9 +154,11 @@ Then, you'll need to map your configuration file inside the container in the `do ```yml elasticsearch: build: elasticsearch/ - command: elasticsearch -Des.network.host=_non_loopback_ ports: - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" volumes: - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml ``` @@ -164,6 +171,9 @@ elasticsearch: command: elasticsearch -Des.network.host=_non_loopback_ -Des.cluster.name: my-cluster ports: - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" ``` # Storage @@ -177,9 +187,11 @@ In order to persist Elasticsearch data even after removing the Elasticsearch con ```yml elasticsearch: build: elasticsearch/ - command: elasticsearch -Des.network.host=_non_loopback_ ports: - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" volumes: - /path/to/storage:/usr/share/elasticsearch/data ``` diff --git a/docker-compose.yml b/docker-compose.yml index 15d70a0..62e1946 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ elasticsearch: environment: ES_JAVA_OPTS: "-Xms1g -Xmx1g" logstash: - image: logstash:5 + build: logstash/ command: -f /etc/logstash/conf.d/ volumes: - ./logstash/config:/etc/logstash/conf.d diff --git a/logstash/Dockerfile b/logstash/Dockerfile new file mode 100644 index 0000000..0bb7979 --- /dev/null +++ b/logstash/Dockerfile @@ -0,0 +1,4 @@ +FROM logstash:latest + +# Add your logstash plugins setup here +# Example: RUN logstash-plugin install logstash-filter-json diff --git a/logstash/config/logstash.conf b/logstash/config/logstash.conf index 2cc24b4..14c76f8 100644 --- a/logstash/config/logstash.conf +++ b/logstash/config/logstash.conf @@ -4,7 +4,7 @@ input { } } -## Add your filters here +## Add your filters / logstash plugins configuration here output { elasticsearch { From bc1cf6eb07bc58fcab0f603fe08aaa350751cd54 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Thu, 7 Jul 2016 10:16:59 +1200 Subject: [PATCH 09/12] fix logstash version in Dockerfile --- logstash/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/Dockerfile b/logstash/Dockerfile index 0bb7979..f3e8783 100644 --- a/logstash/Dockerfile +++ b/logstash/Dockerfile @@ -1,4 +1,4 @@ -FROM logstash:latest +FROM logstash:5 # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json From 426d4600bbe17a901a0deca5f09e7434bb60d5a3 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Thu, 14 Jul 2016 09:48:16 +1200 Subject: [PATCH 10/12] update version support in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4fd4fa9..7179dd0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Join the chat at https://gitter.im/deviantony/fig-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/fig-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -**WARNING: Experimental support of the X-Pack (alpha-3) version of the Elastic stack.** +**WARNING: Experimental support of the X-Pack (alpha-4) version of the Elastic stack.** It is *NOT* recommended to use this in production. From c33c07417a728f3cb2b0500ceeab1de56e8ffa5b Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Fri, 26 Aug 2016 08:51:47 +1200 Subject: [PATCH 11/12] Support x-pack alpha-5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7179dd0..b5cc8d6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Join the chat at https://gitter.im/deviantony/fig-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/fig-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -**WARNING: Experimental support of the X-Pack (alpha-4) version of the Elastic stack.** +**WARNING: Experimental support of the X-Pack (alpha-5) version of the Elastic stack.** It is *NOT* recommended to use this in production. From 7eeb5703ee3267196b80705bbf86bd948bd88a47 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Mon, 24 Oct 2016 09:03:18 +1300 Subject: [PATCH 12/12] Support for X-Pack RC-1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b5cc8d6..e73d052 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Join the chat at https://gitter.im/deviantony/fig-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/fig-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -**WARNING: Experimental support of the X-Pack (alpha-5) version of the Elastic stack.** +**WARNING: Experimental support of the X-Pack (RC-1) version of the Elastic stack.** It is *NOT* recommended to use this in production.