diff --git a/.travis.yml b/.travis.yml index ffeeaf1..493ad82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,53 @@ -sudo: required - -language: generic +language: minimal +services: docker env: - DOCKER_COMPOSE_VERSION=1.14.0 install: - # Installing docker - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y install docker-ce - - # Installing docker-compose - - sudo rm /usr/local/bin/docker-compose - - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - -before_script: - # Check docker & docker-compose versions - - docker --version + # Install Docker Compose + - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o "$HOME/bin/docker-compose" + - chmod +x "$HOME/bin/docker-compose" - docker-compose --version -script: +before_script: + # Build images - docker-compose build - - docker-compose up -d - # Verifications - - sleep 30 + # Pull buildpack image (curl 7.52.0+) + - docker pull buildpack-deps:artful-curl + - shopt -s expand_aliases + - alias curl='docker run --rm --net=host buildpack-deps:artful-curl curl' + +script: + # Compose + - docker-compose up -d + - sleep 60 + - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/' + - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status' + - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty' + - echo 'dockerelk' | nc localhost 5000 + - sleep 2 + - curl -s -XPOST 'http://localhost:9200/_refresh' + - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,' + - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty' + - docker-compose ps - docker-compose logs - - curl --retry 10 --retry-delay 5 -D- http://localhost:9200/ - - curl --retry 10 --retry-delay 5 -D- http://localhost:5601/api/status + - docker-compose down -v + + # Swarm + - docker swarm init + - docker stack deploy -c ./docker-stack.yml elk + - sleep 60 + - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/' + - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status' + - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty' + - echo 'dockerelk' | nc localhost 5000 + - sleep 2 + - curl -s -XPOST 'http://localhost:9200/_refresh' + - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,' + - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty' + - docker stack services elk + - docker service logs elk_elasticsearch + - docker service logs elk_kibana + - docker service logs elk_logstash diff --git a/README.md b/README.md index aa7ae1f..255f87c 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,10 @@ Based on the official Docker images from Elastic: 6. [JVM tuning](#jvm-tuning) * [How can I specify the amount of memory used by a service?](#how-can-i-specify-the-amount-of-memory-used-by-a-service) * [How can I enable a remote JMX connection to a service?](#how-can-i-enable-a-remote-jmx-connection-to-a-service) -7. [Updates](#updates) +7. [Going further](#going-further) * [Using a newer stack version](#using-a-newer-stack-version) + * [Plugins and integrations](#plugins-and-integrations) + * [Docker Swarm](#docker-swarm) ## Requirements @@ -246,7 +248,7 @@ logstash: ### How can I enable a remote JMX connection to a service? -As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the docker +As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the Docker host. Update the `{ES,LS}_JAVA_OPTS` environment variable with the following content (I've mapped the JMX service on the port @@ -260,7 +262,7 @@ logstash: LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false" ``` -## Updates +## Going further ### Using a newer stack version @@ -274,3 +276,28 @@ $ docker-compose up **NOTE**: Always pay attention to the [upgrade instructions](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html) for each individual component before performing a stack upgrade. + +### Plugins and integrations + +See the following Wiki pages: + +* [External applications](https://github.com/deviantony/docker-elk/wiki/External-applications) +* [Popular integrations](https://github.com/deviantony/docker-elk/wiki/Popular-integrations) + +### Docker Swarm + +Experimental support for Docker Swarm is provided in the form of a `docker-stack.yml` file, which can be deployed in an +existing Swarm cluster using the following command: + +```console +$ docker stack deploy -c docker-stack.yml elk +``` + +If all components get deployed without any error, the following command will show 3 running services: + +```console +$ docker stack services elk +``` + +**NOTE:** to scale Elasticsearch in Swarm mode, configure *zen* to use the DNS name `tasks.elasticsearch` instead of +`elasticsearch`. diff --git a/docker-compose.yml b/docker-compose.yml index bd651fb..aa94191 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - ./logstash/pipeline:/usr/share/logstash/pipeline:ro ports: - "5000:5000" + - "9600:9600" environment: LS_JAVA_OPTS: "-Xmx256m -Xms256m" networks: diff --git a/docker-stack.yml b/docker-stack.yml new file mode 100644 index 0000000..d71458c --- /dev/null +++ b/docker-stack.yml @@ -0,0 +1,65 @@ +version: '3.3' + +services: + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.4.0 + ports: + - "9200:9200" + - "9300:9300" + configs: + - source: elastic_config + target: /usr/share/elasticsearch/config/elasticsearch.yml + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + deploy: + mode: replicated + replicas: 1 + + logstash: + image: docker.elastic.co/logstash/logstash-oss:6.4.0 + ports: + - "5000:5000" + - "9600:9600" + configs: + - source: logstash_config + target: /usr/share/logstash/config/logstash.yml + - source: logstash_pipeline + target: /usr/share/logstash/pipeline/logstash.conf + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + deploy: + mode: replicated + replicas: 1 + + kibana: + image: docker.elastic.co/kibana/kibana-oss:6.4.0 + ports: + - "5601:5601" + configs: + - source: kibana_config + target: /usr/share/kibana/config/kibana.yml + networks: + - elk + deploy: + mode: replicated + replicas: 1 + +configs: + + elastic_config: + file: ./elasticsearch/config/elasticsearch.yml + logstash_config: + file: ./logstash/config/logstash.yml + logstash_pipeline: + file: ./logstash/pipeline/logstash.conf + kibana_config: + file: ./kibana/config/kibana.yml + +networks: + elk: + driver: overlay