From 2d90bfce2d475555fb1abf6678f53e220a30d40a Mon Sep 17 00:00:00 2001 From: Jason Yee <446031+jwsy@users.noreply.github.com> Date: Thu, 5 Mar 2020 09:08:54 +0100 Subject: [PATCH] Add App Search extension (#471) Co-authored-by: Antoine Cotten --- .travis.yml | 12 +++ .travis/lib/testing.sh | 3 + .travis/run-tests-app-search.sh | 36 +++++++++ extensions/app-search/Dockerfile | 4 + extensions/app-search/README.md | 83 ++++++++++++++++++++ extensions/app-search/app-search-compose.yml | 22 ++++++ extensions/app-search/config/app-search.yml | 18 +++++ 7 files changed, 178 insertions(+) create mode 100755 .travis/run-tests-app-search.sh create mode 100644 extensions/app-search/Dockerfile create mode 100644 extensions/app-search/README.md create mode 100644 extensions/app-search/app-search-compose.yml create mode 100644 extensions/app-search/config/app-search.yml diff --git a/.travis.yml b/.travis.yml index 1825628..854fe1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ before_script: - sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml - sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml - sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf + - sed -i 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' extensions/app-search/config/app-search.yml script: # Core Elastic Stack @@ -36,6 +37,17 @@ script: - docker-compose logs logstash - docker-compose down -v + # Extensions + - docker-compose up -d elasticsearch + - sleep 30 + - .travis/elasticsearch-setup-passwords.exp + # App Search + - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml up -d app-search + - .travis/run-tests-app-search.sh + - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml ps + - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml logs app-search + - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml down -v + # Swarm - docker swarm init - docker stack deploy -c ./docker-stack.yml elk diff --git a/.travis/lib/testing.sh b/.travis/lib/testing.sh index 1900868..4c535a0 100755 --- a/.travis/lib/testing.sh +++ b/.travis/lib/testing.sh @@ -12,6 +12,9 @@ function poll_ready { if [ "$#" -ge 3 ]; then args+=( '-u' "$3" ) fi + if [ "$#" -ge 4 ]; then + args+=( '-o' '/dev/null' ) + fi local label if [ "$MODE" == "swarm" ]; then diff --git a/.travis/run-tests-app-search.sh b/.travis/run-tests-app-search.sh new file mode 100755 index 0000000..ad74f84 --- /dev/null +++ b/.travis/run-tests-app-search.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + + +source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh" + + +declare MODE="" + +log 'Waiting for readiness of Elasticsearch' +poll_ready elasticsearch 'http://localhost:9200/' 'elastic:testpasswd' + +log 'Waiting for readiness of App Search' +poll_ready app-search 'http://localhost:3002/login' ':' quiet + +log 'Retrieving private key from Elasticsearch' +response="$(curl 'http://localhost:9200/.app-search-actastic-loco_moco_api_tokens/_search?q=name:private-key' -s -u elastic:testpasswd)" +hits="$(jq -rn --argjson data "${response}" '$data.hits.hits')" +echo "$hits" +count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" +if [[ $count -ne 1 ]]; then + echo "Private key not found. Expected 1 result, got ${count}" + exit 1 +fi +key="$(jq -rn --argjson data "${hits}" '$data[0]._source.authentication_token')" + +log 'Creating App Search engine' +response="$(curl 'http://localhost:3002/api/as/v1/engines' -s -d '{"name": "dockerelk"}' -H "Authorization: Bearer ${key}")" +echo "$response" +name="$(jq -rn --argjson data "${response}" '$data.name')" +if [[ $name != 'dockerelk' ]]; then + echo 'Failed to create engine' + exit 1 +fi diff --git a/extensions/app-search/Dockerfile b/extensions/app-search/Dockerfile new file mode 100644 index 0000000..9e32f53 --- /dev/null +++ b/extensions/app-search/Dockerfile @@ -0,0 +1,4 @@ +ARG ELK_VERSION + +# https://www.docker.elastic.co/ +FROM docker.elastic.co/app-search/app-search:${ELK_VERSION} diff --git a/extensions/app-search/README.md b/extensions/app-search/README.md new file mode 100644 index 0000000..1530a54 --- /dev/null +++ b/extensions/app-search/README.md @@ -0,0 +1,83 @@ +# App Search extension + +Elastic App Search provides access to a set of robust APIs and people friendly dashboard controls to deliver amazing +search experiences, all backed by the Elastic Stack. + +## Requirements + +* 2 GB of free RAM, on top of the resources required by the other stack components and extensions + +App Search exposes the TCP port `3002` for its Web UI and API. + +## Usage + +To include App Search in the stack, run Docker Compose from the root of the repository with an additional command +line argument referencing the `app-search-compose.yml` file: + +```console +$ docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml up +``` + +Allow a few minutes for the stack to start, then open your web browser at the address http://localhost:3002 to see the +App Search home page. + +App Search is configured on first boot with the following default credentials: + +* user: *app_search* +* password: *changeme* + +## Security + +The App Search password is defined inside the Compose file via the `APP_SEARCH_DEFAULT_PASSWORD` environment variable. +We highly recommend choosing a more secure password than the default one for security reasons. + +To do so, change the value `APP_SEARCH_DEFAULT_PASSWORD` environment variable inside the Compose file **before the first +boot**: + +```yaml +app-search: + + environment: + APP_SEARCH_DEFAULT_PASSWORD: {{some strong password}} +``` + +> :warning: The default App Search password can only be set during the initial boot. Once the password is persisted in +> Elasticsearch, it can only be changed via the Elasticsearch API. + +For more information, please refer to [Security and User Management][appsearch-security]. + +## Configuring App Search + +The App Search configuration is stored in [`config/app-search.yml`][config-appsearch]. You can modify this file using +the [Default App Search configuration][appsearch-config] as a reference. + +You can also specify the options you want to override by setting environment variables inside the Compose file: + +```yaml +app-search: + + environment: + app_search.auth.source: standard + worker.threads: '6' +``` + +Any change to the App Search configuration requires a restart of the App Search container: + +```console +$ docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml restart app-search +``` + +Please refer to the following documentation page for more details about how to configure App Search inside a Docker +container: [Run App Search as a Docker container][appsearch-docker]. + +## See also + +[App Search Self-Managed documentation][appsearch-selfmanaged] + + +[config-appsearch]: ./config/app-search.yml + +[appsearch-security]: https://swiftype.com/documentation/app-search/self-managed/security +[appsearch-config]: https://swiftype.com/documentation/app-search/self-managed/configuration +[appsearch-docker]: https://swiftype.com/documentation/app-search/self-managed/installation#docker +[appsearch-selfmanaged]: https://swiftype.com/documentation/app-search/self-managed/overview diff --git a/extensions/app-search/app-search-compose.yml b/extensions/app-search/app-search-compose.yml new file mode 100644 index 0000000..1217544 --- /dev/null +++ b/extensions/app-search/app-search-compose.yml @@ -0,0 +1,22 @@ +version: '3.2' + +services: + app-search: + build: + context: extensions/app-search/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./extensions/app-search/config/app-search.yml + target: /usr/share/app-search/config/app-search.yml + read_only: true + environment: + JAVA_OPTS: -Xmx2g -Xms2g + APP_SEARCH_DEFAULT_PASSWORD: changeme + ports: + - '3002:3002' + networks: + - elk + depends_on: + - elasticsearch diff --git a/extensions/app-search/config/app-search.yml b/extensions/app-search/config/app-search.yml new file mode 100644 index 0000000..4fa8a27 --- /dev/null +++ b/extensions/app-search/config/app-search.yml @@ -0,0 +1,18 @@ +--- +## App Search core configuration +## https://swiftype.com/documentation/app-search/self-managed/configuration +# + +# IP address App Search listens on +app_search.listen_host: 0.0.0.0 + +# URL at which users reach App Search +app_search.external_url: http://localhost:3002 + +# Elasticsearch URL and credentials +elasticsearch.host: http://elasticsearch:9200 +elasticsearch.username: elastic +elasticsearch.password: changeme + +# Allow App Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes. +allow_es_settings_modification: true