Skip to content
Snippets Groups Projects
Commit 8331d113 authored by nimrod's avatar nimrod
Browse files

Full stack.

- Elasticsearch, Logstash and Kibana (with LogTrail installed and index
patterns configured).
- Metricbeat for host and containers and setup of dashboards and indices.
- Heartbeat of various services and setup of dashboards and indices.
- Curator for both days and percent of free disk space.
parent 7c6edb7a
No related branches found
No related tags found
No related merge requests found
*.log
~*
*~
*.sw[op]
*.py[cod]
.DS_Store
__pycache__/
.vagrant/
vendor/
Thumbs.db
*.retry
.svn/
.sass-cache/
*.log
*.out
*.so
node_modules/
.npm/
nbproject/
*.ipynb
.idea/
*.egg-info/
*.[ao]
.classpath
.cache/
bower_components/
*.class
*.[ewj]ar
secring.*
.*.kate-swp
.swp.*
.directory
.Trash-*
build/
_build/
dist/
.tox/
*.pdf
*.exe
*.dll
*.gz
*.tgz
*.tar
*.rar
*.zip
*.pid
*.lock
*.env
.bundle/
data/
---
actions:
1:
action: delete_indices
description: Delete old Logstash indices
ignore_empty_list: True
filters:
- filtertype: pattern
kind: timestring
value: '%Y.%m.%d'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: 30
0 1 * * * /usr/local/bin/curator --config /etc/curator.yml /actions.yml
30 * * * * /usr/local/bin/curator-disk-space
#!/bin/sh
set -eu
[ -n "${CURATOR_DISK_SPACE_PERCENT:-}" ] || { echo "Free disk space percent wasn't set, exiting." > /dev/stderr; exit 1; }
disk_size="$( df -Pk /usr/share/elasticsearch/data | awk 'NR==2 {print $2}' )"
disk_avail="$( df -Pk /usr/share/elasticsearch/data | awk 'NR==2 {print $4}' )"
threshold="$(( disk_size * CURATOR_DISK_SPACE_PERCENT ))"
[ "$threshold" -lt "$disk_avail" ] || { echo "Enough disk space, skipping curator run." > /dev/stderr; exit; }
# Assuming the usage is mainly (all) by Elasticsearch.
export CURATROR_DISK_SPACE="$(( (threshold - disk_avail) / (1024*1024) ))"
exec curator_cli --config /etc/curator.yml \
delete_indices \
--ignore_empty_list \
--filter_list "[{\"filtertype\": \"pattern\",
\"kind\": \"timestring\",
\"value\": \"%Y.%m.%d\"},
{\"filtertype\": \"space\",
\"use_age\": true,
\"disk_space\": $CURATROR_DISK_SPACE}]"
---
client:
hosts:
- ${ELASTICSEARCH_HOST:localhost}
---
version: '3'
services:
metricbeat:
image: docker.elastic.co/beats/metricbeat:${VERSION}
volumes: &metricbeat_volumes
- /:/hostfs:ro
- /proc:/hostfs/proc:ro
- /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
- /var/run/docker.sock:/hostfs/var/run/docker.sock
- ./metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
- ./metricbeat-entrypoint:/usr/local/bin/metricbeat-entrypoint:ro
depends_on:
- elasticsearch
- kibana
- metricbeat_setup
- logstash
healthcheck:
test: ["CMD", "pgrep", "metricbeat"]
interval: 10s
timeout: 3s
retries: 10
environment:
KIBANA_HOST: localhost
ELASTICSEARCH_HOST: localhost
LOGSTASH_HOST: localhost
network_mode: host
pid: host
restart: always
user: root
entrypoint:
- /usr/local/bin/metricbeat-entrypoint
command:
- -e
logging: &logging
driver: gelf
options:
gelf-address: "udp://localhost:12201"
metricbeat_setup:
image: docker.elastic.co/beats/metricbeat:${VERSION}
environment: &environment
KIBANA_HOST: kibana
ELASTICSEARCH_HOST: elasticsearch
LOGSTASH_HOST: logstash
links:
- elasticsearch
- kibana
- logstash
volumes: *metricbeat_volumes
command: metricbeat setup -e
restart: on-failure
logging: *logging
heartbeat:
image: docker.elastic.co/beats/heartbeat:${VERSION}
environment: *environment
links:
- elasticsearch
- kibana
- logstash
depends_on:
- heartbeat_setup
volumes:
- ./heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro
healthcheck:
test: ["CMD", "pgrep", "heartbeat"]
interval: 10s
timeout: 3s
retries: 10
restart: always
logging: *logging
heartbeat_setup:
image: docker.elastic.co/beats/heartbeat:${VERSION}
environment: *environment
links:
- elasticsearch
- kibana
volumes:
- ./heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro
command: heartbeat setup -e
restart: on-failure
logging: *logging
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:${VERSION}
environment:
<<: *environment
'cluster.name': mobile
'discovery.type': 'single-node'
'ES_JAVA_OPTS': '-Xms2g -Xmx2g'
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:9200/_cat/health"]
interval: 10s
timeout: 3s
retries: 10
volumes:
- ./data:/usr/share/elasticsearch/data
ulimits:
nproc: 4096
nofile:
soft: 65536
hard: 65536
#sysctls:
# vm.max_map_count: 262144
restart: always
logging: *logging
ports:
- '9200:9200'
logstash:
image: docker.elastic.co/logstash/logstash-oss:${VERSION}
command: logstash -e "input { http { add_field => { 'input' => 'http' } } beats { port => 5044 add_field => { 'input' => 'beats' } } gelf { add_field => { 'input' => 'gelf' } } } output { elasticsearch { hosts => [ 'elasticsearch' ] index => 'logstash-%{[input]}-%{+YYYY.MM.dd}' }}" --path.settings="" --http.host="0.0.0.0"
links:
- elasticsearch
ports:
- 8080:8080
- 5044:5044
- 12201:12201/udp
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:9600/_node"]
interval: 10s
timeout: 3s
retries: 10
restart: always
logging: *logging
expose:
- "8080"
- "5044"
- "9600"
- "12201"
kibana:
build:
context: ./kibana
args:
KIBANA_VERSION: "${VERSION}"
LOGTRAIL_VERSION: "0.1.24"
links:
- elasticsearch
ports:
- 5601:5601
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5601/"]
interval: 10s
timeout: 3s
retries: 10
restart: always
logging: *logging
environment:
<<: *environment
ELASTICSEARCH_URL: http://elasticsearch:9200
STATUS_ALLOWANONYMOUS: 'True'
SERVER_HOST: 0.0.0.0
kibana_setup:
image: centos:7
links:
- kibana
restart: on-failure
logging: *logging
volumes:
- ./kibana-setup:/usr/local/bin/kibana-setup:ro
command: /usr/local/bin/kibana-setup
environment: *environment
curator:
image: adarnimrod/curator
#build:
# context: ./curator
links:
- elasticsearch
healthcheck:
test: ["CMD", "pgrep", "crond"]
interval: 10s
timeout: 3s
retries: 10
restart: always
logging: *logging
environment:
<<: *environment
CURATOR_DISK_SPACE_PERCENT: 20
volumes:
- ./data:/usr/share/elasticsearch/data:ro
- ./curator/actions.yml:/actions.yml:ro
- ./curator/curator.yml:/etc/curator.yml:ro
- ./curator/crontab:/var/spool/cron/crontabs/nobody:ro
- ./curator/curator-disk-space:/usr/local/bin/curator-disk-space:ro
---
heartbeat:
monitors:
- name: Elasticsearch
type: http
schedule: "@every 1m"
urls:
- "http://${ELASTICSEARCH_HOST}:9200/_cat/health"
- name: Kibana
type: http
schedule: '@every 1m'
urls:
- "http://${KIBANA_HOST}:5601/"
- name: Logstash HTTP
type: http
schedule: '@every 1m'
urls:
- "http://${LOGSTASH_HOST}:9600/?pretty"
- "http://${LOGSTASH_HOST}:9600/_node/"
- name: Logstash TCP
type: tcp
schedule: '@every 1m'
hosts:
- "${LOGSTASH_HOST}"
ports:
- 5044
- 8080
output:
elasticsearch:
hosts:
- "${ELASTICSEARCH_HOST}:9200"
system:
hostfs: /hostfs
setup:
kibana:
host: "${KIBANA_HOST}"
dashboards:
enabled: True
template:
overwrite: true
#!/bin/sh
set -eu
command -v curl>/dev/null || { echo "Can't find curl, exiting."; exit 1; }
[ -n "${KIBANA_HOST:-}" ] || { echo "KIBANA_HOST is not specified, exiting."; exit 1; }
alias curl="curl --verbose \
--fail \
-H 'Accept: application/json, text/plain, */*' \
--compressed \
--retry-delay 2 \
--retry 99 \
-H 'Content-Type: application/json;charset=utf-8' \
-H 'kbn-xsrf: something'"
echo 'Create the logstash-* index pattern.'
while ! curl \
"http://$KIBANA_HOST:5601/api/saved_objects/index-pattern/logstash?overwrite=true" \
--data '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
do
sleep 2
done
echo 'Create the logstash-gelf-* index pattern.'
while ! curl \
"http://$KIBANA_HOST:5601/api/saved_objects/index-pattern/logstash-gelf?overwrite=true" \
--data '{"attributes":{"title":"logstash-gelf-*","timeFieldName":"@timestamp"}}'
do
sleep 2
done
echo 'Create the logstash-http-* index pattern.'
while ! curl \
"http://$KIBANA_HOST:5601/api/saved_objects/index-pattern/logstash-http?overwrite=true" \
--data '{"attributes":{"title":"logstash-http-*","timeFieldName":"@timestamp"}}'
do
sleep 2
done
echo 'Create the logstash-beats-* index pattern.'
while ! curl \
"http://$KIBANA_HOST:5601/api/saved_objects/index-pattern/logstash-beats?overwrite=true" \
--data '{"attributes":{"title":"logstash-beats-*","timeFieldName":"@timestamp"}}'
do
sleep 2
done
echo 'Set the logstash-* index pattern as default.'
while ! curl \
"http://$KIBANA_HOST:5601/api/kibana/settings/defaultIndex?overwrite=true" \
--data '{"value":"logstash"}'
do
sleep 2
done
echo
ARG KIBANA_VERSION
FROM docker.elastic.co/kibana/kibana-oss:${KIBANA_VERSION}
ARG KIBANA_VERSION
ARG LOGTRAIL_VERSION
ENV KIBANA_VERSION ${KIBANA_VERSION}
ENV LOGTRAIL_VERSION ${LOGTRAIL_VERSION}
RUN ./bin/kibana-plugin install "https://github.com/sivasamyk/logtrail/releases/download/v${LOGTRAIL_VERSION}/logtrail-${KIBANA_VERSION}-${LOGTRAIL_VERSION}.zip"
COPY logtrail.json ./plugins/logtrail/logtrail.json
{
"version" : 1,
"index_patterns" : [
{
"es": {
"default_index": "logstash-gelf-*"
},
"tail_interval_in_seconds": 10,
"es_index_time_offset_in_seconds": 0,
"display_timezone": "local",
"display_timestamp_format": "MMM DD HH:mm:ss",
"max_buckets": 500,
"default_time_range_in_days" : 0,
"max_hosts": 100,
"max_events_to_keep_in_viewer": 5000,
"fields" : {
"mapping" : {
"timestamp" : "@timestamp",
"hostname" : "host",
"program": "container_name",
"message": "message"
},
"message_format": "{{{message}}}"
},
"color_mapping" : {
}
},
{
"es": {
"default_index": "logstash-http-*"
},
"tail_interval_in_seconds": 10,
"es_index_time_offset_in_seconds": 0,
"display_timezone": "local",
"display_timestamp_format": "MMM DD HH:mm:ss",
"max_buckets": 500,
"default_time_range_in_days" : 0,
"max_hosts": 100,
"max_events_to_keep_in_viewer": 5000,
"fields" : {
"mapping" : {
"timestamp" : "@timestamp",
"hostname" : "headers.http_host",
"program": "headers.request_uri",
"message": "message"
},
"message_format": "{{{message}}}"
},
"color_mapping" : {
}
},
{
"es": {
"default_index": "logstash-beats-*"
},
"tail_interval_in_seconds": 10,
"es_index_time_offset_in_seconds": 0,
"display_timezone": "local",
"display_timestamp_format": "MMM DD HH:mm:ss",
"max_buckets": 500,
"default_time_range_in_days" : 0,
"max_hosts": 100,
"max_events_to_keep_in_viewer": 5000,
"fields" : {
"mapping" : {
"timestamp" : "@timestamp",
"hostname" : "beat.hostname",
"program": "source",
"message": "message"
},
"message_format": "{{{message}}}"
},
"color_mapping" : {
}
}
]
}
#!/bin/sh
set -eu
[ "$(whoami)" = "root" ] || { echo "Not running as root, exiting." >> /dev/stderr; exit 1; }
gid="$(stat --format='%g' /hostfs/var/run/docker.sock)"
groupadd --force --non-unique --gid "$gid" docker
usermod --append --groups docker metricbeat
eval "exec su -pc '/usr/local/bin/docker-entrypoint $@' metricbeat"
---
metricbeat:
modules:
- module: docker
metricsets:
- container
- cpu
- diskio
- healthcheck
- info
- memory
- network
hosts:
- unix:///hostfs/var/run/docker.sock
- module: system
metricsets:
- core
- cpu
- diskio
- filesystem
- fsstat
- load
- memory
- network
- process
- process_summary
- socket
- uptime
- module: elasticsearch
metricsets:
- node
- node_stats
hosts:
- "${ELASTICSEARCH_HOST}:9200"
- module: kibana
metricsets:
- status
hosts:
- "${KIBANA_HOST}:5601"
- module: logstash
metricsets:
- node
- node_stats
hosts:
- "${LOGSTASH_HOST}"
output:
elasticsearch:
hosts:
- "${ELASTICSEARCH_HOST}:9200"
system:
hostfs: /hostfs
setup:
kibana:
host: "${KIBANA_HOST}"
dashboards:
enabled: True
template:
overwrite: true
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment