Edited at

ELK(Elasticsearch Logstash Kibana) + nginxをdocker-composeで起動

More than 1 year has passed since last update.


はじめに

docker上でしnginxを動かしaccessログをLogstashでelasticsearchにためてkibanaで表示できるまでをやります。

ソースは github にあげました。


環境


  • docker-compoase

  • elasticsearch

  • kibana

  • Logstash

  • nginx


ディレクトリ構成

└── es_logstash

└── es_d
├── docker-compose.yml
├── Dockerfile
└── config
└── elasticsearch.yml
└── kibana_d
├── docker-compose.yml
├── Dockerfile
└── config
└── kibana.yml
└── logstash_d
├── docker-compose.yml
├── Dockerfile
└── config
└── logstash.conf
└── nginx_d
└── docker-compose.yml


es_d/docker-compose.yml

version: '2'

services:
elasticsearch:
mem_limit: 1024m
build: .
container_name: es_c_el
image: es_i_el:1.0.5
volumes:
- ../data/es:/usr/share/elasticsearch/data
ports:
- 9200:9200
environment:
- ES_JAVA_OPTS=-Xms512m -Xmx512m


es_d/Dockerfile

FROM docker.elastic.co/elasticsearch/elasticsearch:6.2.3

COPY ./config/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml

# kuromojiをインストール
RUN elasticsearch-plugin install analysis-kuromoji

# RUN elasticsearch-plugin remove x-pack



es_d/config/elasticsearch.yml

http.host: 0.0.0.0

cluster.name: "docker-cluster"
discovery.type: single-node

### x-pack functions
xpack.security.enabled: false
# 無償利用は1クラスタまで
xpack.monitoring.enabled: true
xpack.graph.enabled: false
xpack.watcher.enabled: false



kibana_d/docker-compose.yml

version: '2'

services:
kibana:
mem_limit: 512m
build: .
container_name: kibana_c_el
image: kibana_i_el:1.0.4
external_links:
- elasticsearch
ports:
- 5601:5601
networks:
- default
- es1_default

networks:
es1_default:
external:
name: es_d_default



kibana_d/Dockerfile

FROM docker.elastic.co/kibana/kibana:6.2.3

COPY ./config/kibana.yml /opt/kibana/config/kibana.yml
# RUN kibana-plugin remove x-pack



kibana_d/kibana.yml

server.name: kibana

server.host: "0"
elasticsearch.url: http://elasticsearch:9200
elasticsearch.username: elastic
elasticsearch.password: changeme
# xpack.monitoring.ui.container.elasticsearch.enabled: true


logstash_d/docker-compose.yml

version: '2'

services:
logstash:
mem_limit: 512m
build: .
container_name: logstash_c_el
image: logstash_i_el:1.0.20
volumes:
- ../data/nginx:/var/log/nginx/
external_links:
- elasticsearch
# ports:
# - 5601:5601
networks:
- default
- es1_default

networks:
es1_default:
external:
name: es_d_default



logstash_d/Dockerfile

FROM docker.elastic.co/logstash/logstash:6.2.3

ADD ./config/logstash.conf /usr/share/logstash/pipeline/logstash.conf



logstash_d/config/logstash.conf

input {

file {
path => "/var/log/nginx/access.log"
start_position => beginning
}
}

filter {
grok {
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[nginx][access][time]"
}
useragent {
source => "[nginx][access][agent]"
target => "[nginx][access][user_agent]"
remove_field => "[nginx][access][agent]"
}
geoip {
source => "[nginx][access][remote_ip]"
target => "[nginx][access][geoip]"
}
}

output {
elasticsearch {
hosts => [ 'elasticsearch' ]
index => "access_log1"
}
}



nginx_d/docker-compose.yml

version: '2'

services:
web:
mem_limit: 512m
image: nginx:1.10
ports:
- "80:80"
volumes:
- ../data/nginx:/var/log/nginx


動作確認


コンテナを起動

コンテナをelasticsearch, Logstash, kibana, nginx の順にアップします。

$ docker-compose up -d


nginxにアクセス

$ curl http://localhost

access.log が更新され、Logstash経由でelasticsearchに格納されます。


elasticsearch

$ curl -XGET 'http://localhost:9200/_cat/count/access_log1'

nginx にアクセスした回数が返ってきます。 


kibana

http://localhost:5601/

kibanaにアクセスして GET /access_log1/_search?pretty=trueを実行すれば確認できます。


最後に

elasticsearch深いです。

github