docker上で Filebeat nginx Module を使って nginxのlogをKibanaで表示させた

はじめに

前回は
docker上でしnginxを動かしaccessログをFilebeat　から Logstashに送信していましたが、
今回はFilebeat Moduleを使ってElasticsearchに送信するように変更しました。

ソースは github にあげました。

環境

• docker-compoase
• elasticsearch
• kibana
• Filebeat (nginx Module)
• nginx

ディレクトリ構成

└── es_logstash
    └── es_d
        ├── docker-compose.yml
        ├── Dockerfile
        └── config
            └── elasticsearch.yml
    └── filebeat_d
        ├── docker-compose.yml
        ├── Dockerfile
        └── config
            └── filebeat.yml
            └── nginx.yml
    └── kibana_d
        ├── docker-compose.yml
        ├── Dockerfile
        └── config
            └── kibana.yml
    └── nginx_d
        └── docker-compose.yml

es_d/docker-compose.yml

version: '2'
services:
  elasticsearch:
    mem_limit: 512m
    build: .
    container_name: es_c_el
    image: es_i_el:1.0.10
    volumes:
      - ../data/es:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    environment:
      - ES_JAVA_OPTS=-Xms256m -Xmx256m

es_d/Dockerfile

FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.3

COPY ./config/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml

# kuromojiをインストール
RUN elasticsearch-plugin  install analysis-kuromoji

es_d/config/elasticsearch.yml

http.host: 0.0.0.0

cluster.name: "docker-cluster"

filebeat_d/docker-compose.yml

version: '2'
services:
  filebeat:
    mem_limit: 64m
    build: .
    container_name: filebeat_c_el
    image: filebeat_i_el:1.0.1
    volumes:
      - ../data/nginx:/var/log/nginx/
    external_links:
      - elasticsearch
      - kibana
    networks:
      - default
      - es1_default
      - kibana1_default

networks:
  es1_default:
    external:
      name: es_d_default
  kibana1_default:
    external:
      name: kibana_d_default

filebeat_d/Dockerfile

FROM docker.elastic.co/beats/filebeat:6.2.3

COPY ./config/filebeat.yml /usr/share/filebeat/filebeat.yml
USER root
RUN chown root:filebeat /usr/share/filebeat/filebeat.yml
USER filebeat

COPY ./config/nginx.yml /usr/share/filebeat/modules.d/nginx.yml
USER root
RUN chown root:filebeat /usr/share/filebeat/modules.d/nginx.yml
USER filebeat

filebeat_d/config/filebeat.yml

filebeat.config:
  prospectors:
    path: ${path.config}/prospectors.d/*.yml
    reload.enabled: false
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
- add_cloud_metadata:

output.elasticsearch:
  hosts: ['elasticsearch:9200']
  username: elastic
  password: changeme

setup.dashboards.enabled: true

setup.kibana:
  host: "kibana:5601"

filebeat_d/config/ngins.yml

- module: nginx
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths:
      - /var/log/nginx/access.log

  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths:
      - /var/log/nginx/error.log

kibana_d/docker-compose.yml

version: '2'
services:
  kibana:
    mem_limit: 128m
    build: .
    container_name: kibana_c_el
    image: kibana_i_el:1.0.9
    external_links:
      - elasticsearch
    ports:
      - 5601:5601
    networks:
      - default
      - es1_default
    environment:
      NODE_OPTIONS: "--max-old-space-size=100"

networks:
  es1_default:
    external:
      name: es_d_default

kibana_d/Dockerfile

FROM docker.elastic.co/kibana/kibana-oss:6.2.3

COPY ./config/kibana.yml /opt/kibana/config/kibana.yml

kibana_d/config/kibana.yml

server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch:9200
elasticsearch.username: elastic
elasticsearch.password: changeme

nginx_d/docker-compose.yml

version: '2'
services:
  web:
    image: nginx:1.10
    ports:
      - "80:80"
    volumes:
      - ../data/nginx:/var/log/nginx

動作確認

コンテナを起動

コンテナをelasticsearch, kibana, filebeat, nginx の順にアップします。

$ docker-compose up -d

nginxにアクセス

$ curl http://localhost

access.log が更新され、Filebea経由でelasticsearchに格納されます。

kibana

Dashboard / [Filebeat Nginx] Access and error logs にアクセスすればグラフが表示されます。

最後に

elasticsearch深いです。

github