Fluentdで収集したログのElasticsearchのIndexをIndex Lifecycle Management(ILM)の管理対象にする

環境

AWS Workspaces
Amazon Linux2
Elasticsearch 7.5.2
Kibana 7.5.2
Fluentd 1.9.3
MariaDB 10.4.12

/home/username/dkwork/es
|--docker-compose.yml
|--es
|  |--config
|  |  |--elasticsearch.yml
|  |--data
|--fluentd
|  |--dockerfiles
|  |  |--Dockerfile
|  |--etc
|  |  |--fluent.conf
|  |  |--mysql_template.json
|--kibana
|  |--config
|  |  |--kibana.yml
|--mariadb
|  |--etc
|  |  |--mymariadb.cnf
|  |--log
|  |  |--error.log
|  |  |--general.log
|  |  |--slow.log

ファイル

/home/username/dkwork/docker-compose.yml

version: '3.7'
services:
  elasticsearch:
    container_name: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
    volumes:
      - ./es/data:/usr/share/elasticsearch/data
      - ./es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      - ES_JAVA_OPTS=-Xms128m -Xmx128m
      - discovery.type=single-node
    restart: unless-stopped

  kibana:
    container_name: kibana
    image: docker.elastic.co/kibana/kibana:7.5.2
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
    restart: unless-stopped

  fluentd:
    container_name: fluentd
    build: ./fluentd/dockerfiles
    image: fluentd:1.9.3
    ports:
      - 24224:24224
      - 24224:24224/udp
    environment:
      - FLUENTD_CONF=fluent.conf
    volumes:
      - ./fluentd/etc/fluent.conf:/fluentd/etc/fluent.conf
      - ./fluentd/etc/mysql_template.json:/fluentd/etc/mysql_template.json
      - ./mariadb/log:/var/log/mysql
    user: root
    restart: unless-stopped

  mariadb:
    container_name: mariadb
    image: mariadb:10.4.12
    ports:
      - 3306:3306
    environment:
      - MYSQL_ROOT_PASSWORD=mariadb
    volumes:
      - ./mariadb/etc:/etc/mysql/conf.d
      - ./mariadb/log:/var/log/mysql
    restart: unless-stopped

/home/username/dkwork/es/es/config/elasticsearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1

/home/username/dkwork/es/fluentd/dockerfiles/Dockerfile

FROM fluent/fluentd:v1.9.3-debian-1.0

# Use root account to use apt
USER root

# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish
RUN buildDeps="sudo make gcc g++ libc-dev" \
 && apt-get update \
 && apt-get install -y --no-install-recommends $buildDeps \
 && sudo gem install fluent-plugin-elasticsearch -v 4.0.5 \
 && sudo gem install fluent-plugin-mysqlslowquery -v 0.0.9 \
 && sudo gem install elasticsearch-xpack -v 7.5.0 \
 && sudo gem sources --clear-all \
 && SUDO_FORCE_REMOVE=yes \
    apt-get purge -y --auto-remove \
                  -o APT::AutoRemove::RecommendsImportant=false \
                  $buildDeps \
 && rm -rf /var/lib/apt/lists/* \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

USER fluent

/home/username/dkwork/es/fluentd/etc/fluent.conf

<source>
  @type mysql_slow_query
  path /var/log/mysql/slow.log
  pos_file /tmp/mysql/slow.pos
  tag mysqld.slow_query
   <parse>
     @type none
   </parse>
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/general.log
  pos_file /tmp/mysql/general.pos
  tag mysqld.general
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/error.log
  pos_file /tmp/mysql/error.pos
  tag mysqld.error
</source>

<match **.**>
  @type copy
  <store>
    @type stdout
  </store>
  <store>
    @type elasticsearch
    include_tag_key true
    include_timestamp true # defaults to false
    tag_key @log_name
    host elasticsearch
    port 9200
    # Rollover index configuration
    rollover_index true # defaults to false
    index_name mysql
    # fluent-plugin-elasticsearchの4.0.0から
    # index_prefixは廃止になったのでコメント化
    #index_prefix mysql # defaults to "logstash"
    application_name log # defaults to "default"
    deflector_alias mysql-log
    template_name mysql_template
    template_file /fluentd/etc/mysql_template.json
    # elasticsearch-xpack gem
    enable_ilm true # Default value is false 
    ilm_policy_id mysql-policy # Default value is logstash-policy
    ilm_policy { "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "rollover": { "max_age": "7d", "max_docs": 20, "max_size": "5gb" } } }, "delete": { "min_age": "1h", "actions": { "delete": {} } } } } }
    flush_interval 10s
  </store>
</match>

/home/username/dkwork/es/fluentd/etc/mysql_template.json

{
  "index_patterns": ["mysql-log-*"], 
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "myisql-policy", 
    "index.lifecycle.rollover_alias": "mysql-log"
  }
}

/home/username/dkwork/es/kibana/config/kibana.yml

server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch:9200

/home/username/dkwork/es/mariadb/etc/mymariadb.cnf

[mysqld]
general_log
general_log_file=/var/log/mysql/general.log
slow_query_log
slow_query_log_file=/var/log/mysql/slow.log
long_query_time=5 # 5秒以上処理に時間がかかったクエリを記録
log-queries-not-using-indexes # インデックスが使用されていないクエリをログに出力
log-error=/var/log/mysql/error.log

準備

$ cd /home/username/dkwork
$ mkdir es
$ cd es
$ mkdir -p es/data
$ chmod 777 es/data 
$ mkdir -p mariadb/log
$ chmod 777 mariadb/log

その他のファイル作成

実行

$ docker-compose up -d

確認