ElastAlertでアラート発生時にsnmptrap コマンドでSNMPTrap送信

環境

Elasticsearch、Kibana、Fluentd、ElastAlert、MariaDB

AWS Workspaces
Amazon Linux 2
Docker 18.09
Elasticsearch 7.6.1
Kibana 7.6.1
Fluentd 1.9.3
Mariadb 10.4.12

SNMPTrap受信用サーバ

EC2
Amazon Linux2
Dockerイメージ「sig9/snmptrapd」を使用

ディレクトリ構成

/home/ユーザー名/dkwork/snmp-test
|--docker-compose.yml
|--elastalert
|  |--config
|  |  |--api.config.json
|  |  |--elastalert.yaml
|  |--dockerfiles
|  |  |--Dockerfile
|  |--nginx_config
|  |  |--default.conf
|  |  |--nginx.conf
|  |--public
|  |  |--favicon.ico
|  |  |--index.html
|  |  |--js
|  |  |  |--cron-ui.min.js
|  |  |--praeco.config.json
|  |--rule_templates
|  |--rules
|  |  |--mariadb-error-log-warning.yaml
|--es
|  |--config
|  |  |--elasticsearch.yml
|  |--data
|--fluentd
|  |--dockerfiles
|  |  |--Dockerfile
|  |--etc
|  |  |--fluent.conf
|--kibana
|  |--config
|  |  |--kibana.yml
|--mariadb
|  |--data
|  |--etc
|  |  |--mymariadb.cnf
|  |--log
|  |  |--error.log
|  |  |--general.log
|  |  |--slow.log

環境構築

mkdir -p elastalert/config
mkdir -p elastalert/dockerfiles
mkdir -p elastalert/nginx_config
mkdir -p elastalert/public
mkdir -p elastalert/public/js
mkdir -p elastalert/rule_templates
mkdir -p elastalert/rules
mkdir -p es/config
mkdir -p es/data
chmod 777 es/data
mkdir -p fluentd/dockerfiles
mkdir -p fluentd/etc
mkdir -p kibana/config
mkdir -p mariadb/data
chmod 777 mariadb/data
mkdir -p mariadb/etc
mkdir -p mariadb/log
chmod 777 mariadb/log

以下のファイルはPraecoのサイトの物を使用
https://github.com/ServerCentral/praeco
・elastalert/config/api.config.json
・elastalert/config/elastalert.yaml
・elastalert/nginx_config/default.conf
・elastalert/nginx_config/nginx.conf
・elastalert/public/js/cron-ui.min.js
・elastalert/public/favicon.ico
・elastalert/public/index.html
・elastalert/public/praeco.config.json

docker-compose.yml

version: '3.7'
services:
  elasticsearch:
    container_name: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      - ES_JAVA_OPTS=-Xms256m -Xmx256m
      - discovery.type=single-node
    restart: always
    volumes:
      - ./es/data:/usr/share/elasticsearch/data
      - ./es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:9200 || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 180s

  kibana:
    container_name: kibana
    image: docker.elastic.co/kibana/kibana:7.6.1
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
    restart: always
    volumes:
      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:5601/api/status || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 200s

  elastalert:
    container_name: elastalert
    build: ./elastalert/dockerfiles
    image: elastalert:0.2.1
    ports:
      - 3030:3030
      - 3333:3333
    depends_on:
      - elasticsearch
      - kibana
    restart: always
    volumes:
      - ./elastalert/config/elastalert.yaml:/opt/elastalert/config.yaml
      - ./elastalert/config/api.config.json:/opt/elastalert-server/config/config.json
      - ./elastalert/rules:/opt/elastalert/rules
      - ./elastalert/rule_templates:/opt/elastalert/rule_templates
    healthcheck:
        test: ["CMD-SHELL", "curl -f http://localhost:3030 || exit 1"]
        interval: 30s
        timeout: 15s
        retries: 3
        start_period: 200s

  fluentd:
    container_name: fluentd
    build: ./fluentd/dockerfiles
    image: fluentd:1.9.3
    ports:
      - 24224:24224
      - 24224:24224/udp
    environment:
      - FLUENTD_CONF=fluent.conf
    volumes:
      - ./fluentd/etc/fluent.conf:/fluentd/etc/fluent.conf
      - ./mariadb/log:/var/log/mysql
    user: root
    restart: always

  mariadb:
    container_name: mariadb
    image: mariadb:10.4.12
    ports:
      - 3306:3306
    environment:
      - MYSQL_ROOT_PASSWORD=mariadb
    volumes:
      - ./mariadb/etc:/etc/mysql/conf.d
      - ./mariadb/log:/var/log/mysql
    restart: always

elastalert/dockerfiles/Dockerfile

FROM servercentral/elastalert:latest

USER root

RUN apk add --update --no-cache net-snmp net-snmp-tools

USER node

「xx.xx.xx.xx:162」の「xx.xx.xx.xx」にはSNMPTrap受信サーバーのIPアドレスを指定しました

elastalert/rules/mariadb-error-log-warning

name: mariadb-error-log-warning
type: frequency
index: mariadb-*
num_events: 1
timeframe:
    minutes: 5
realert:
  minutes: 1
filter:
  - query:
      query_string:
        query: '@log_name:mysqld.error AND message:Warning'
alert:
  - command
command: ["/usr/bin/snmptrap", "-IR", "-v", "2c", "-c", "public", "xx.xx.xx.xx:162", "", "netSnmp.99999", "netSnmp.99999.1", "s", "Hello, World"]
is_enabled: true
timestamp_field: '@timestamp'
timestamp_type: iso
use_strftime_index: false

es/config/elasticsearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1

kibana/config/kibana.yml

server.name: kibana
server.host: "0"
elasticsearch.hosts: http://elasticsearch:9200
xpack.monitoring.ui.container.elasticsearch.enabled: true

fluentd/dockerfiles/Dockerfile

FROM fluent/fluentd:v1.9.3-debian-1.0

# Use root account to use apt
USER root

# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish
RUN buildDeps="sudo make gcc g++ libc-dev" \
 && apt-get update \
 && apt-get install -y --no-install-recommends $buildDeps \
 && sudo gem install fluent-plugin-elasticsearch -v 4.0.6 \
 && sudo gem install fluent-plugin-mysqlslowquery -v 0.0.9 \
 && sudo gem sources --clear-all \
 && SUDO_FORCE_REMOVE=yes \
    apt-get purge -y --auto-remove \
                  -o APT::AutoRemove::RecommendsImportant=false \
                  $buildDeps \
 && rm -rf /var/lib/apt/lists/* \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

USER fluent

fluentd/etc/fluent.conf

<source>
  @type mysql_slow_query
  path /var/log/mysql/slow.log
  pos_file /tmp/mysql/slow.pos
  tag mysqld.slow_query
   <parse>
     @type none
   </parse>
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/general.log
  pos_file /tmp/mysql/general.pos
  tag mysqld.general
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/error.log
  pos_file /tmp/mysql/error.pos
  tag mysqld.error
</source>

<match **.**>
  @type copy
  <store>
    @type stdout
  </store>
  <store>
    @type elasticsearch
    include_tag_key true
    tag_key @log_name
    logstash_format true
    logstash_prefix mariadb-log
    host elasticsearch
    port 9200
    index_name mysql
    flush_interval 10s
  </store>
</match>

mariadb/etc/mymariadb.cnf

[mysqld]
general_log
general_log_file=/var/log/mysql/general.log
slow_query_log
slow_query_log_file=/var/log/mysql/slow.log
long_query_time=5 # 5秒以上処理に時間がかかったクエリを記録
log-queries-not-using-indexes # インデックスが使用されていないクエリをログに出力
log-error=/var/log/mysql/error.log

起動・動作確認

SNMPTrap受信サーバー

docker run -it --rm -p 162:162/udp --name snmptrapd sig9/snmptrapd

Created directory: /var/lib/net-snmp/mib_indexes
NET-SNMP version 5.7.3

AWS Workspaces

docker-compose up -d
docker exec -it mariadb bash
root@b078796c824f:/# mysql -u root -px
ERROR 1045 (28000): Access denied for user 'root'@'localhost' (using password: YES)
root@b078796c824f:/# mysql -u root -px
ERROR 1045 (28000): Access denied for user 'root'@'localhost' (using password: YES)
root@b078796c824f:/# exit

SNMPTrap受信サーバー

# IPアドレス部分は「xx」で加工しています
2020-03-23 15:39:57 ec2-xx-xx-xx-xx.ap-northeast-1.compute.amazonaws.com [UDP: [xx.xx.xx.xx]:56622->[172.17.0.2]:162]:
DISMAN-EVENT-MIB::sysUpTimeInstance = Timeticks: (51438887) 5 days, 22:53:08.87	SNMPv2-MIB::snmpTrapOID.0 = OID: NET-SNMP-MIB::netSnmp.99999	NET-SNMP-MIB::netSnmp.99999.1 = STRING: "Hello, World"