概要
DNSサーバーのqueriesログをElasticsearchに保存する
検証環境
用途 | ip |
---|---|
マスターDNSサーバー | 192.168.24.101 |
スレーブDNSサーバー | 192.168.24.102 |
Elasticsearchサーバー | 192.168.24.103 |
作業履歴
bind の queriesログをsyslogに出力させる設定を入れる
/var/named/chroot/etc/named.conf
(略)
logging {
channel "syslog_local1" {
syslog local1;
};
category queries {
"syslog_local1";
};
};
(略)
bind(chroot環境) の queries ログを syslog で Elasticsearch サーバーへ転送する
shell
cat << EOF > /etc/rsyslog.d/bind_chroot.conf
$AddUnixListenSocket /var/named/chroot/dev/log
local1.* @192.168.24.103:42185
EOF
/etc/init.d/rsyslog restart
Elasticsearch の インストール
shell
yum install java-1.8.0-openjdk-deve
rpm -ivh https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.noarch.rpm
shell
iptables -A INPUT -p tcp -m multiport --dports 8080,9200,9300 -s 127.0.0.1 -j ACCEPT
iptables -A INPUT -p tcp -m multiport --dports 8080,9200,9300 -s 192.168.24.0/24 -j ACCEPT
iptables -A INPUT -p tcp -m multiport --dports 8080,9200,9300 -j DROP
Elasticsearch 設定ファイル修正
shell
sed -i.org \
-e 's/#cluster\.name.*/cluster.name: named.elasticsearch/g' \
-e 's/#discovery\.zen\.ping\.multicast\.enabled.*/discovery.zen.ping.multicast.enabled: false/g' \
/etc/elasticsearch/elasticsearch.yml
diff--- /etc/elasticsearch/elasticsearch.yml.org 2015-02-07 02:19:24.338361576 +0900 +++ /etc/elasticsearch/elasticsearch.yml 2015-02-07 02:29:39.266361349 +0900 @@ -29,7 +29,7 @@ # Cluster name identifies your cluster for auto-discovery. If you're running # multiple clusters on the same network, make sure you're using unique names. # -#cluster.name: elasticsearch +cluster.name: named.elasticsearch #################################### Node ##################################### @@ -319,7 +319,7 @@ # # 1. Disable multicast discovery (enabled by default): # -#discovery.zen.ping.multicast.enabled: false +discovery.zen.ping.multicast.enabled: false # # 2. Configure an initial list of master nodes in the cluster # to perform discovery when new nodes (master or data) are started:
Elasticsearch 起動と確認
shell
/etc/init.d/elasticsearch start
curl -X GET http://localhost:9200/
result{ "status" : 200, "name" : "Vashti", "cluster_name" : "named.elasticsearch", "version" : { "number" : "1.4.2", "build_hash" : "927caff6f05403e936c20bf4529f144f0c89fd8c", "build_timestamp" : "2014-12-16T14:11:12Z", "build_snapshot" : false, "lucene_version" : "4.10.2" }, "tagline" : "You Know, for Search" }
Elasticsearch マッピング確認
shell
curl http://localhost:9200/namedlog-*/_mapping?pretty
Elasticsearch テンプレート確認
shell
curl -XGET localhost:9200/_template/template_1
全テンプレートを確認する場合
shellcurl -XGET localhost:9200/_template/template*
template_1を消す場合
shellcurl -XDELETE localhost:9200/_template/template_1
Elasticsearch テンプレート適用
shell
curl -XPUT localhost:9200/_template/template_1 -d '
{
"template" : "namedlog-*",
"mappings" : {
"fluentd" : {
"properties" : {
"@log_name" : {
"type" : "string"
},
"@timestamp" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"class_type" : {
"type" : "string",
"index" : "not_analyzed"
},
"country" : {
"type" : "string",
"index" : "not_analyzed"
},
"dst" : {
"type" : "string",
"index" : "not_analyzed"
},
"fqdn" : {
"type" : "string",
"index" : "not_analyzed"
},
"log_type" : {
"type" : "string",
"index" : "not_analyzed"
},
"src" : {
"type" : "string",
"index" : "not_analyzed"
},
"view" : {
"type" : "string",
"index" : "not_analyzed"
}
}
}
}
}
}'
Elasticsearch 全てのデータ(index)を削除する
shell
curl -XDELETE 'http://localhost:9200/namedlog-*'
Elasticsearch の WEBフロントエンド インストール
shell
/usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head
Fluent のインストール
shell
curl -L http://toolbelt.treasuredata.com/sh/install-redhat.sh | sh
iptables -A INPUT -s 192.168.24.101 -p udp --dport 42185 -j ACCEPT
iptables -A INPUT -s 192.168.24.102 -p udp --dport 42185 -j ACCEPT
iptables -A INPUT -p udp --dport 42185 -j DROP
mkdir -p /etc/td-agent/conf.d
Fluent の plugin インストール
shell
yum install geoip-devel --enablerepo=epel
/usr/lib64/fluent/ruby/bin/gem install fluent-plugin-parser
/usr/lib64/fluent/ruby/bin/gem install fluent-plugin-geoip
/usr/lib64/fluent/ruby/bin/gem install fluent-plugin-flatten-hash
/usr/lib64/fluent/ruby/bin/gem install fluent-plugin-elasticsearch
tipsCentOS5系にインストールするときはremiのlibcurl-develが必要になる(公式やepelはダメ、バージョンが合わない) yum install gcc libcurl-devel --enablerepo=remi
shell
echo 'include conf.d/*.conf' > /etc/td-agent/td-agent.conf
vi /etc/td-agent/conf.d/bind_queries.conf
/etc/td-agent/conf.d/bind_queries.conf#---------------------------------------------------------- # 各DNSサーバーのrsyslog から のログを取得する処理 # local1.info -> syslog:42185 -> named.syslog.local1.info #---------------------------------------------------------- <source> type syslog port 42185 tag named.rewrite </source> #---------------------------------------------------------- # フィルタリング処理 # /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-rewrite #---------------------------------------------------------- <match named.rewrite.local1.info> type copy #---------------------------------------------------------- # テスト出力 #---------------------------------------------------------- #<store> # type stdout #</store> #---------------------------------------------------------- # named.rewrite.local1.info -> named.parser #---------------------------------------------------------- <store> type rewrite remove_prefix named.rewrite.local1.info add_prefix named.parser #--- QUERY <rule> key message pattern client ([.0-9]+).[0-9]*: view ([^ ]*): [^ ]* ([^ ]*) ([^ ]* [^ ]* [^ ]*) \(([.0-9:]+)\) replace {"log_type":"QUERY", "src":"\1","view":"\2","fqdn":"\3","class_type":"\4","dst":"\5"} last true </rule> #--- パターンにマッチしないものは捨てる <rule> key message pattern .* ignore true </rule> </store> </match> #---------------------------------------------------------- # フィルタリングした文字列をJSONフォーマットに変換 # /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-parser #---------------------------------------------------------- <match named.parser> type copy #---------------------------------------------------------- # テスト出力 #---------------------------------------------------------- #<store> # type stdout #</store> #---------------------------------------------------------- # named.parser -> named.geoip #---------------------------------------------------------- <store> type parser tag named.geoip key_name message format json reserve_data yes </store> </match> #---------------------------------------------------------- # 国別コード付与処理 # yum install geoip-devel --enablerepo=epel # /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-geoip #---------------------------------------------------------- <match named.geoip> type copy #---------------------------------------------------------- # テスト出力 #---------------------------------------------------------- #<store> # type stdout #</store> #---------------------------------------------------------- # named.geoip -> named.flatten_hash #---------------------------------------------------------- <store> type geoip geoip_lookup_key src <record> country ${country_code['src']} </record> tag named.flatten_hash log_level debug flush_interval 1s </store> </match> #---------------------------------------------------------- # ネストしたJSONをフラット化する # /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-flatten-hash #---------------------------------------------------------- <match named.flatten_hash> type copy #---------------------------------------------------------- # テスト出力 #---------------------------------------------------------- #<store> # type stdout #</store> #---------------------------------------------------------- # named.flatten_hash -> named.record_reformer #---------------------------------------------------------- <store> type flatten_hash tag named.record_reformer separator _ </store> </match> #---------------------------------------------------------- # 不必要なキーを捨てる # /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-record-reformer #---------------------------------------------------------- <match named.record_reformer> type copy #---------------------------------------------------------- # テスト出力 #---------------------------------------------------------- #<store> # type stdout #</store> #---------------------------------------------------------- # named.record_reformer -> named.elasticsearch #---------------------------------------------------------- <store> type record_reformer tag named.elasticsearch remove_keys message,ident,pid </store> </match> #---------------------------------------------------------- # Elasticsearch に登録する # /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-elasticsearch #---------------------------------------------------------- <match named.elasticsearch> type copy #---------------------------------------------------------- # テスト出力 #---------------------------------------------------------- <store> type stdout </store> #---------------------------------------------------------- # named.elasticsearch -> elasticsearch [localhost:9200] #---------------------------------------------------------- <store> type elasticsearch include_tag_key true tag_key @log_name host localhost port 9200 logstash_format true logstash_prefix namedlog flush_interval 10s </store> </match>
shell
/etc/init.d/td-agent start
kibana インストール
shell
cd /usr/local/src/
wget https://download.elasticsearch.org/kibana/kibana/kibana-3.1.2.tar.gz
tar zxvf kibana-3.1.2.tar.gz
cd /usr/local/src/kibana-3.1.2
python -m SimpleHTTPServer 8080