fluentd+dstat+libratoでサーバのメトリクスを可視化する

fluent-plugin-dstatを使った、dstatメトリクスのlibratoでの可視化について書きます。

用意するもの

• fluentd (v0.12以降)
• fluent-plugin-dstat
• fluent-plugin-librato
• fluent-plugin-filter-object-flatten
• fluent-plugin-filter-record-map
• librato

fluent.conf

以下のようなfluent.confでfluentdを立ち上げると、libratoにメトリクスが送信されます。

<source>
  type dstat
  tag dstat.${hostname}
  option -clmdrn --freespace
  delay 15
</source>

<filter>
  type object_flatten
  tr [" /", "__"]
</filter>

<filter>
  type grep
  exclude1 hostname .+
</filter>

<filter>
  type record_map
  map1 key, value = record.first
  map2 key = key.split(".").slice(1..-1)
  map3 new_record["key"] = (["dstat", hostname] + key.slice(0...-1)).join(".")
  map4 new_record["source"] = key.last
  map5 new_record["value"] = value.to_f
</filter>

<filter>
  type grep
  exclude1 key \.(_dev|_shm)$
</filter>

<match dstat.**>
  type librato
  email [e-mail]
  apikey ...
</match>

librato

こんな感じに出力されます。

ダッシュボードはこんな感じで作れます。

説明

まず、fluent-plugin-dstatを使ってdstatのメトリクスを取得します。

<source>
  type dstat
  tag dstat.${hostname}
  option -clmdrn --freespace
  delay 15
</source>

dstatの出力はこんな感じ。

fluentdに流れるデータはこんな感じ。

{"hostname":"vagrant-ubuntu-trusty-64","dstat":{"total cpu usage":{"usr":"0.773","sys":"0.427","idl":"98.686","wai":"0.084","hiq":"0.0","siq":"0.030"},"load avg":{"1m":"0.0","5m":"0.020","15m":"0.050"},"memory usage":{"used":"206450688.0","buff":"11268096.0","cach":"258924544.0","free":"37146624.0"},"dsk/total":{"read":"108779.758","writ":"130194.143"},"io/total":{"read":"5.093","writ":"1.956"},"net/total":{"recv":"0.0","send":"0.0"},"/":{"used":"3714433024.0","free":"38526730240.0"},"/dev":{"used":"12288.0","free":"251576320.0"},"/run":{"used":"364544.0","free":"51015680.0"},"/rpc_pipefs":{"used":"0.0","free":"0.0"},"/systemd":{"used":"0.0","free":"0.0"},"/vagrant":{"used":"68055891968.0","free":"4.31026593792e+11"}}}

ネストされたデータなので、fluent-plugin-object-flattenを使ってフラットにします。

<filter>
  type object_flatten
  tr [" /", "__"]
</filter>

{"hostname":"vagrant-ubuntu-trusty-64"}
{"dstat.total_cpu_usage.usr":"0.758"}
{"dstat.total_cpu_usage.sys":"0.415"}
{"dstat.total_cpu_usage.idl":"98.717"}
{"dstat.total_cpu_usage.wai":"0.080"}
{"dstat.total_cpu_usage.hiq":"0.0"}
{"dstat.total_cpu_usage.siq":"0.029"}
{"dstat.load_avg.1m":"0.0"}
{"dstat.load_avg.5m":"0.010"}
{"dstat.load_avg.15m":"0.050"}
{"dstat.memory_usage.used":"207048704.0"}
{"dstat.memory_usage.buff":"11325440.0"}
{"dstat.memory_usage.cach":"258924544.0"}
{"dstat.memory_usage.free":"36491264.0"}
{"dstat.dsk_total.read":"104752.233"}
{"dstat.dsk_total.writ":"125423.125"}
{"dstat.io_total.read":"4.906"}
{"dstat.io_total.writ":"1.892"}
{"dstat.net_total.recv":"0.0"}
{"dstat.net_total.send":"0.0"}
{"dstat._.used":"3714433024.0"}
{"dstat._.free":"38526730240.0"}
{"dstat._dev.used":"12288.0"}
{"dstat._dev.free":"251576320.0"}
{"dstat._run.used":"364544.0"}
{"dstat._run.free":"51015680.0"}
{"dstat._rpc_pipefs.used":"0.0"}
{"dstat._rpc_pipefs.free":"0.0"}
{"dstat._systemd.used":"0.0"}
{"dstat._systemd.free":"0.0"}
{"dstat._vagrant.used":"68056227840.0"}
{"dstat._vagrant.free":"4.3102625792e+11"}

hostnameのレコードはいらないので除外。

<filter>
  type grep
  exclude1 hostname .+
</filter>

fluent-plugin-record-mapでlibratoに送信できるフォーマットに修正。

<filter>
  type record_map
  map1 key, value = record.first
  map2 key = key.split(".").slice(1..-1)
  map3 new_record["key"] = (["dstat", hostname] + key.slice(0...-1)).join(".")
  map4 new_record["source"] = key.last
  map5 new_record["value"] = value.to_f
</filter>

{"key":"dstat.vagrant-ubuntu-trusty-64.total_cpu_usage","source":"usr","value":0.733}
{"key":"dstat.vagrant-ubuntu-trusty-64.total_cpu_usage","source":"sys","value":0.399}
{"key":"dstat.vagrant-ubuntu-trusty-64.total_cpu_usage","source":"idl","value":98.765}
{"key":"dstat.vagrant-ubuntu-trusty-64.total_cpu_usage","source":"wai","value":0.075}
{"key":"dstat.vagrant-ubuntu-trusty-64.total_cpu_usage","source":"hiq","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64.total_cpu_usage","source":"siq","value":0.028}
{"key":"dstat.vagrant-ubuntu-trusty-64.load_avg","source":"1m","value":0.07}
{"key":"dstat.vagrant-ubuntu-trusty-64.load_avg","source":"5m","value":0.03}
{"key":"dstat.vagrant-ubuntu-trusty-64.load_avg","source":"15m","value":0.05}
{"key":"dstat.vagrant-ubuntu-trusty-64.memory_usage","source":"used","value":206934016.0}
{"key":"dstat.vagrant-ubuntu-trusty-64.memory_usage","source":"buff","value":11431936.0}
{"key":"dstat.vagrant-ubuntu-trusty-64.memory_usage","source":"cach","value":258945024.0}
{"key":"dstat.vagrant-ubuntu-trusty-64.memory_usage","source":"free","value":36478976.0}
{"key":"dstat.vagrant-ubuntu-trusty-64.dsk_total","source":"read","value":97892.773}
{"key":"dstat.vagrant-ubuntu-trusty-64.dsk_total","source":"writ","value":117304.612}
{"key":"dstat.vagrant-ubuntu-trusty-64.io_total","source":"read","value":4.586}
{"key":"dstat.vagrant-ubuntu-trusty-64.io_total","source":"writ","value":1.783}
{"key":"dstat.vagrant-ubuntu-trusty-64.net_total","source":"recv","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64.net_total","source":"send","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._","source":"used","value":3714433024.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._","source":"free","value":38526730240.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._dev","source":"used","value":12288.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._dev","source":"free","value":251576320.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._run","source":"used","value":364544.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._run","source":"free","value":51015680.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._rpc_pipefs","source":"used","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._rpc_pipefs","source":"free","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._systemd","source":"used","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._systemd","source":"free","value":0.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._vagrant","source":"used","value":68052557824.0}
{"key":"dstat.vagrant-ubuntu-trusty-64._vagrant","source":"free","value":431029927936.0}

いくつかの不要なメトリクスを除外します。

<filter>
  type grep
  exclude1 key \.(_dev|_shm)$
</filter>

最後にfluent-plugin-libratoでlibratoにメトリクスを送信します。

<match dstat.**>
  type librato
  email [e-mail]
  apikey ...
</match>