nginxのログをS3に送信(td-agent)

やりたいこと

• nginxのaccesslogをS3に貯めたい

td-agent install

$ curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh | sh

S3 outputプラグインインストール

$ sudo /opt/td-agent/embedded/bin/fluent-gem update
$ sudo /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-s3
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-forest

td-agent.confの編集 /etc/td-agent/td-agent.conf

<source>
  type config_expander
  <config>
    type tail
    # format nginx
    format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarder>[^\"]*)")?/
    time_format %d/%b/%Y:%H:%M:%S %z
    pos_file /var/log/td-agent/access.log.pos
    path /var/log/nginx/access.log
    tag ${hostname}/nginx.access
  </config>
</source>
<source>
  type config_expander
  <config>
    type tail
    format /^[[^ ]* (?<time>[^]]*)] [(?<level>[^]]*)] (?<message>.*)$/
    time_format %b %d %H:%M:%S
    pos_file /var/log/td-agent/error.log.pos
    path /var/log/nginx/error.log
    tag ${hostname}/nginx.error
  </config>
</source>

<source>
  type config_expander
  <config>
    type tail
    format syslog
    pos_file /var/log/td-agent/syslog.pos
    path /var/log/messages
    tag ${hostname}/syslog.messages
  </config>
</source>

<match *.**>
  type forest
  subtype s3
   <template>
     # /etc/sysconfig/td-agent
     aws_key_id "#{ENV['AWS_KEY_ID']}" 
     aws_sec_key "#{ENV['AWS_SECRET_KEY']}"
     s3_bucket fluent-log-test

     path
     buffer_path /var/log/td-agent/buffer/${tag}
     time_slice_format %Y/%m/%d/${tag}/ec2-%Y-%m-%d-%H
     retry_wait 30s
     retry_limit 5
     flush_interval 1s
     flush_at_shutdown true
   </template>
</match>

AWS設定 /etc/sysconfig/td-agent

export AWS_KEY_ID="xxxxxxxxx"
export AWS_SECRET_KEY="xxxxxxxxxxxx"

td-agent 起動

service td-agent start