ELBのアクセスログをJava8のStreamで見る

普段インラインのスクリプトで中身を調べてたELBのアクセスログを、Java8のStreamを使って見てみたら意外に使い勝手が良かったのでメモ。
パッケージ名は適当に変更してください。

コードは下に貼っておきます。

サンプル

ログの個数を調べる

System.out.println(parser.getLogs().size());

Backend Processing Timeの最大値・最小値・平均を調べる

OptionalDouble max = parser.getLogs().stream()
  .mapToDouble(l -> { return l.backendProcessingTime; })
  .max();
OptionalDouble min = parser.getLogs().stream()
    .mapToDouble(l -> { return l.backendProcessingTime; })
    .min();
OptionalDouble avg = parser.getLogs().stream()
    .mapToDouble(l -> { return l.backendProcessingTime; })
    .average();

System.out.println(String.format("Max/Min/Avg = %s/%s/%s", max.getAsDouble(), min.getAsDouble(), avg.getAsDouble()));

Backend Status Codeの種類と数を調べる

parser.getLogs().stream()
  .map(l -> { return l.backendStatusCode; })
  .collect(Collectors.groupingBy(l -> l, Collectors.counting()))
  .forEach((k, v) -> System.out.println(k + ":" + v));

Backend Processingに1秒以上掛かってるURLとMethodを調べる

parser.getLogs().stream()
  .filter(l -> { return l.backendProcessingTime > 1.0; })
  .map(l -> { return String.format("%s %s\t%s", l.requestMethod, l.requestURL, l.backendProcessingTime); } )
  .forEach(System.out::println);

ELBLogParser.java

正規表現がかなり適当だけどこんな感じのクラスを作っておく。

package com.makeitsmartjp.java;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

class ElbLog {
    public String timestamp;
    public String elb;
    public String clientWithPort;
    public String backendWithPort;
    public double requestProcessingTime;
    public double backendProcessingTime;
    public double responseProcessingTime;
    public int elbStatusCode;
    public int backendStatusCode;
    public int receivedBytes;
    public int sentBytes;
    public String requestMethod;
    public String requestURL;
    public String requestHTTPVersion;

    @Override
    public String toString() {
        return String.format("timestamp:%s\telb:%s\tclientWithPort:%s\tbackendWithPort:%s\trequetProcessingTime:%s\tbackendProcessingTime:%s\tresponseProcessingTime:%s\telbStatusCode:%s\tbackendStatusCode:%s\treceivedBytes:%s\tsentBytes:%s\trequestMethod:%s\trequestURL:%s\trequestHTTPVersion:%s",
                timestamp, elb, clientWithPort, backendWithPort,
                requestProcessingTime, backendProcessingTime, responseProcessingTime,
                elbStatusCode, backendStatusCode, receivedBytes, sentBytes,
                requestMethod, requestURL, requestHTTPVersion
                );
    }
}

public class ELBLogParser {
    protected final String patternString = "^(.[^ ]+) (.[^ ]+) (.[^ ]+) (.[^ ]+) (.[^ ]+) (.[^ ]+) (.[^ ]+) (.[^ ]+) (.[^ ]+) (\\d+) (\\d+) \"(\\w+) (.[^ ]+) (.[^ ]+)\"";
    private final Pattern elblogPattern;
    private List<ElbLog> logs;

    public ELBLogParser() {
        logs = new ArrayList<ElbLog>();
        elblogPattern = Pattern.compile(patternString);
    }

    public void parse(String filePath) {
        try (
            InputStream in = new FileInputStream(filePath);
            InputStreamReader isr = new InputStreamReader(in, Charset.forName("UTF-8"));
            BufferedReader br = new BufferedReader(isr);
        ) {
            br.lines().forEach(log -> {
                ElbLog buff = parseLine(log);
                if (buff != null) {
                    logs.add(buff);
                }
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public List<ElbLog> getLogs() {
        return logs;
    }

    protected ElbLog parseLine(String log) {
        Matcher m = elblogPattern.matcher(log);
        if (!m.find()) {
            return null;
        } else if (m.groupCount() < 14) {
            return null;
        }
        ElbLog elbLog = new ElbLog();
        elbLog.timestamp = m.group(1);
        elbLog.elb = m.group(2);
        elbLog.clientWithPort = m.group(3);
        elbLog.backendWithPort = m.group(4);
        elbLog.requestProcessingTime = Double.valueOf(m.group(5));
        elbLog.backendProcessingTime = Double.valueOf(m.group(6));
        elbLog.responseProcessingTime = Double.valueOf(m.group(7));
        elbLog.elbStatusCode = Integer.valueOf(m.group(8));
        elbLog.backendStatusCode = Integer.valueOf(m.group(9));
        elbLog.receivedBytes = Integer.valueOf(m.group(10));
        elbLog.sentBytes = Integer.valueOf(m.group(11));
        elbLog.requestMethod = m.group(12);
        elbLog.requestURL = m.group(13);
        elbLog.requestHTTPVersion = m.group(14);

        return elbLog;
    }
}

Main.java

Mainではこんな感じでダウンロードしたファイルを読み込ませる。

package com.makeitsmartjp.java;


public class Main {

    public static void main(String[] args) {
        String path = "/path/to/elb.log";
        ELBLogParser parser = new ELBLogParser();
        parser.parse(path);

    // 以下に調べたいコードを書いてく
    }
}