9
10

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?

More than 5 years have passed since last update.

CloudWatchLogsからS3へのエクスポートを自動化

Last updated at Posted at 2018-06-11

この辺りをものすごく参考

Lambda を使って定期的にS3へとログを飛ばす

Lambda関数のロール設定例

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "logs:CreateExportTask",
                "logs:CreateLogStream",
                "logs:CreateLogGroup",
                "logs:PutLogEvents"
            ],
            "Resource": "arn:aws:logs:*:*:*"
        }
    ]
}

logs:CreateExportTask が CloudWatchLogs からログをエクスポートするために必要

それ以外は、LambdaからCloudWatchLogsへのログ書き込みに必要

Lambda関数作成

以下、ソース

Node.js (8.10)

'use strict';

const aws = require('aws-sdk');
aws.config.update({region: 'ap-northeast-1'});

exports.handler = (event, context, callback) => {
    const getToTS = function() {
        let now = new Date();
        now.setHours(0);
        now.setMinutes(0);
        now.setSeconds(0);
        now.setMilliseconds(0);
        return now.getTime();
    };
    const getFromTS = function(toTS, from) {
        let to = new Date(toTS);
        to.setDate(to.getDate() - from);
        return to.getTime();
    };
    const dateFormat = function(date) {
        let y = ('000' + date.getFullYear()).slice(-4);
        let m = ('0' + (date.getMonth() + 1)).slice(-2);
        let d = ('0' + date.getDate()).slice(-2);
        return y + "-" + m + "-" + d;
    };

    const s3 = new aws.S3();
    const cloudwatchlogs = new aws.CloudWatchLogs();
    const createExportTask = (opt) => {
        return new Promise((resolve, reject) => {
            cloudwatchlogs.createExportTask(opt, (err, data) => {
                if (err) {
                    console.log("createExportTask failed", err, err.stack);
                    resolve(false);
                    return;
                }
                console.log("createExportTask success", opt.logGroupName, data);
                resolve(data);
            });
        });
    };
    const sleep = (msec) => {
        return new Promise((resolve, reject) => {
            setTimeout(() => {
                resolve();
            }, msec);
        });
    };

    const s3BucketName = 's3-bucket'; // 出力先のS3バケット
    const logGroups = [
        {'name':'/var/log/httpd/access_log','s3prefix':'httpd_access_log'},
        {'name':'/var/log/httpd/error_log','s3prefix':'httpd_error_log'},
    ];

    // 日次での処理を想定
    const toTS = getToTS();             // 実行日の00:00:00.0
    const fromTS = getFromTS(toTS, 30); // 実行日30日前の00:00:00.0

    const trial = 5;    // 失敗時の試行回数
    const trial_duration = 6000;

    const run = async function() {
        for (let i = 0; i < logGroups.length; i++) {
            let s3prefix = `${logGroups[i].s3prefix}/` + dateFormat(new Date(fromTS));
            let taskName = `export_task_${logGroups[i].s3prefix}` + (dateFormat(new Date(fromTS)));
            // logGroups[i].name というロググループに溜められたログの内容が s3://${s3BucketName}/${logGroups[i].name}/yyyy-mm-dd/ に出力される。
            try {
                let count = 1;
                let opt = {
                    'taskName': taskName,
                    'logGroupName': logGroups[i].name,
                    'from': fromTS,
                    'to': toTS,
                    'destination': s3BucketName,
                    'destinationPrefix': s3prefix
                };
                let result = await createExportTask(opt);
                while (result == false) {
                    if (count >= trial) {
                        break;
                    }
                    await sleep(trial_duration);
                    count++;
                    result = await createExportTask(opt);
                }
                if (result == false) {
                    throw new Error("createExportTask failed. logGroupName: " + logGroups[i].name);
                }
            } catch (err) {
                console.error("createExportTask Exception", err);
                callback(err);
                return;
            }
        }
        callback(null, "Finish");
    };
    run();
};

Lambda関数のタイムアウト設定

image.png

各々設定


あとは、CloudWatchEventsで上記が実行されるように設定するのと、CloudWatchLogsのログ保持期間を設定するだけ

適宜 logGroups を増やして対応していく

9
10
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
9
10

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?