cloudwatch,slack,nginxログを送る

nginx to cloudwatchの設定

AWS CloudWatch Logsエージェントのインストール

cloudwatch to slack

blueprint使うと早い
slackのwebhookとchannelで十分
https://dev.classmethod.jp/cloud/aws/lambda-blueprint-slack-cloudwatch/
https://qiita.com/yahagin/items/2248287c5285cd1e9201

ログ内容を出力するスクリプトhttp://htnosm.hatenablog.com/entry/2016/05/05/090000

SNSログの形

{
"AlarmName": "sample-error",
"AlarmDescription": "sampleでエラーが発生しました。",
"AWSAccountId": "xxxxxxxxxxxx",
"NewStateValue": "ALARM",
"NewStateReason": "Threshold Crossed: 1 datapoint [2.0 (29/11/17 01:09:00)] was greater than or equal to the threshold (1.0).",
"StateChangeTime": "2017-11-29T01:10:32.907+0000",
"Region": "Asia Pacific (Tokyo)",
"OldStateValue": "OK",
"Trigger": {
"MetricName": "sample-metric",
"Namespace": "LogMetrics",
"StatisticType": "Statistic",
"Statistic": "SUM",
"Unit": null,
"Dimensions": [],
"Period": 60,
"EvaluationPeriods": 1,
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
"Threshold": 1,
"TreatMissingData": "- TreatMissingData: NonBreaching",
"EvaluateLowSampleCountPercentile": ""
}
}

import boto3
import json
import logging
import os
import datetime
import calendar

from base64 import b64decode
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError


# The base-64 encoded, encrypted key (CiphertextBlob) stored in the kmsEncryptedHookUrl environment variable
ENCRYPTED_HOOK_URL = os.environ['kmsEncryptedHookUrl']
# The Slack channel to send a message to stored in the slackChannel environment variable
SLACK_CHANNEL = os.environ['slackChannel']

HOOK_URL = "https://" + boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED_HOOK_URL))['Plaintext'].decode('utf-8')

logger = logging.getLogger()
logger.setLevel(logging.INFO)

FILTER_PATTERN='{ $.log_level = "ALERT" }'
OUTPUT_LIMIT=5
TIME_FROM_MIN=5

def lambda_handler(event, context):
    logger.info("Event: " + str(event))
    message = json.loads(event['Records'][0]['Sns']['Message'])
    logger.info("Message: " + str(message))

    alarm_name = message['AlarmName']
    #old_state = message['OldStateValue']
    new_state = message['NewStateValue']
    reason = message['NewStateReason']
    metric = message['Trigger']['MetricName']
    metricNamespace = message['Trigger']['Namespace']

    timeto = datetime.datetime.strptime(message['StateChangeTime'][:19] ,'%Y-%m-%dT%H:%M:%S') + datetime.timedelta(minutes=1)
    u_to = calendar.timegm(timeto.utctimetuple()) * 1000
    timefrom = timeto - datetime.timedelta(minutes=TIME_FROM_MIN)
    u_from = calendar.timegm(timefrom.utctimetuple()) * 1000
    logs = boto3.client('logs')
    metricfilters = logs.describe_metric_filters(
            metricName = message['Trigger']['MetricName'] ,
            metricNamespace = message['Trigger']['Namespace']
    )
    response = logs.filter_log_events(
            logGroupName = metricfilters['metricFilters'][0]['logGroupName'] ,
            filterPattern = metricfilters['metricFilters'][0]['filterPattern'],
            startTime = u_from,
            endTime = u_to,
            limit = OUTPUT_LIMIT
        )

    log_events = response['events']
    for e in log_events:
        date = datetime.datetime.fromtimestamp(int(str(e['timestamp'])[:10])) + datetime.timedelta(hours=9)
        log_message = '{"Timestamp":"' + str(date) + '","Message":' + e['message'] + '}'

    slack_message = {
        'channel': SLACK_CHANNEL,
        'text': "%s state is now %s: %s\n%s" % (alarm_name, new_state, reason, log_message)
    }

    req = Request(HOOK_URL, json.dumps(slack_message).encode('utf-8'))
    try:
        response = urlopen(req)
        response.read()
        logger.info("Message posted to %s", slack_message['channel'])
    except HTTPError as e:
        logger.error("Request failed: %d %s", e.code, e.reason)
    except URLError as e:
        logger.error("Server connection failed: %s", e.reason)

INSUFFICIENT DATAが出るとき

Errorのしきい値は>=1にする
default Valueを0にする
0でも1でもない値が大量に存在しているためOKに戻らない

KMSで

クラウドウォッチログ読み取り権限を付与する

cloudwatchのログの確認の仕方