はじめに

こんにちわ。Wano株式会社エンジニアのnariと申します。
Terraform記事第三弾ということで、今回はAWS上のPipelineのモニタリング機構に関して記事にしたいと思います。

何を作ったか

AWS CodePipeline,ECS,CodeBuildのステータスが変化すると、slackに投げてくれる仕組みをApex、Terraform)GoでIaC化して作った

全体構成

CodePipeline/Codebuild/ECSのステータスをCloudWatchで監視
Statusの変更があった場合に、Lambdaを起動
Lambda関数でステータスを、Slackへ通知

何故作ったか

Codepipelineの進捗や結果を確認しに行くのにコンソールまで行くのが面倒くさい
buildやdeploy失敗を見逃さないアラートの仕組みが欲しかった

何故Apexなのか

モニタリングリソースが増えるたびにCloudWatchEventsを手動で設定するのが面倒くさい
かといって、lambda関連のリソースをterraformだけで管理するのは辛い(バイナリ化してzip、バージョン管理etc)
Apexならapex infraというterraformコマンドをラップした便利コマンドを使用できる(関数名などをvariableに設定するだけでよしなに補填してくれる)

開発環境

Apex 1.0.0-rc2
Terraform 0.12.0
Provider(aws) 2.12.0
Go 1.12.4

ディレクトリ構成

monitoring/  
    ├ functions/
    |    └ pipeline_notice/
    |       ├ function.json
    |       └ main.go  
    ├ infrastructure/
    |    ├ main.tf
    |    ├ outputs.tf
    |    ├ variables.tf
    |    └ event_pattern/
    |       ├ codepipeline.json
    |       ├ codebuild.json
    |       └ ecs.json
    └ project.json

それぞれのコードのご紹介

Apex

project.json にproject全体の設定

project.json

{
  "name": "hogehoge",
  "description": "hogehoge",
  "nameTemplate": "{{.Project.Name}}_{{.Function.Name}}"
}

functions.jsonにそれぞれの関数の設定を

function.json

{
  "name": "hogehoge",
  "description": "hogehoge",
  "runtime": "go1.x",
  "role": "arn:aws:iam::xxxxxxxxxxxxx:role/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
  "environment": {
    "webHookUrl": "https://hooks.slack.com/services/xxxxxxx/xxxxxxxxx/xxxxxxxxxxxxxxx",
    "slackChannel": "xxxxxxxxxxxxxxxxx"
  },
  "memory": 1024,
  "timeout": 120,
  "handler": "main",
  "hooks": {
    "build": "go get -v -t -d ./... && GOOS=linux GOARCH=amd64 go build -o main main.go",
    "clean": "rm -f main"
  }
}

Go(lambda function)

main.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/aws/aws-lambda-go/events"
	"github.com/aws/aws-lambda-go/lambda"
	"os"
	"strings"
     ...
)

type CodePipelineStatus struct {
	Pipeline    string `json:"pipeline"`
	ExecutionID string `json:"execution-id"`
	State       string `json:"state"`
	Version     int    `json:"version"`
}

type CodeBuildStatus struct {
	BuildStatus string `json:"build-status"`
	ProjectName string `json:"project-name"`
	BuildID     string `json:"build-id"`
}

type EcsStatus struct {
	ClusterArn        string `json:"clusterArn"`
	TaskArn           string `json:"taskArn"`
	TaskDefinitionArn string `json:"taskDefinitionArn"`
	DesiredStatus     string `json:"desiredStatus"`
	LastStatus        string `json:"lastStatus"`
}

type CODEPIPELINE_STATE string

const (
	CODEPIPELINE_STARTED_STATE   CODEPIPELINE_STATE = "STARTED"
	CODEPIPELINE_SUCCEEDED_STATE CODEPIPELINE_STATE = "SUCCEEDED"
	CODEPIPELINE_RESUMED_STATE   CODEPIPELINE_STATE = "RESUMED"
	CODEPIPELINE_FAILED_STATE    CODEPIPELINE_STATE = "FAILED"
	CODEPIPELINE_CANCELED_STATE  CODEPIPELINE_STATE = "CANCELED"
)

type CODEBUILD_STATE string

const (
	CODEBUILD_STOPPED_STATE     CODEBUILD_STATE = "STOPPED"
	CODEBUILD_SUCCEEDED_STATE   CODEBUILD_STATE = "SUCCEEDED"
	CODEBUILD_IN_PROGRESS_STATE CODEBUILD_STATE = "IN_PROGRESS"
	CODEBUILD_FAILED_STATE      CODEBUILD_STATE = "FAILED"
)

type ECS_STATE string

const (
	ECS_RUNNING_STATE ECS_STATE = "RUNNING"
	ECS_STOPPED_STATE ECS_STATE = "STOPPED"
)

const (
	SLACK_ICON = ":crocus:"
	SLACK_NAME = "stagingのdeploypipelineを監視するクロッカス"
)

func main() {
	lambda.Start(noticeHandler)
}

func noticeHandler(context context.Context, event events.CloudWatchEvent) (e error) {
	webhookURL := os.Getenv("webHookUrl")
	channel := os.Getenv("slackChannel")
	switch event.Source {
	case "aws.codepipeline":
		if err := notifyCodePipelineStatus(event, webhookURL, channel); err != nil {
			log.Error(err)
			return err
		}
		return nil
	case "aws.codebuild":
		if err := notifyCodeBuildStatus(event, webhookURL, channel); err != nil {
			log.Error(err)
			return err
		}
		return nil
	case "aws.ecs":
		if err := notifyEcsStatus(event, webhookURL, channel); err != nil {
			log.Error(err)
			return err
		}
		return nil

	default:
		log.Info("想定するリソースのイベントではない")
		return nil
	}
}

func notifyCodePipelineStatus(event events.CloudWatchEvent, webhookURL string, channel string) (e error) {
	status := &CodePipelineStatus{}
	err := json.Unmarshal([]byte(event.Detail), status)
	if err != nil {
		log.Error(err)
		return err
	}

	pipelineURL := fmt.Sprintf("https://%v.console.aws.amazon.com/codesuite/codepipeline/pipelines/%v/executions/%v",
		event.Region, status.Pipeline, status.ExecutionID)

	text := fmt.Sprintf("*execution-id : %v*\n `state:%v` \n 詳細は %v", status.ExecutionID, status.State, pipelineURL)

	var messsageLevel slack_reporter.SLACK_MESSAGE_LEVEL
	var title string
	switch CODEPIPELINE_STATE(status.State) {
	case CODEPIPELINE_STARTED_STATE:
		title = fmt.Sprintf("*%vのpipeline開始*", status.Pipeline)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_NOTIFY
	case CODEPIPELINE_CANCELED_STATE, CODEPIPELINE_RESUMED_STATE:
		title = fmt.Sprintf("*%vが%v*", status.Pipeline, status.State)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_NOTIFY
	case CODEPIPELINE_FAILED_STATE:
		title = fmt.Sprintf("*%vのpipeline失敗。。。*", status.Pipeline)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_ALART
	case CODEPIPELINE_SUCCEEDED_STATE:
		title = fmt.Sprintf("*%vのpipeline成功！！*", status.Pipeline)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_OK
	default:
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_NOTIFY
	}

	err = slack_reporter.ReportToLaboonSlack(webhookURL, channel, SLACK_MONITORING_PIPELINE_NAME, SLACK_MONITORING_PIPELINE_ICON, title, text, messsageLevel)
	if err != nil {
		log.Error(err)
		return err
	}
	return nil
}

func notifyCodeBuildStatus(event events.CloudWatchEvent, webhookURL string, channel string) (e error) {
	status := &CodeBuildStatus{}
	err := json.Unmarshal([]byte(event.Detail), status)
	if err != nil {
		log.Error(err)
		return err
	}
	buildID := strings.Split(status.BuildID, "/")[1]

	//test-ciのbuildは無視
	if status.ProjectName == "hoge-auto-test" {
		return nil
	}

	codebuildURL := fmt.Sprintf("https://%v.console.aws.amazon.com/codebuild/home?%v#/builds/%v/view/new",
		event.Region, event.Region, buildID)

	text := fmt.Sprintf("*build-id : %v*\n `state:%v` \n 詳細は %v", buildID, status.BuildStatus, codebuildURL)

	var title string
	var messsageLevel slack_reporter.SLACK_MESSAGE_LEVEL

	switch CODEBUILD_STATE(status.BuildStatus) {
	case CODEBUILD_IN_PROGRESS_STATE:
		title = fmt.Sprintf("*%vがbuild開始*", status.ProjectName)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_NOTIFY
	case CODEBUILD_FAILED_STATE, CODEBUILD_STOPPED_STATE:
		title = fmt.Sprintf("*%vがbuild失敗か停止した。。。*", status.ProjectName)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_ALART
	case CODEBUILD_SUCCEEDED_STATE:
		title = fmt.Sprintf("*%vがbuild成功*", status.ProjectName)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_OK
	default:
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_NOTIFY
	}

	err = slack_reporter.ReportToLaboonSlack(webhookURL, channel, SLACK_MONITORING_PIPELINE_NAME, SLACK_MONITORING_PIPELINE_ICON, title, text, messsageLevel)
	if err != nil {
		log.Error(err)
		return err
	}
	return nil
}

func notifyEcsStatus(event events.CloudWatchEvent, webhookURL string, channel string) (e error) {
	status := &EcsStatus{}
	err := json.Unmarshal([]byte(event.Detail), status)
	if err != nil {
		log.Error(err)
		return err
	}

	// 期待するステートになるまでの過程は通知しない(予期せぬSTOPPEDを除いて)
	if status.LastStatus != status.DesiredStatus && status.LastStatus != string(ECS_STOPPED_STATE) {
		return nil
	}

	clusterID := strings.Split(status.ClusterArn, "/")[1]
	taskID := strings.Split(status.TaskArn, "/")[1]
	taskDefinitionID := strings.Split(status.TaskDefinitionArn, "/")[1]
	taskURL := fmt.Sprintf("https://%v.console.aws.amazon.com/ecs/home?region=%v#/clusters/%v/tasks/%v/details",
		event.Region, event.Region, clusterID, taskID)
	taskDefinitionURL := fmt.Sprintf("https://%v.console.aws.amazon.com/ecs/home?region=%v#/taskDefinitions/%v",
		event.Region, event.Region, strings.Replace(taskDefinitionID, ":", "/", 1))

	text := fmt.Sprintf("*clusterID:%v* \n `state:%v` \n <%v|Task> (_<%v|%v>_)", clusterID, status.LastStatus, taskURL, taskDefinitionURL, taskDefinitionID)

	var title string
	var messsageLevel slack_reporter.SLACK_MESSAGE_LEVEL

	switch ECS_STATE(status.LastStatus) {
	case ECS_RUNNING_STATE:
		title = fmt.Sprintf("*%vが起動に成功*", taskDefinitionID)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_OK
	case ECS_STOPPED_STATE:
		//もし期待された停止であるならば、OK
		if status.DesiredStatus == string(ECS_STOPPED_STATE) {
			title = fmt.Sprintf("*%vが停止に成功*", taskDefinitionID)
			messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_OK
			break
		}
		//そうでないならアラート
		title = fmt.Sprintf("*%vが予期せぬ停止*", taskDefinitionID)
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_ALART
	default:
		messsageLevel = slack_reporter.SLACK_MESSAGE_LEVEL_NOTIFY
	}

	err = slack_reporter.ReportToLaboonSlack(webhookURL, channel, SLACK_MONITORING_PIPELINE_NAME, SLACK_MONITORING_PIPELINE_ICON, title, text, messsageLevel)
	if err != nil {
		log.Error(err)
		return err
	}
	return nil
}

ポイント

event.Sourceで処理を分岐し、events.CloudWatchEvent(json.RawMessage型)をそれぞれのstatusにunmarshalして扱う(そうすることで、監視するリソース毎に関数を分けない)

Terraform

monitoring module(今回のdirectory構成の外部にあります詳細はこちら

main.tf

/*  monitoring  */
resource "aws_cloudwatch_event_rule" "default" {
  count         = length(var.cloud_watch_event_objs)
  name          = var.cloud_watch_event_objs[count.index]["cloud_watch_event_rule_name"]
  description   = var.cloud_watch_event_objs[count.index]["cloud_watch_event_rule_description"]
  event_pattern = var.cloud_watch_event_objs[count.index]["event_pattern"]
}

resource "aws_cloudwatch_event_target" "default" {
  count     = length(aws_cloudwatch_event_rule.default)
  rule      = aws_cloudwatch_event_rule.default[count.index].name
  target_id = var.cloud_watch_event_objs[count.index]["aws_cloudwatch_event_target_id"]
  arn       = var.cloud_watch_event_objs[count.index]["function_arn"]
}

resource "aws_lambda_permission" "default" {
  count         = length(aws_cloudwatch_event_target.default)
  statement_id  = var.cloud_watch_event_objs[count.index]["statement_id"]
  action        = "lambda:InvokeFunction"
  function_name = aws_cloudwatch_event_target.default[count.index].arn
  principal     = "events.amazonaws.com"
  source_arn    = aws_cloudwatch_event_rule.default[count.index].arn
}

variables.tf

/* required */
variable "cloud_watch_event_objs" {
  //map(string)でもいいんだけどparameterとしてフィールドが見えてる方がわかりやすいかなと
  type = list(object({
    statement_id                       = string
    cloud_watch_event_rule_name        = string
    cloud_watch_event_rule_description = string
    event_pattern                      = string
    aws_cloudwatch_event_target_id     = string
    function_arn                       = string
    })
  )
  description = "watchしたいリソースとイベントパターンとlambda関数を打ち込む"
}

moduleを使う側(main.tf variables.tf)

main.tf

...
/* monitoring  */
module "laboon_monitoring" {
  source                 = "../../../../modules/common/monitoring"
  cloud_watch_event_objs = local.cloud_watch_event_objs
}

/* locals */
locals {
  /* monitoring required */
  cloud_watch_event_objs = [
    //この配列に監視したいリソースとイベントパターンと発火したい関数の入ったobjを追加する
    //codepipeline
    {
      statement_id                       = "AllowExecutionFromCloudWatchForCodePipeline"
      cloud_watch_event_rule_name        = "ad-pipeline-notice"
      cloud_watch_event_rule_description = "ad-pipeline-notice"
      event_pattern                      = file("./event_pattern/codepipeline.json")
      aws_cloudwatch_event_target_id     = "ad-pipeline-notice"
      function_arn                       = var.apex_function_pipeline_notice
    },
    //codebuild
    {
      statement_id                       = "AllowExecutionFromCloudWatchForCodeBuild"
      cloud_watch_event_rule_name        = "ad-codebuild-notice"
      cloud_watch_event_rule_description = "ad-codebuild-notice"
      event_pattern                      = file("./event_pattern/codebuild.json")
      aws_cloudwatch_event_target_id     = "ad-codebuild-notice"
      function_arn                       = var.apex_function_pipeline_notice
    },
    //ecs
    {
      statement_id                       = "AllowExecutionFromCloudWatchForECS"
      cloud_watch_event_rule_name        = "ad-ecs-notice"
      cloud_watch_event_rule_description = "ad-ecs-notice"
      event_pattern                      = file("./event_pattern/ecs.json")
      aws_cloudwatch_event_target_id     = "ad-ecs-notice"
      function_arn                       = var.apex_function_pipeline_notice
    }
  ]

}

variables.tf

variable "apex_function_pipeline_notice" {}
variable "apex_function_pipeline_notice_name" {}
variable "apex_function_names" {}
variable "apex_function_role" {}
variable "aws_region" {}
variable "apex_environment" {}
variable "apex_function_arns" {}

event_pattern/codebuild.json

{
  "source": [
    "aws.codebuild"
  ],
  "detail-type": [
    "CodeBuild Build State Change"
  ]
}

event_pattern/codepipeline.json

{
  "source": [
    "aws.codepipeline"
  ],
  "detail-type": [
    "CodePipeline Pipeline Execution State Change"
  ],
  "detail": {
    "state": [
      "RESUMED",
      "CANCELED",
      "STARTED",
      "FAILED",
      "SUCCEEDED"
    ]
  }
}

event_pattern/ecs.json

{
  "source": [
    "aws.ecs"
  ],
  "detail-type": [
    "ECS Task State Change"
  ]
}

ポイント

list(object)型を用いることで、簡単に監視リソースを増やせるモジュールに

はじめに

何を作ったか

全体構成

何故作ったか

何故Apexなのか

開発環境

ディレクトリ構成

それぞれのコードのご紹介

Apex

Go(lambda function)

ポイント

Terraform

monitoring module(今回のdirectory構成の外部にあります 詳細はこちら

moduleを使う側(main.tf variables.tf)

ポイント

参考文献

monitoring module(今回のdirectory構成の外部にあります詳細はこちら