AWS CodeBuildをパブリックサブネットで起動しようとしたが…

TL;DR

codebuildでVPC_CLIENT_ERRORがでるならプライベートサブネットで起動しないとだめ

再現

まずは環境を作る

variables.tf

variable "ip_range" {
  type        = string
  default     = "192.168.0.0/24"
  description = ""
}

variable "private_ip" {
  type        = string
  default     = "192.168.0.10"
  description = ""
}

variable "bucket" {
  type        = string
  default     = "some_example_bucket_name"
  description = ""
}

main.tf

resource "aws_vpc" "sandbox" {
  cidr_block = var.ip_range
}

data "aws_availability_zones" "all" {}

locals {
  sorted_availability_zones = sort(data.aws_availability_zones.all.names)
  selected_availability_zones = tolist([
    local.sorted_availability_zones[0],
    local.sorted_availability_zones[1],
  ])
}

resource "aws_subnet" "public" {
  vpc_id            = aws_vpc.sandbox.id
  availability_zone = local.selected_availability_zones[0]
  cidr_block        = cidrsubnet(var.ip_range, 1, 0)
}

resource "aws_internet_gateway" "sandbox" {
  vpc_id = aws_vpc.sandbox.id
}

resource "aws_route_table" "public" {
  vpc_id = aws_vpc.sandbox.id
}

resource "aws_route" "public" {
  destination_cidr_block = "0.0.0.0/0"
  route_table_id         = aws_route_table.public.id
  gateway_id             = aws_internet_gateway.sandbox.id
}

resource "aws_route_table_association" "public" {
  subnet_id      = aws_subnet.public.id
  route_table_id = aws_route_table.public.id
}

resource "aws_subnet" "private" {
  vpc_id            = aws_vpc.sandbox.id
  availability_zone = local.selected_availability_zones[1]
  cidr_block        = cidrsubnet(var.ip_range, 1, 1)
}

resource "aws_eip" "nat" {
  domain            = "vpc"
}

resource "aws_nat_gateway" "sandbox" {
  subnet_id = aws_subnet.public.id
  allocation_id = aws_eip.nat.id
}

resource "aws_route_table" "private" {
  vpc_id = aws_vpc.sandbox.id
}

resource "aws_route" "private" {
  destination_cidr_block = "0.0.0.0/0"
  route_table_id         = aws_route_table.private.id
  gateway_id             = aws_nat_gateway.sandbox.id
}

resource "aws_route_table_association" "private" {
  subnet_id      = aws_subnet.private.id
  route_table_id = aws_route_table.private.id
}

```iam
data "aws_iam_policy_document" "codebuild" {
  statement {
    effect = "Allow"

    principals {
      type        = "Service"
      identifiers = ["codebuild.amazonaws.com"]
    }

    actions = ["sts:AssumeRole"]
  }
}

data "aws_iam_policy_document" "metric" {
  statement {
    effect = "Allow"

    actions = [
      "codebuild:CreateReportGroup",
      "codebuild:CreateReport",
      "codebuild:UpdateReport",
      "codebuild:BatchPutTestCases",
      "codebuild:BatchPutCodeCoverages"
    ]

    resources = [
      "*"
    ]
  }
  statement {
    effect = "Allow"

    actions = [
      "logs:CreateLogGroup",
      "logs:CreateLogStream",
      "logs:PutLogEvents"
    ]

    resources = [
      "arn:aws:logs:${local.region}:${local.aid}:log-group:/aws/codebuild/${aws_codebuild_project.codebuild.name}",
      "arn:aws:logs:${local.region}:${local.aid}:log-group:/aws/codebuild/${aws_codebuild_project.codebuild.name}:*"
    ]
  }

}


data "aws_iam_policy_document" "network" {
  statement {
    effect = "Allow"

    actions = [
      "ec2:CreateNetworkInterface",
      "ec2:DescribeDhcpOptions",
      "ec2:DescribeNetworkInterfaces",
      "ec2:DeleteNetworkInterface",
      "ec2:DescribeSubnets",
      "ec2:DescribeSecurityGroups",
      "ec2:DescribeVpcs"
    ]

    resources = ["*"]
  }

  statement {
    effect = "Allow"

    actions = [
      "ec2:CreateNetworkInterfacePermission"
    ]

    resources = [
      "arn:aws:ec2:${local.region}:${local.aid}:network-interface/*"
    ]
    condition {
      test = "StringEquals"
      values = [
        aws_subnet.public.arn,
        aws_subnet.private.arn,
      ]
      variable = "ec2:Subnet"
    }
    condition {
      test = "StringEquals"
      values = [
        "codebuild.amazonaws.com"
      ]
      variable = "ec2:AuthorizedService"
    }
  }
}

data "aws_iam_policy_document" "lambda" {
  statement {
    effect = "Allow"

    principals {
      type        = "Service"
      identifiers = ["lambda.amazonaws.com"]
    }

    actions = ["sts:AssumeRole"]
  }
}

resource "aws_iam_role" "codebuild" {
  name               = "codebuild"
  assume_role_policy = data.aws_iam_policy_document.codebuild.json
}

resource "aws_iam_role_policy" "metric" {
  role   = aws_iam_role.codebuild.name
  policy = data.aws_iam_policy_document.metric.json
}

resource "aws_iam_role_policy" "network" {
  for_each = toset([
    aws_iam_role.codebuild.name,
    aws_iam_role.lambda.name,
  ])
  role   = each.value
  policy = data.aws_iam_policy_document.network.json
}

resource "aws_iam_role" "lambda" {
  name               = "lambda"
  assume_role_policy = data.aws_iam_policy_document.lambda.json
}

resource "aws_iam_role_policy_attachment" "lambda" {
  role       = aws_iam_role.lambda.name
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

そしてlambda

lambda.tf

data "archive_file" "lambda" {
  type        = "zip"
  source_file = "handler.py"
  output_path = "handler.zip"
}

resource "aws_lambda_function" "lambda" {
  filename      = "handler.zip"
   function_name = "ExampleLambda"

  role          = aws_iam_role.lambda.arn
  architectures = ["x86_64"]
  runtime       = "python3.9"
  handler       = "handler.handler"

  vpc_config {
    subnet_ids = [aws_subnet.public.id]
    security_group_ids = [
      aws_vpc.sandbox.default_security_group_id,
    ]
  }

  memory_size = 256
  timeout     = 30
}

lambdaの中身はこんな感じ

handler.py

import json
import urllib.request

url = "http://httpbin.org/ip"
req = urllib.request.Request(url)


def main():
    with urllib.request.urlopen(req, timeout=10) as res:
        body = res.read().decode()
        return json.loads(body)


def handler(event, context):
    return main()


if __name__ == "__main__":
    print(main())

実験

返ってこなくなる

START RequestId: 323b5f40-9a32-470e-9c0d-1fb2669edaea Version: $LATEST
[ERROR] URLError: <urlopen error timed out>
Traceback (most recent call last):
  File "/var/task/handler.py", line 15, in handler
    return main()
  File "/var/task/handler.py", line 9, in main
    with urllib.request.urlopen(req,timeout=10) as res:
  File "/var/lang/lib/python3.9/urllib/request.py", line 214, in urlopen
    return opener.open(url, data, timeout)
  File "/var/lang/lib/python3.9/urllib/request.py", line 517, in open
    response = self._open(req, data)
  File "/var/lang/lib/python3.9/urllib/request.py", line 534, in _open
    result = self._call_chain(self.handle_open, protocol, protocol +
  File "/var/lang/lib/python3.9/urllib/request.py", line 494, in _call_chain
    result = func(*args)
  File "/var/lang/lib/python3.9/urllib/request.py", line 1375, in http_open
    return self.do_open(http.client.HTTPConnection, req)
  File "/var/lang/lib/python3.9/urllib/request.py", line 1349, in do_open
    raise URLError(err)END RequestId: 323b5f40-9a32-470e-9c0d-1fb2669edaea
REPORT RequestId: 323b5f40-9a32-470e-9c0d-1fb2669edaea	Duration: 20027.89 ms	Billed Duration: 20028 ms	Memory Size: 256 MB	Max Memory Used: 41 MB	Init Duration: 132.33 ms

セキュリティーグループやルーティングの設定はあっている

lambdaを動かすために必要なこと

色々調べるとENIにパブリックアドレスが必要なことが分かった

terraformでENIにパブリックアドレスを与えてあげる

data "aws_network_interface" "lambda" {
  depends_on = [aws_lambda_function.lambda]

  filter {
    name   = "interface-type"
    values = ["lambda"]
  }
}

resource "aws_eip" "lambda" {
  domain            = "vpc"
  network_interface = data.aws_network_interface.lambda.id
}

すると動くようになった。

コスト的にもNAT gateway使うより安上がり

同じ要領でcodebuildもできるかと思ったが

codebuildがnat gatewayを必要な理由

codebuildでeniをいじることはできない！

おとなしくcodebuildをVPC内で使うならプライベートサブネットに置く

data "aws_caller_identity" "current" {}
data "aws_region" "current" {}

locals {
  aid    = data.aws_caller_identity.current.account_id
  region = data.aws_region.current.name
}

resource "aws_codebuild_project" "codebuild" {
  name         = "ExampleBuild"
  service_role = aws_iam_role.codebuild.arn

  artifacts {
    type = "NO_ARTIFACTS"
  }

  environment {
    compute_type                = "BUILD_GENERAL1_SMALL"
    image                       = "aws/codebuild/amazonlinux2-x86_64-standard:4.0"
    type                        = "LINUX_CONTAINER"
    image_pull_credentials_type = "CODEBUILD"
  }

  source {
    type     = "NO_SOURCE"
    buildspec = file("./buildspec.yml")
  }

  vpc_config {
    vpc_id = aws_vpc.sandbox.id

    subnets = [
      aws_subnet.private.id,
    ]

    security_group_ids = [
      aws_vpc.sandbox.default_security_group_id,
    ]
  }
}

指定されたサブネット ID subnet-0c50e6b2503dd05fe がパブリックであるため、IDが vpc-0c2642250f48d89f4 である VPC が、インターネットに接続していない可能性があります。ターゲット NAT ゲートウェイの 0.0.0.0/0 送信先があるプライベートサブネットを指定して、もう一度やり直してください。

SUBMITTED
成功
<1 sec 8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00) 8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00)
QUEUED
成功
1 sec 8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00) 8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00)8月 21, 2023 5:37 午後 (UTC+9:00)
PROVISIONING
クライアントエラー
VPC_CLIENT_ERROR: Unexpected EC2 error: UnauthorizedOperation

参考

重要
VPC で CodeBuild を使用するには、NAT ゲートウェイまたは NAT インスタンスが必要です。これにより、CodeBuild はパブリックエンドポイントにアクセスできるようになります (ビルドの実行時に CLI コマンドを実行する場合など)。CodeBuild は、作成したネットワークインターフェイスへの Elastic IP アドレスの割り当てをサポートしていないため、NAT ゲートウェイや NAT インスタンスの代わりにインターネットゲートウェイを使用することはできません。また、Amazon EC2 は、 Amazon EC2 インスタンスの起動以外で作成されたネットワークインターフェイスに対しては、パブリック IP アドレスの自動割り当てをサポートしていません。

eniをいじれないのでパブリック IP アドレスの自動割り当てをサポートしていないのですね。