解決したいエラー

VPC 内で ECR のイメージを使って ECS Cluster を起動しようとすると、CannotPullContainerError のようなエラーが頻出する。

原因

ECR からイメージを取得するためには、ECR, S3 に接続する必要があるが、private subnet を使用している場合、これらのリソースにアクセスできず、取得に失敗している。

解決方法

ECR のイメージ取得をインターネット接続を経由して行えるようにする
VPC 内に各リソースを繋ぐトンネルを構築し、そこ経由で ECR のイメージ等が取得できるようにする(VPC Endpoint)

Ref: https://aws.amazon.com/premiumsupport/knowledge-center/ecs-pull-container-api-error-ecr/?nc1=h_ls

2番目の方が綺麗だけれど、Cloud Formation を書くのがめんどくさい。
一度作成したので、使いまわせるようにここに置いておく。

Cloud Formation

VPC

AWSTemplateFormatVersion: "2010-09-09"
Description:
  VPC and Subnet Create

Metadata:
  "AWS::CloudFormation::Interface":
    ParameterGroups:
      - Label:
          default: "Project Name Prefix"
        Parameters:
          - PJPrefix
      - Label:
          default: "Network Configuration"
        Parameters:
          - VPCCIDR
          - PublicSubnetACIDR
          - PublicSubnetCCIDR
          - PrivateSubnetACIDR
          - PrivateSubnetCCIDR
    ParameterLabels:
      VPCCIDR:
        default: "VPC CIDR"
      PublicSubnetACIDR:
        default: "PublicSubnetA CIDR"
      PublicSubnetCCIDR:
        default: "PublicSubnetC CIDR"
      PrivateSubnetACIDR:
        default: "PrivateSubnetA CIDR"
      PrivateSubnetCCIDR:
        default: "PrivateSubnetC CIDR"

# ------------------------------------------------------------#
# Input Parameters
# ------------------------------------------------------------#
Parameters:
  PJPrefix:
    Type: String

  VPCCIDR:
    Type: String
    Default: "10.1.0.0/16"

  PublicSubnetACIDR:
    Type: String
    Default: "10.1.10.0/24"

  PublicSubnetCCIDR:
    Type: String
    Default: "10.1.20.0/24"

  PrivateSubnetACIDR:
    Type: String
    Default: "10.1.100.0/24"

  PrivateSubnetCCIDR:
    Type: String
    Default: "10.1.200.0/24"

Resources:
# ------------------------------------------------------------#
#  VPC
# ------------------------------------------------------------#
# VPC Create
  VPC:
    Type: "AWS::EC2::VPC"
    Properties:
      CidrBlock: !Ref VPCCIDR
      EnableDnsSupport: "true"
      EnableDnsHostnames: "true"
      InstanceTenancy: default
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-vpc"

# InternetGateway Create
  InternetGateway:
    Type: "AWS::EC2::InternetGateway"
    Properties:
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-igw"

# IGW Attach
  InternetGatewayAttachment:
    Type: "AWS::EC2::VPCGatewayAttachment"
    Properties:
      InternetGatewayId: !Ref InternetGateway
      VpcId: !Ref VPC

# ------------------------------------------------------------#
#  Subnet
# ------------------------------------------------------------#
# Public SubnetA Create
  PublicSubnetA:
    Type: "AWS::EC2::Subnet"
    Properties:
      AvailabilityZone: "ap-northeast-1a"
      CidrBlock: !Ref PublicSubnetACIDR
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-public-subnet-a"

# Public SubnetC Create
  PublicSubnetC:
    Type: "AWS::EC2::Subnet"
    Properties:
      AvailabilityZone: "ap-northeast-1c"
      CidrBlock: !Ref PublicSubnetCCIDR
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-public-subnet-c"

# Private SubnetA Create
  PrivateSubnetA:
    Type: "AWS::EC2::Subnet"
    Properties:
      AvailabilityZone: "ap-northeast-1a"
      CidrBlock: !Ref PrivateSubnetACIDR
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-private-subnet-a"

# Private SubnetC Create
  PrivateSubnetC:
    Type: "AWS::EC2::Subnet"
    Properties:
      AvailabilityZone: "ap-northeast-1c"
      CidrBlock: !Ref PrivateSubnetCCIDR
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-private-subnet-c"

# ------------------------------------------------------------#
#  RouteTable
# ------------------------------------------------------------#
# Public RouteTableA Create
  PublicRouteTableA:
    Type: "AWS::EC2::RouteTable"
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-public-route-a"

# Public RouteTableC Create
  PublicRouteTableC:
    Type: "AWS::EC2::RouteTable"
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-public-route-c"

# Private RouteTableA Create
  PrivateRouteTableA:
    Type: "AWS::EC2::RouteTable"
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-private-route-a"

# Private RouteTableC Create
  PrivateRouteTableC:
    Type: "AWS::EC2::RouteTable"
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-private-route-c"

# ------------------------------------------------------------#
# Routing
# ------------------------------------------------------------#
# PublicRouteA Create
  PublicRouteA:
    Type: "AWS::EC2::Route"
    Properties:
      RouteTableId: !Ref PublicRouteTableA
      DestinationCidrBlock: "0.0.0.0/0"
      GatewayId: !Ref InternetGateway

# PublicRouteC Create
  PublicRouteC:
    Type: "AWS::EC2::Route"
    Properties:
      RouteTableId: !Ref PublicRouteTableC
      DestinationCidrBlock: "0.0.0.0/0"
      GatewayId: !Ref InternetGateway

# ------------------------------------------------------------#
# RouteTable Associate
# ------------------------------------------------------------#
# PublicRouteTable Associate SubnetA
  PublicSubnetARouteTableAssociation:
    Type: "AWS::EC2::SubnetRouteTableAssociation"
    Properties:
      SubnetId: !Ref PublicSubnetA
      RouteTableId: !Ref PublicRouteTableA

# PublicRouteTable Associate SubnetC
  PublicSubnetCRouteTableAssociation:
    Type: "AWS::EC2::SubnetRouteTableAssociation"
    Properties:
      SubnetId: !Ref PublicSubnetC
      RouteTableId: !Ref PublicRouteTableC

# PrivateRouteTable Associate SubnetA
  PrivateSubnetARouteTableAssociation:
    Type: "AWS::EC2::SubnetRouteTableAssociation"
    Properties:
      SubnetId: !Ref PrivateSubnetA
      RouteTableId: !Ref PrivateRouteTableA

# PrivateRouteTable Associate SubnetC
  PrivateSubnetCRouteTableAssociation:
    Type: "AWS::EC2::SubnetRouteTableAssociation"
    Properties:
      SubnetId: !Ref PrivateSubnetC
      RouteTableId: !Ref PrivateRouteTableC

  SecurityGroup:
    Type: "AWS::EC2::SecurityGroup"
    Properties:
      VpcId: !Ref VPC
      GroupName: !Sub "${PJPrefix}-lambda-sg"
      GroupDescription: "for biomass sam"

# ------------------------------------------------------------#
# Output Parameters
# ------------------------------------------------------------#
Outputs:
# VPC
  VPC:
    Value: !Ref VPC
    Export:
      Name: !Sub "${PJPrefix}-vpc"

  VPCCIDR:
    Value: !Ref VPCCIDR
    Export:
      Name: !Sub "${PJPrefix}-vpc-cidr"

# Subnet
  PublicSubnetA:
    Value: !Ref PublicSubnetA
    Export:
      Name: !Sub "${PJPrefix}-public-subnet-a"

  PublicSubnetACIDR:
    Value: !Ref PublicSubnetACIDR
    Export:
      Name: !Sub "${PJPrefix}-public-subnet-a-cidr"

  PublicSubnetC:
    Value: !Ref PublicSubnetC
    Export:
      Name: !Sub "${PJPrefix}-public-subnet-c"

  PublicSubnetCCIDR:
    Value: !Ref PublicSubnetCCIDR
    Export:
      Name: !Sub "${PJPrefix}-public-subnet-c-cidr"

  PrivateSubnetA:
    Value: !Ref PrivateSubnetA
    Export:
      Name: !Sub "${PJPrefix}-private-subnet-a"

  PrivateSubnetACIDR:
    Value: !Ref PrivateSubnetACIDR
    Export:
      Name: !Sub "${PJPrefix}-private-subnet-a-cidr"

  PrivateSubnetC:
    Value: !Ref PrivateSubnetC
    Export:
      Name: !Sub "${PJPrefix}-private-subnet-c"

  PrivateSubnetCCIDR:
    Value: !Ref PrivateSubnetCCIDR
    Export:
      Name: !Sub "${PJPrefix}-private-subnet-c-cidr"

# Route
  PublicRouteTableA:
    Value: !Ref PublicRouteTableA
    Export:
      Name: !Sub "${PJPrefix}-public-route-a"

  PublicRouteTableC:
    Value: !Ref PublicRouteTableC
    Export:
      Name: !Sub "${PJPrefix}-public-route-c"

  PrivateRouteTableA:
    Value: !Ref PrivateRouteTableA
    Export:
      Name: !Sub "${PJPrefix}-private-route-a"

  PrivateRouteTableC:
    Value: !Ref PrivateRouteTableC
    Export:
      Name: !Sub "${PJPrefix}-private-route-c"

  SecurityGroup:
    Value: !Ref SecurityGroup
    Export:
      Name: !Sub "${PJPrefix}-lambda-sg"

ECR

AWSTemplateFormatVersion: '2010-09-09'
Description: ECR repository

Parameters:
  PJPrefix:
    Type: String
  ENVPrefix:
    Type: String

Resources:
  ECR:
    Type: AWS::ECR::Repository
    Properties:
      RepositoryName: !Sub ${PJPrefix}-${ENVPrefix}-migration-ecr

Outputs:
  ECR:
    Value: !Ref ECR
    Export:
      Name: !Sub ${PJPrefix}-migration-ECR

ECS Cluster

AWSTemplateFormatVersion: '2010-09-09'
Description: Create ECS Cluster

Parameters:
  PJPrefix:
    Type: String

Resources:
  ECSCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: !Sub ${PJPrefix}
  MigrationSecurityGroup:
    Type: "AWS::EC2::SecurityGroup"
    Properties:
      VpcId: { "Fn::ImportValue": !Sub "${PJPrefix}-vpc" }
      GroupName: !Sub "${PJPrefix}-migration-task-sg"
      GroupDescription: !Sub "${PJPrefix}-migration-task-sg"

Outputs:
  ECSCluster:
    Value: !Ref ECSCluster
    Export:
      Name: !Sub ${PJPrefix}-Cluster
  MigrationSecurityGroup:
    Value: !Ref MigrationSecurityGroup
    Export:
      Name: !Sub ${PJPrefix}-migration-sg

Task Definition

AWSTemplateFormatVersion: '2010-09-09'
Description: Create for RDS migration

Parameters:
  PJPrefix:
    Type: String
  ENVPrefix:
    Type: String
  ImageTag:
    Type: String
  AwsDefaultRegion:
    Type: String
  AwsAccessKeyId:
    Type: String
  AwsSecretAccessKey:
    Type: String

Mappings:
  NodeEnvMap:
    dev:
      VALUE: development
    stg:
      VALUE: production
    prd:
      VALUE: production

Resources:
  LogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub '/ecs/logs/${PJPrefix}-${ENVPrefix}-migration-groups'

  EcsTaskExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service:
                - "ecs-tasks.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
        - arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess
        - arn:aws:iam::aws:policy/SecretsManagerReadWrite

  ECSTask:
    Type: AWS::ECS::TaskDefinition
    Properties:
      ContainerDefinitions:
        - Name: !Sub ${PJPrefix}-ECR
          Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${PJPrefix}-${ENVPrefix}-migration-ecr:${ImageTag}'
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref LogGroup
              awslogs-region: ap-northeast-1
              awslogs-stream-prefix: ecs
          MemoryReservation: 1024
          Cpu: 256
          Environment:
            - Name: DB_HOST
              Value: !Sub  "{{resolve:secretsmanager:${PJPrefix}-rds-${ENVPrefix}:SecretString:host}}"
            - Name: DB_PORT
              Value: !Sub  "{{resolve:secretsmanager:${PJPrefix}-rds-${ENVPrefix}:SecretString:port}}"
            - Name: DB_USER
              Value: !Sub  "{{resolve:secretsmanager:${PJPrefix}-rds-${ENVPrefix}:SecretString:username}}"
            - Name: DB_PASSWORD
              Value: !Sub  "{{resolve:secretsmanager:${PJPrefix}-rds-${ENVPrefix}:SecretString:password}}"
            - Name: DB_DATABASE_NAME
              Value: !Sub  "{{resolve:secretsmanager:${PJPrefix}-rds-${ENVPrefix}:SecretString:dbname}}"
            - Name: DYNAMO_ENDPOINT
              Value: !Sub "http://dynamodb.${AwsDefaultRegion}.amazonaws.com"
            - Name: NODE_ENV
              Value: !FindInMap [NodeEnvMap, !Ref "ENVPrefix", VALUE]
            - Name: AWS_ACCESS_KEY_ID
              Value: !Sub "${AwsAccessKeyId}"
            - Name: AWS_SECRET_ACCESS_KEY
              Value: !Sub "${AwsSecretAccessKey}"
            - Name: AWS_DEFAULT_REGION
              Value: !Sub "${AwsDefaultRegion}"
      Cpu: '256'
      ExecutionRoleArn: !Ref EcsTaskExecutionRole
      Family: !Sub ${PJPrefix}-migration-task
      Memory: 1024
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE

Outputs:
  LogGroup:
    Value: !Ref LogGroup
    Export:
      Name: !Sub ${PJPrefix}-${ENVPrefix}-LogGroup
  ECSTask:
    Value: !Ref ECSTask
    Export:
      Name: !Sub ${PJPrefix}-${ENVPrefix}-ECSTask
  EcsTaskExecutionRole:
    Value: !Ref EcsTaskExecutionRole
    Export:
      Name: !Sub ${PJPrefix}-${ENVPrefix}-EcsTaskExecutionRole

VPC Endpoint

これで、パブリックネットワークを経由せず各リソース間の連携ができるようになる。

AWSTemplateFormatVersion: "2010-09-09"
Description: Create for RDS migration

Parameters:
  PJPrefix:
    Type: String

Resources:
  MigrationEndpointSecurityGroup:
    Type: "AWS::EC2::SecurityGroup"
    Properties:
      VpcId: { "Fn::ImportValue": !Sub "${PJPrefix}-vpc" }
      GroupName: !Sub "${PJPrefix}-migration-vpce-sg"
      GroupDescription: !Sub "${PJPrefix}-migration-vpce-sg"
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: "443"
          ToPort: "443"
          SourceSecurityGroupId: { "Fn::ImportValue": !Sub "${PJPrefix}-migration-sg" }
      Tags:
        - Key: Name
          Value: !Sub "${PJPrefix}-migration-vpce-sg"
  ECRApiEndpoint:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.ecr.api"
      VpcEndpointType: Interface
      PrivateDnsEnabled: true
      VpcId: { "Fn::ImportValue": !Sub "${PJPrefix}-vpc" }
      SubnetIds:
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-subnet-a" }
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-subnet-c" }
      SecurityGroupIds:
        - !Ref MigrationEndpointSecurityGroup
  ECRDkrEndpoint:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.ecr.dkr"
      VpcEndpointType: Interface
      PrivateDnsEnabled: true
      VpcId: { "Fn::ImportValue": !Sub "${PJPrefix}-vpc" }
      SubnetIds:
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-subnet-a" }
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-subnet-c" }
      SecurityGroupIds:
        - !Ref MigrationEndpointSecurityGroup
  S3Endpoint:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.s3"
      VpcId: { "Fn::ImportValue": !Sub "${PJPrefix}-vpc" }
      RouteTableIds:
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-route-a" }
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-route-c" }
  LogsEndpoint:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      ServiceName: !Sub "com.amazonaws.${AWS::Region}.logs"
      VpcEndpointType: Interface
      PrivateDnsEnabled: true
      VpcId: { "Fn::ImportValue": !Sub "${PJPrefix}-vpc" }
      SubnetIds:
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-subnet-a" }
        - { "Fn::ImportValue": !Sub "${PJPrefix}-private-subnet-c" }
      SecurityGroupIds:
        - !Ref MigrationEndpointSecurityGroup
        - { "Fn::ImportValue": !Sub "${PJPrefix}-lambda-sg" }

Outputs:
  MigrationEndpointSecurityGroup:
    Value: !Ref MigrationEndpointSecurityGroup
    Export:
      Name: !Sub "${PJPrefix}-MigrationEndpointSecurityGroup"
  ECRApiEndpoint:
    Value: !Ref ECRApiEndpoint
    Export:
      Name: !Sub "${PJPrefix}-ECRApiEndpoint"
  ECRDkrEndpoint:
    Value: !Ref ECRDkrEndpoint
    Export:
      Name: !Sub "${PJPrefix}-ECRDkrEndpoint"
  S3Endpoint:
    Value: !Ref S3Endpoint
    Export:
      Name: !Sub "${PJPrefix}-S3Endpoint"
  LogsEndpoint:
    Value: !Ref LogsEndpoint
    Export:
      Name: !Sub "${PJPrefix}-LogsEndpoint"