[雑メモ]Cluster Autoscalerデプロイのterraform化

Cluster Autoscalerとは

バージョン、プロバイダー

terraform {
  required_version = "~> 1.2.6"

  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 4.24.0"
    }
    kubernetes = {
      version = "~> 2.12.1"
    }
    kubectl = {
      source  = "gavinbunney/kubectl"
      version = "1.14.0"
    }
  }
}

provider "aws" {
  region = "ap-northeast-1"
}

provider "kubernetes" {
  host                   = data.aws_eks_cluster.cluster.endpoint
  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
  token                  = data.aws_eks_cluster_auth.cluster.token
}

IAM ポリシーとロールを作成する

data "aws_caller_identity" "current" {}

locals {
  eks_cluster_name             = <EKS クラスター名>
  eks_cluster_oidc_issuer_url  = <EKS OpenID Connect プロバイダー URL>
  this_cluster_oidc_issuer_url = replace(local.eks_cluster_oidc_issuer_url, "https://", "")
}

resource "aws_iam_policy" "cluster_autoscaler" {
  name = "cluster-autoscaler-policy"
  policy = templatefile(
    "./cluster-autoscaler-policy.json",
    {
      cluster_name = local.eks_cluster_name
    }
  )
}

data "aws_iam_policy_document" "assume_role_with_oidc" {
  statement {
    effect  = "Allow"
    actions = ["sts:AssumeRoleWithWebIdentity"]
    principals {
      type        = "Federated"
      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.this_cluster_oidc_issuer_url}"]
    }
    condition {
      test     = "StringEquals"
      variable = "${local.this_cluster_oidc_issuer_url}:sub"
      values   = ["system:serviceaccount:kube-system:cluster-autoscaler"]
    }
  }
}

resource "aws_iam_role" "cluster_autoscaler" {
  name               = "cluster-autoscaler-role"
  assume_role_policy = data.aws_iam_policy_document.assume_role_with_oidc.json
}

resource "aws_iam_role_policy_attachment" "cluster_autoscaler" {
  role       = aws_iam_role.cluster_autoscaler.name
  policy_arn = aws_iam_policy.cluster_autoscaler.arn
}

cluster-autoscaler-policy.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "autoscaling:SetDesiredCapacity",
                "autoscaling:TerminateInstanceInAutoScalingGroup"
            ],
            "Resource": "*",
            "Condition": {
                "StringEquals": {
                    "aws:ResourceTag/k8s.io/cluster-autoscaler/${cluster_name}": "owned"
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": [
                "autoscaling:DescribeAutoScalingInstances",
                "autoscaling:DescribeAutoScalingGroups",
                "ec2:DescribeLaunchTemplateVersions",
                "autoscaling:DescribeTags",
                "autoscaling:DescribeLaunchConfigurations"
            ],
            "Resource": "*"
        }
    ]
}

Cluster Autoscalerのデプロイ

data "aws_eks_cluster" "cluster" {
  name = local.eks_cluster_name
}

data "aws_eks_cluster_auth" "cluster" {
  name = local.eks_cluster_name
}

data "kubectl_path_documents" "cluster_autoscaler_manifest" {
  pattern = "./cluster-autoscaler-autodiscover.yaml"
  vars = {
    cluster_name = local.eks_cluster_name
  }
}

resource "kubectl_manifest" "cluster_autoscaler" {
  count     = length(data.kubectl_path_documents.cluster_autoscaler_manifest.documents)
  yaml_body = element(data.kubectl_path_documents.cluster_autoscaler_manifest.documents, count.index)
}

resource "null_resource" "annotate_serviceaccount_cluster_autoscaler" {
  depends_on = [kubectl_manifest.cluster_autoscaler]

  provisioner "local-exec" {
    command = "kubectl annotate serviceaccount cluster-autoscaler -n kube-system eks.amazonaws.com/role-arn=${aws_iam_role.cluster_autoscaler.arn}"
  }
}

resource "null_resource" "patch_deployment_cluster_autoscaler" {
  depends_on = [kubectl_manifest.cluster_autoscaler]

  provisioner "local-exec" {
    command = "kubectl patch deployment cluster-autoscaler -n kube-system -p '{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\": \"false\"}}}}}'"
  }
}

resource "null_resource" "set_image_deployment_cluster_autoscaler" {
  depends_on = [kubectl_manifest.cluster_autoscaler]

  provisioner "local-exec" {
    command = "kubectl set image deployment cluster-autoscaler -n kube-system cluster-autoscaler=k8s.gcr.io/autoscaling/cluster-autoscaler:v1.21.1"
  }
}

cluster-autoscaler-autodiscover.yaml

以下URLから取得。　
https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml

一部定義を変更。

            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/<YOUR CLUSTER NAME>

↓

            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/${cluster_name}
            - --balance-similar-node-groups
            - --skip-nodes-with-system-pods=false

参考にしたサイト

https://github.com/gavinbunney/terraform-provider-kubectl/issues/52
https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/kubectl_path_documents