LoginSignup
1
4

More than 3 years have passed since last update.

@settembre21(アナログ野郎)inAliEaters (Alibaba Cloud Developers meetup)

AWSにKubernetesクラスタをTerraformで構築する（デジャヴ）

Last updated at Posted at 2020-02-27

AWS上にマスターnodeをマルチゾーン化したKubernetesクラスタを構築する

イメージ図
image.png

マスターnodeをマルチゾーン化し、LBで束ねて高可用性なKubernetesクラスタを構築するのを、Terraformで自動化する。
Kubernetesのnodeは、あらかじめkubeadm,kubectl,kubelet,docker-ceをインストールしたイメージを作成しておく。

作業場所

Teraformを実行する端末は何でもよく、手元のMacやLinux PC、Azure等パブリッククラウドのLinuxインスタンスでも構わない。今回は自宅のLinux PC(Ubuntu 18.04)から実行している。Terraformのバージョンは次の通り。

root@liva-z:~# terraform version
Terraform v0.12.21

各種tfファイル

main.tf
# AWS Providerの設定
provider "aws" {
  region = var.aws_region
}

# 有効なゾーンを問い合わせlocal.all_zonesで参照する
data "aws_availability_zones" "available" {
  state = "available"
}

# local変数の設定
locals {
  all_zones        = data.aws_availability_zones.available.names
}

# sshキーペアの登録
resource "aws_key_pair" "deployer" {
  key_name   = "${var.cluster_name}-deployer-key"
  public_key = file(var.ssh_public_key_file)
}

# セキュリティグループの作成(common)
resource "aws_security_group" "common" {
  name        = "${var.cluster_name}-common"
  description = "cluster common rules"
  vpc_id      = aws_vpc.vpc.id
  tags        = map("Name", "${var.cluster_name}-sg-common", )
  ingress {
    from_port   = var.ssh_port
    to_port     = var.ssh_port
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    from_port = 0
    to_port   = 0
    protocol  = "-1"
    self      = true
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

# セキュリティグループの作成(master)
resource "aws_security_group" "master" {
  name        = "${var.cluster_name}-masters"
  description = "cluster masters"
  vpc_id      = aws_vpc.vpc.id
  tags        = map("Name", "${var.cluster_name}-sg-master", )

  ingress {
    from_port   = 6443
    to_port     = 6443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

# VPCの作成
resource "aws_vpc" "vpc" {
  cidr_block = var.vpc_cidr
  tags       = map("Name", "${var.cluster_name}-vpc", )
}

# Subnetの作成(public)
resource "aws_subnet" "public" {
  count                   = length(local.all_zones)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = cidrsubnet(var.vpc_cidr, var.subnet_netmask_bits, var.subnet_offset + count.index)
  availability_zone       = local.all_zones[count.index]
  map_public_ip_on_launch = true
  tags                    = map("Name", "${var.cluster_name}_public_${local.all_zones[count.index]}", )
}

# Subnetの作成(private)
resource "aws_subnet" "private" {
  count                   = length(local.all_zones)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = cidrsubnet(var.vpc_cidr, var.subnet_netmask_bits, var.subnet_offset + length(local.all_zones) + count.index)
  availability_zone       = local.all_zones[count.index]
  map_public_ip_on_launch = false
  tags                    = map("Name", "${var.cluster_name}_private_${local.all_zones[count.index]}", )
}

# eipの作成
resource "aws_eip" "nat" {
  vpc = true
}

# インターネットゲートウェイの作成
resource "aws_internet_gateway" "igw" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_igw", )
}

# NATゲートウェイの作成
resource "aws_nat_gateway" "ngw" {
  allocation_id = aws_eip.nat.id
  subnet_id     = aws_subnet.public.0.id
  tags          = map("Name", "${var.cluster_name}_nat_gateway-ngw", )
}

# 以下ルートテーブルの作成
resource "aws_route_table" "public" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_public_route_table", )

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw.id
  }
}

resource "aws_route_table" "private" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_private_route_table", )

  route {
    cidr_block     = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.ngw.id
  }
}

resource "aws_route_table_association" "public" {
  count          = length(local.all_zones)
  subnet_id      = element(aws_subnet.public.*.id, count.index)
  route_table_id = aws_route_table.public.id
}

resource "aws_route_table_association" "private" {
  count          = length(local.all_zones)
  subnet_id      = element(aws_subnet.private.*.id, count.index)
  route_table_id = aws_route_table.private.id
}

# ロードバランサーの作成
resource "aws_lb" "master" {
  name               = "${var.cluster_name}-api-lb"
  internal           = false
  load_balancer_type = "network"
  subnets            = aws_subnet.public.*.id
  tags               = map("Name", "${var.cluster_name}-master", )
}

# リスナーの作成
resource "aws_lb_listener" "master_api" {
  load_balancer_arn = aws_lb.master.arn
  port              = 6443
  protocol          = "TCP"

  default_action {
    target_group_arn = aws_lb_target_group.master_api.arn
    type             = "forward"
  }
}

resource "aws_lb_target_group" "master_api" {
  name     = "${var.cluster_name}-api"
  port     = 6443
  protocol = "TCP"
  vpc_id   = aws_vpc.vpc.id
}

# インスタンスのアタッチ
resource "aws_lb_target_group_attachment" "master_api" {
  count            = length(local.all_zones)
  target_group_arn = aws_lb_target_group.master_api.arn
  target_id        = element(aws_instance.master.*.id, count.index)
  port             = 6443
}

# Kubernetesマスターノードの作成(ゾーン数分)
resource "aws_instance" "master" {
  count                       = length(local.all_zones)
  tags                        = map("Name", "${var.cluster_name}-master-${count.index + 1}", )
  instance_type               = "t3.medium"
  ami                         = var.node_image
  key_name                    = aws_key_pair.deployer.key_name
  vpc_security_group_ids      = [aws_security_group.common.id, aws_security_group.master.id]
  availability_zone           = local.all_zones[count.index % length(local.all_zones)]
  subnet_id                   = element(aws_subnet.private.*.id, count.index % length(local.all_zones))
  associate_public_ip_address = false
  ebs_optimized               = true
  root_block_device {
    volume_type = "gp2"
    volume_size = 30
  }
}

# Kubernetesワーカーノードの作成(6つ)
resource "aws_instance" "workers" {
  count                       = 6
  tags                        = map("Name", "${var.cluster_name}-worker-${count.index + 1}", )
  instance_type               = "t3.medium"
  ami                         = var.node_image
  key_name                    = aws_key_pair.deployer.key_name
  vpc_security_group_ids      = [aws_security_group.common.id]
  availability_zone           = local.all_zones[count.index % length(local.all_zones)]
  subnet_id                   = element(aws_subnet.private.*.id, count.index % length(local.all_zones))
  associate_public_ip_address = false
  root_block_device {
    volume_type = "gp2"
    volume_size = 30
  }
}

# 踏み台サーバの作成
resource "aws_instance" "bastion" {
  tags                        = map("Name", "${var.cluster_name}-bastion", )
  instance_type               = "t3.nano"
  ami                         = var.bastion_image
  key_name                    = aws_key_pair.deployer.key_name
  vpc_security_group_ids      = [aws_security_group.common.id]
  availability_zone           = local.all_zones[0]
  subnet_id                   = aws_subnet.public.0.id
  associate_public_ip_address = true
  root_block_device {
    volume_type = "gp2"
    volume_size = 30
  }
}
variables.tf
# 共通設定
variable "cluster_name" {
  default = "fabric"
}

variable "ssh_public_key_file" {
  default = "~/.ssh/id_rsa.pub"
}

variable "ssh_port" {
  default = 22
}

variable "vpc_cidr" {
  default = "10.10.0.0/16"
}

variable "subnet_offset" {
  default = 0
}

variable "subnet_netmask_bits" {
  default = 8
}

# クラウドベンダー依存部分
variable "aws_region" {
  default = "ap-northeast-1"
}

variable "node_image" {
  default = "ami-05b96e7df14c3437a"  # kubernetes構築済みイメージ
}

variable "bastion_image" {
  default = "ami-07f4cb4629342979c"  # 素のUbuntu 18.04
}
output.tf
output "kubeadm_api" {
  value = {
    endpoint = aws_lb.master.dns_name
  }
}

output "kubernetes_bastion" {
  value = {
    bastion_ip = aws_instance.bastion.public_ip
  }
}

output "kubernetes_master" {
  value = {
    master_ip = aws_instance.master.*.private_ip
  }
}

output "kubernetes_worker" {
  value = {
    worker_ip = aws_instance.workers.*.private_ip
  }
}

Alicloudとの大きな違いは、LBのみだとendpointが作成できない点である。インターネットゲートウェイとNATゲートウェイを設定する必要がある。

terraform applyを実行

Apply complete! Resources: 37 added, 0 changed, 0 destroyed.

Outputs:

kubeadm_api = {
  "endpoint" = "fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com"
}
kubernetes_bastion = {
  "bastion_ip" = "54.178.4.137"
}
kubernetes_master = {
  "master_ip" = [
    "10.10.3.147",
    "10.10.4.161",
    "10.10.5.111",
  ]
}
kubernetes_worker = {
  "worker_ip" = [
    "10.10.3.170",
    "10.10.4.192",
    "10.10.5.245",
    "10.10.3.182",
    "10.10.4.246",
    "10.10.5.244",
  ]
}

Alicloudのコンソールでインスタンスを確認。ちゃんとマスターnodeがゾーンで分かれている。
image.png

LB(NLB)にマスターnodeが3つぶら下がっているのがわかる。
image.png

Kubernetesの設定

これでインフラが構築できたので、ここからKubernetes環境を設定していく。基本的にはこのページのMaster nodes:以下を実行すれば良い。

あとすべてのnodeであらかじめ、

swapoff -a
export KUBECONFIG=/etc/kubernetes/admin.conf

してある。というかマシンイメージの.bashrcに書いてある。
1つのマスターnodeへ入って
/etc/kubernetes/kubeadm/kubeadm-config.yamlを次の通り記述。

kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: "fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443"

controlPlaneEndpointには、Outputs:のkubeadm_api.endpointを設定する。

kubeadm initを実行。

root@fabric-master-2:~# kubeadm init --config=/etc/kubernetes/kubeadm/kubeadm-config.yaml --upload-certs
- 途中のkubeadmのメッセージは省略 -
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443 --token 6cwazk.a4669gp7f4uj4srh \
    --discovery-token-ca-cert-hash sha256:75bf6bfbeb01315f7b4ddcd49e0569bcd0f86bde0a35d40531cf4c221dc2f1e2 \
    --control-plane --certificate-key 76e2af2667fdaf453312df5be62ad1622a77d0e3b1d8a9130e37741cbfa72a32

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443 --token 6cwazk.a4669gp7f4uj4srh \
    --discovery-token-ca-cert-hash sha256:75bf6bfbeb01315f7b4ddcd49e0569bcd0f86bde0a35d40531cf4c221dc2f1e2

残り2つのマスターnodeでは、kubeadm initの出力にあるcontrol-plane node用のjoinコマンドを実行する。
6つのワーカーnodeでは、worker node用のjoinコマンドをひたすら実行する。

できた環境を確認する

terraformを実行したLinux PCへ、Kubernetes環境を一式（たぶんkubectlだけで良いと思う）インストール。
最初にkubeadm initしたマスターnodeから/etc/kubernetes/admin.confをコピーする。
これでkubectlが利用できるようになる。

export KUBECONFIG=/etc/kubernetes/admin.conf

を忘れずに。

root@liva-z:fabric-dev# kubectl get node -o wide
NAME             STATUS   ROLES    AGE     VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION    CONTAINER-RUNTIME
ip-10-10-3-147   Ready    master   19m     v1.17.2   10.10.3.147   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-3-170   Ready    <none>   13m     v1.17.2   10.10.3.170   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-3-182   Ready    <none>   10m     v1.17.2   10.10.3.182   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-4-161   Ready    master   16m     v1.17.2   10.10.4.161   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-4-192   Ready    <none>   11m     v1.17.2   10.10.4.192   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-4-246   Ready    <none>   9m46s   v1.17.2   10.10.4.246   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-5-111   Ready    master   14m     v1.17.2   10.10.5.111   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-5-244   Ready    <none>   8m58s   v1.17.2   10.10.5.244   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-5-245   Ready    <none>   11m     v1.17.2   10.10.5.245   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9

マスターnodeが3台、ワーカーnodeが6台確認でき、すべてReadyになっている。

root@liva-z:fabric-dev# kubectl get pod -A -o wide
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE     IP              NODE             NOMINATED NODE   READINESS GATES
kube-system   calico-kube-controllers-5c45f5bd9f-ljq95   1/1     Running   0          2m25s   192.168.101.3   ip-10-10-3-147   <none>           <none>
kube-system   calico-node-6hpbg                          1/1     Running   0          2m25s   10.10.3.170     ip-10-10-3-170   <none>           <none>
kube-system   calico-node-9xmjj                          1/1     Running   0          2m25s   10.10.3.182     ip-10-10-3-182   <none>           <none>
kube-system   calico-node-h5xrp                          1/1     Running   0          2m25s   10.10.5.244     ip-10-10-5-244   <none>           <none>
kube-system   calico-node-jkkmb                          1/1     Running   0          2m25s   10.10.4.192     ip-10-10-4-192   <none>           <none>
kube-system   calico-node-jmxl4                          1/1     Running   0          2m25s   10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   calico-node-lc7kt                          1/1     Running   0          2m25s   10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   calico-node-njh55                          1/1     Running   0          2m25s   10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   calico-node-pzr6q                          1/1     Running   0          2m25s   10.10.4.246     ip-10-10-4-246   <none>           <none>
kube-system   calico-node-sqhc8                          1/1     Running   0          2m25s   10.10.5.245     ip-10-10-5-245   <none>           <none>
kube-system   coredns-6955765f44-bxmtc                   1/1     Running   0          19m     192.168.101.2   ip-10-10-3-147   <none>           <none>
kube-system   coredns-6955765f44-l8rjh                   1/1     Running   0          19m     192.168.101.1   ip-10-10-3-147   <none>           <none>
kube-system   etcd-ip-10-10-3-147                        1/1     Running   0          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   etcd-ip-10-10-4-161                        1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   etcd-ip-10-10-5-111                        1/1     Running   0          15m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-apiserver-ip-10-10-3-147              1/1     Running   0          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-apiserver-ip-10-10-4-161              1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-apiserver-ip-10-10-5-111              1/1     Running   0          14m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-controller-manager-ip-10-10-3-147     1/1     Running   1          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-controller-manager-ip-10-10-4-161     1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-controller-manager-ip-10-10-5-111     1/1     Running   0          14m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-proxy-62zdx                           1/1     Running   0          12m     10.10.5.245     ip-10-10-5-245   <none>           <none>
kube-system   kube-proxy-dqb4v                           1/1     Running   0          19m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-proxy-ktpt8                           1/1     Running   0          14m     10.10.3.170     ip-10-10-3-170   <none>           <none>
kube-system   kube-proxy-mmfcr                           1/1     Running   0          11m     10.10.3.182     ip-10-10-3-182   <none>           <none>
kube-system   kube-proxy-nqjbj                           1/1     Running   0          9m54s   10.10.5.244     ip-10-10-5-244   <none>           <none>
kube-system   kube-proxy-rjdmw                           1/1     Running   0          15m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-proxy-sw7pg                           1/1     Running   0          10m     10.10.4.246     ip-10-10-4-246   <none>           <none>
kube-system   kube-proxy-t6km4                           1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-proxy-xn2nl                           1/1     Running   0          12m     10.10.4.192     ip-10-10-4-192   <none>           <none>
kube-system   kube-scheduler-ip-10-10-3-147              1/1     Running   1          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-scheduler-ip-10-10-4-161              1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-scheduler-ip-10-10-5-111              1/1     Running   0          14m     10.10.5.111     ip-10-10-5-111   <none>           <none>

kube-system podも問題なく動作している。
eipを3つ取ってゾーン毎にNATゲートウェイへ登録すれば、もっと可用性が上がるだろう（たぶん）。
↓変更例。

main.tf
# eipの作成
resource "aws_eip" "nat" {
  count = length(local.all_zones)
  vpc   = true
}

# NATゲートウェイの作成
resource "aws_nat_gateway" "ngw" {
  count         = length(local.all_zones)
  allocation_id = element(aws_eip.nat.*.id, count.index)
  subnet_id     = element(aws_subnet.public.*.id, count.index)
}

resource "aws_route_table" "private" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_private_route_table", )
  route {
    cidr_block     = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.ngw.*.id
  }
}

先にAlicloudの記事を書いたのでAWSでも同等のことが出来ることを示してみた。デジャヴを感じたかもしれない。。

Azureでも出来ると思うがマルチゾーンの考え方が異なるので、その辺りのコードが違ってくるであろう。

1
4
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
1
4