#AWS上にマスターnodeをマルチゾーン化したKubernetesクラスタを構築する
イメージ図
マスターnodeをマルチゾーン化し、LBで束ねて高可用性なKubernetesクラスタを構築するのを、Terraformで自動化する。
Kubernetesのnodeは、あらかじめkubeadm,kubectl,kubelet,docker-ceをインストールしたイメージを作成しておく。
#作業場所
Teraformを実行する端末は何でもよく、手元のMacやLinux PC、Azure等パブリッククラウドのLinuxインスタンスでも構わない。今回は自宅のLinux PC(Ubuntu 18.04)から実行している。Terraformのバージョンは次の通り。
root@liva-z:~# terraform version
Terraform v0.12.21
#各種tfファイル
main.tf
# AWS Providerの設定
provider "aws" {
region = var.aws_region
}
# 有効なゾーンを問い合わせlocal.all_zonesで参照する
data "aws_availability_zones" "available" {
state = "available"
}
# local変数の設定
locals {
all_zones = data.aws_availability_zones.available.names
}
# sshキーペアの登録
resource "aws_key_pair" "deployer" {
key_name = "${var.cluster_name}-deployer-key"
public_key = file(var.ssh_public_key_file)
}
# セキュリティグループの作成(common)
resource "aws_security_group" "common" {
name = "${var.cluster_name}-common"
description = "cluster common rules"
vpc_id = aws_vpc.vpc.id
tags = map("Name", "${var.cluster_name}-sg-common", )
ingress {
from_port = var.ssh_port
to_port = var.ssh_port
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
ingress {
from_port = 0
to_port = 0
protocol = "-1"
self = true
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
# セキュリティグループの作成(master)
resource "aws_security_group" "master" {
name = "${var.cluster_name}-masters"
description = "cluster masters"
vpc_id = aws_vpc.vpc.id
tags = map("Name", "${var.cluster_name}-sg-master", )
ingress {
from_port = 6443
to_port = 6443
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
}
# VPCの作成
resource "aws_vpc" "vpc" {
cidr_block = var.vpc_cidr
tags = map("Name", "${var.cluster_name}-vpc", )
}
# Subnetの作成(public)
resource "aws_subnet" "public" {
count = length(local.all_zones)
vpc_id = aws_vpc.vpc.id
cidr_block = cidrsubnet(var.vpc_cidr, var.subnet_netmask_bits, var.subnet_offset + count.index)
availability_zone = local.all_zones[count.index]
map_public_ip_on_launch = true
tags = map("Name", "${var.cluster_name}_public_${local.all_zones[count.index]}", )
}
# Subnetの作成(private)
resource "aws_subnet" "private" {
count = length(local.all_zones)
vpc_id = aws_vpc.vpc.id
cidr_block = cidrsubnet(var.vpc_cidr, var.subnet_netmask_bits, var.subnet_offset + length(local.all_zones) + count.index)
availability_zone = local.all_zones[count.index]
map_public_ip_on_launch = false
tags = map("Name", "${var.cluster_name}_private_${local.all_zones[count.index]}", )
}
# eipの作成
resource "aws_eip" "nat" {
vpc = true
}
# インターネットゲートウェイの作成
resource "aws_internet_gateway" "igw" {
vpc_id = aws_vpc.vpc.id
tags = map("Name", "${var.cluster_name}_igw", )
}
# NATゲートウェイの作成
resource "aws_nat_gateway" "ngw" {
allocation_id = aws_eip.nat.id
subnet_id = aws_subnet.public.0.id
tags = map("Name", "${var.cluster_name}_nat_gateway-ngw", )
}
# 以下ルートテーブルの作成
resource "aws_route_table" "public" {
vpc_id = aws_vpc.vpc.id
tags = map("Name", "${var.cluster_name}_public_route_table", )
route {
cidr_block = "0.0.0.0/0"
gateway_id = aws_internet_gateway.igw.id
}
}
resource "aws_route_table" "private" {
vpc_id = aws_vpc.vpc.id
tags = map("Name", "${var.cluster_name}_private_route_table", )
route {
cidr_block = "0.0.0.0/0"
nat_gateway_id = aws_nat_gateway.ngw.id
}
}
resource "aws_route_table_association" "public" {
count = length(local.all_zones)
subnet_id = element(aws_subnet.public.*.id, count.index)
route_table_id = aws_route_table.public.id
}
resource "aws_route_table_association" "private" {
count = length(local.all_zones)
subnet_id = element(aws_subnet.private.*.id, count.index)
route_table_id = aws_route_table.private.id
}
# ロードバランサーの作成
resource "aws_lb" "master" {
name = "${var.cluster_name}-api-lb"
internal = false
load_balancer_type = "network"
subnets = aws_subnet.public.*.id
tags = map("Name", "${var.cluster_name}-master", )
}
# リスナーの作成
resource "aws_lb_listener" "master_api" {
load_balancer_arn = aws_lb.master.arn
port = 6443
protocol = "TCP"
default_action {
target_group_arn = aws_lb_target_group.master_api.arn
type = "forward"
}
}
resource "aws_lb_target_group" "master_api" {
name = "${var.cluster_name}-api"
port = 6443
protocol = "TCP"
vpc_id = aws_vpc.vpc.id
}
# インスタンスのアタッチ
resource "aws_lb_target_group_attachment" "master_api" {
count = length(local.all_zones)
target_group_arn = aws_lb_target_group.master_api.arn
target_id = element(aws_instance.master.*.id, count.index)
port = 6443
}
# Kubernetesマスターノードの作成(ゾーン数分)
resource "aws_instance" "master" {
count = length(local.all_zones)
tags = map("Name", "${var.cluster_name}-master-${count.index + 1}", )
instance_type = "t3.medium"
ami = var.node_image
key_name = aws_key_pair.deployer.key_name
vpc_security_group_ids = [aws_security_group.common.id, aws_security_group.master.id]
availability_zone = local.all_zones[count.index % length(local.all_zones)]
subnet_id = element(aws_subnet.private.*.id, count.index % length(local.all_zones))
associate_public_ip_address = false
ebs_optimized = true
root_block_device {
volume_type = "gp2"
volume_size = 30
}
}
# Kubernetesワーカーノードの作成(6つ)
resource "aws_instance" "workers" {
count = 6
tags = map("Name", "${var.cluster_name}-worker-${count.index + 1}", )
instance_type = "t3.medium"
ami = var.node_image
key_name = aws_key_pair.deployer.key_name
vpc_security_group_ids = [aws_security_group.common.id]
availability_zone = local.all_zones[count.index % length(local.all_zones)]
subnet_id = element(aws_subnet.private.*.id, count.index % length(local.all_zones))
associate_public_ip_address = false
root_block_device {
volume_type = "gp2"
volume_size = 30
}
}
# 踏み台サーバの作成
resource "aws_instance" "bastion" {
tags = map("Name", "${var.cluster_name}-bastion", )
instance_type = "t3.nano"
ami = var.bastion_image
key_name = aws_key_pair.deployer.key_name
vpc_security_group_ids = [aws_security_group.common.id]
availability_zone = local.all_zones[0]
subnet_id = aws_subnet.public.0.id
associate_public_ip_address = true
root_block_device {
volume_type = "gp2"
volume_size = 30
}
}
variables.tf
# 共通設定
variable "cluster_name" {
default = "fabric"
}
variable "ssh_public_key_file" {
default = "~/.ssh/id_rsa.pub"
}
variable "ssh_port" {
default = 22
}
variable "vpc_cidr" {
default = "10.10.0.0/16"
}
variable "subnet_offset" {
default = 0
}
variable "subnet_netmask_bits" {
default = 8
}
# クラウドベンダー依存部分
variable "aws_region" {
default = "ap-northeast-1"
}
variable "node_image" {
default = "ami-05b96e7df14c3437a" # kubernetes構築済みイメージ
}
variable "bastion_image" {
default = "ami-07f4cb4629342979c" # 素のUbuntu 18.04
}
output.tf
output "kubeadm_api" {
value = {
endpoint = aws_lb.master.dns_name
}
}
output "kubernetes_bastion" {
value = {
bastion_ip = aws_instance.bastion.public_ip
}
}
output "kubernetes_master" {
value = {
master_ip = aws_instance.master.*.private_ip
}
}
output "kubernetes_worker" {
value = {
worker_ip = aws_instance.workers.*.private_ip
}
}
Alicloudとの大きな違いは、LBのみだとendpointが作成できない点である。インターネットゲートウェイとNATゲートウェイを設定する必要がある。
#terraform applyを実行
Apply complete! Resources: 37 added, 0 changed, 0 destroyed.
Outputs:
kubeadm_api = {
"endpoint" = "fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com"
}
kubernetes_bastion = {
"bastion_ip" = "54.178.4.137"
}
kubernetes_master = {
"master_ip" = [
"10.10.3.147",
"10.10.4.161",
"10.10.5.111",
]
}
kubernetes_worker = {
"worker_ip" = [
"10.10.3.170",
"10.10.4.192",
"10.10.5.245",
"10.10.3.182",
"10.10.4.246",
"10.10.5.244",
]
}
Alicloudのコンソールでインスタンスを確認。ちゃんとマスターnodeがゾーンで分かれている。
LB(NLB)にマスターnodeが3つぶら下がっているのがわかる。
#Kubernetesの設定
これでインフラが構築できたので、ここからKubernetes環境を設定していく。基本的には[この][link2]ページのMaster nodes:以下を実行すれば良い。
[link2]:https://kruschecompany.com/kubernetes-1-15-whats-new/
あとすべてのnodeであらかじめ、
swapoff -a
export KUBECONFIG=/etc/kubernetes/admin.conf
してある。というかマシンイメージの.bashrcに書いてある。
1つのマスターnodeへ入って
/etc/kubernetes/kubeadm/kubeadm-config.yamlを次の通り記述。
kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: "fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443"
controlPlaneEndpointには、Outputs:のkubeadm_api.endpointを設定する。
kubeadm initを実行。
root@fabric-master-2:~# kubeadm init --config=/etc/kubernetes/kubeadm/kubeadm-config.yaml --upload-certs
- 途中のkubeadmのメッセージは省略 -
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443 --token 6cwazk.a4669gp7f4uj4srh \
--discovery-token-ca-cert-hash sha256:75bf6bfbeb01315f7b4ddcd49e0569bcd0f86bde0a35d40531cf4c221dc2f1e2 \
--control-plane --certificate-key 76e2af2667fdaf453312df5be62ad1622a77d0e3b1d8a9130e37741cbfa72a32
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443 --token 6cwazk.a4669gp7f4uj4srh \
--discovery-token-ca-cert-hash sha256:75bf6bfbeb01315f7b4ddcd49e0569bcd0f86bde0a35d40531cf4c221dc2f1e2
残り2つのマスターnodeでは、kubeadm initの出力にあるcontrol-plane node用のjoinコマンドを実行する。
6つのワーカーnodeでは、worker node用のjoinコマンドをひたすら実行する。
#できた環境を確認する
terraformを実行したLinux PCへ、Kubernetes環境を一式（たぶんkubectlだけで良いと思う）インストール。
最初にkubeadm initしたマスターnodeから/etc/kubernetes/admin.confをコピーする。
これでkubectlが利用できるようになる。
export KUBECONFIG=/etc/kubernetes/admin.conf
を忘れずに。
root@liva-z:fabric-dev# kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
ip-10-10-3-147 Ready master 19m v1.17.2 10.10.3.147 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-3-170 Ready <none> 13m v1.17.2 10.10.3.170 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-3-182 Ready <none> 10m v1.17.2 10.10.3.182 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-4-161 Ready master 16m v1.17.2 10.10.4.161 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-4-192 Ready <none> 11m v1.17.2 10.10.4.192 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-4-246 Ready <none> 9m46s v1.17.2 10.10.4.246 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-5-111 Ready master 14m v1.17.2 10.10.5.111 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-5-244 Ready <none> 8m58s v1.17.2 10.10.5.244 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
ip-10-10-5-245 Ready <none> 11m v1.17.2 10.10.5.245 <none> Ubuntu 18.04.3 LTS 4.15.0-1057-aws docker://18.9.9
マスターnodeが3台、ワーカーnodeが6台確認でき、すべてReadyになっている。
root@liva-z:fabric-dev# kubectl get pod -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system calico-kube-controllers-5c45f5bd9f-ljq95 1/1 Running 0 2m25s 192.168.101.3 ip-10-10-3-147 <none> <none>
kube-system calico-node-6hpbg 1/1 Running 0 2m25s 10.10.3.170 ip-10-10-3-170 <none> <none>
kube-system calico-node-9xmjj 1/1 Running 0 2m25s 10.10.3.182 ip-10-10-3-182 <none> <none>
kube-system calico-node-h5xrp 1/1 Running 0 2m25s 10.10.5.244 ip-10-10-5-244 <none> <none>
kube-system calico-node-jkkmb 1/1 Running 0 2m25s 10.10.4.192 ip-10-10-4-192 <none> <none>
kube-system calico-node-jmxl4 1/1 Running 0 2m25s 10.10.4.161 ip-10-10-4-161 <none> <none>
kube-system calico-node-lc7kt 1/1 Running 0 2m25s 10.10.5.111 ip-10-10-5-111 <none> <none>
kube-system calico-node-njh55 1/1 Running 0 2m25s 10.10.3.147 ip-10-10-3-147 <none> <none>
kube-system calico-node-pzr6q 1/1 Running 0 2m25s 10.10.4.246 ip-10-10-4-246 <none> <none>
kube-system calico-node-sqhc8 1/1 Running 0 2m25s 10.10.5.245 ip-10-10-5-245 <none> <none>
kube-system coredns-6955765f44-bxmtc 1/1 Running 0 19m 192.168.101.2 ip-10-10-3-147 <none> <none>
kube-system coredns-6955765f44-l8rjh 1/1 Running 0 19m 192.168.101.1 ip-10-10-3-147 <none> <none>
kube-system etcd-ip-10-10-3-147 1/1 Running 0 20m 10.10.3.147 ip-10-10-3-147 <none> <none>
kube-system etcd-ip-10-10-4-161 1/1 Running 0 17m 10.10.4.161 ip-10-10-4-161 <none> <none>
kube-system etcd-ip-10-10-5-111 1/1 Running 0 15m 10.10.5.111 ip-10-10-5-111 <none> <none>
kube-system kube-apiserver-ip-10-10-3-147 1/1 Running 0 20m 10.10.3.147 ip-10-10-3-147 <none> <none>
kube-system kube-apiserver-ip-10-10-4-161 1/1 Running 0 17m 10.10.4.161 ip-10-10-4-161 <none> <none>
kube-system kube-apiserver-ip-10-10-5-111 1/1 Running 0 14m 10.10.5.111 ip-10-10-5-111 <none> <none>
kube-system kube-controller-manager-ip-10-10-3-147 1/1 Running 1 20m 10.10.3.147 ip-10-10-3-147 <none> <none>
kube-system kube-controller-manager-ip-10-10-4-161 1/1 Running 0 17m 10.10.4.161 ip-10-10-4-161 <none> <none>
kube-system kube-controller-manager-ip-10-10-5-111 1/1 Running 0 14m 10.10.5.111 ip-10-10-5-111 <none> <none>
kube-system kube-proxy-62zdx 1/1 Running 0 12m 10.10.5.245 ip-10-10-5-245 <none> <none>
kube-system kube-proxy-dqb4v 1/1 Running 0 19m 10.10.3.147 ip-10-10-3-147 <none> <none>
kube-system kube-proxy-ktpt8 1/1 Running 0 14m 10.10.3.170 ip-10-10-3-170 <none> <none>
kube-system kube-proxy-mmfcr 1/1 Running 0 11m 10.10.3.182 ip-10-10-3-182 <none> <none>
kube-system kube-proxy-nqjbj 1/1 Running 0 9m54s 10.10.5.244 ip-10-10-5-244 <none> <none>
kube-system kube-proxy-rjdmw 1/1 Running 0 15m 10.10.5.111 ip-10-10-5-111 <none> <none>
kube-system kube-proxy-sw7pg 1/1 Running 0 10m 10.10.4.246 ip-10-10-4-246 <none> <none>
kube-system kube-proxy-t6km4 1/1 Running 0 17m 10.10.4.161 ip-10-10-4-161 <none> <none>
kube-system kube-proxy-xn2nl 1/1 Running 0 12m 10.10.4.192 ip-10-10-4-192 <none> <none>
kube-system kube-scheduler-ip-10-10-3-147 1/1 Running 1 20m 10.10.3.147 ip-10-10-3-147 <none> <none>
kube-system kube-scheduler-ip-10-10-4-161 1/1 Running 0 17m 10.10.4.161 ip-10-10-4-161 <none> <none>
kube-system kube-scheduler-ip-10-10-5-111 1/1 Running 0 14m 10.10.5.111 ip-10-10-5-111 <none> <none>
kube-system podも問題なく動作している。
eipを3つ取ってゾーン毎にNATゲートウェイへ登録すれば、もっと可用性が上がるだろう（たぶん）。
↓変更例。
main.tf
# eipの作成
resource "aws_eip" "nat" {
count = length(local.all_zones)
vpc = true
}
# NATゲートウェイの作成
resource "aws_nat_gateway" "ngw" {
count = length(local.all_zones)
allocation_id = element(aws_eip.nat.*.id, count.index)
subnet_id = element(aws_subnet.public.*.id, count.index)
}
resource "aws_route_table" "private" {
vpc_id = aws_vpc.vpc.id
tags = map("Name", "${var.cluster_name}_private_route_table", )
route {
cidr_block = "0.0.0.0/0"
nat_gateway_id = aws_nat_gateway.ngw.*.id
}
}
先に[Alicloudの記事][link3]を書いたのでAWSでも同等のことが出来ることを示してみた。デジャヴを感じたかもしれない。。
[link3]:https://qiita.com/settembre21/items/87b4285dc66cafb57cef
Azureでも出来ると思うがマルチゾーンの考え方が異なるので、その辺りのコードが違ってくるであろう。