お手軽に試すには minikube などの手段がありますが、本番環境に近いマルチノードがほしいと思い、構築してみました。
Kubernetes the hard way のように根本を理解するための難しい手順もありますが、簡単に作れる方法で試そうと思います。
ここでは、
Kubernetes Setup Using Ansible and Vagrant
に従って作成します。が、すんなりはいかずいくつかエラーの対処をしました。
また、Ubuntu のバージョンや、各ノードの memory/CPU を変更しています。
実行環境とツール
macOS Catalina 10.15.4
Vagrant 2.2.7
Ansible 2.9.6
クラスタ構成
Ubuntu 18.04
kubernetes v1.18.3
master(x1): memory 1536MB, CPU 4
node(x2): memory 2048MB, CPU 4
構築する
必要なファイルを用意する
./Vagrantfile
IMAGE_NAME = "generic/ubuntu1804"
N = 2
Vagrant.configure("2") do |config|
config.ssh.insert_key = false
config.vm.provider "virtualbox" do |v|
v.cpus = 4
v.gui = false
end
config.vm.define "k8s-master" do |master|
master.vm.box = IMAGE_NAME
master.vm.network "private_network", ip: "192.168.50.10"
master.vm.hostname = "k8s-master"
master.vm.provision "ansible" do |ansible|
ansible.playbook = "kubernetes-setup/master-playbook.yml"
ansible.extra_vars = {
node_ip: "192.168.50.10",
}
end
master.vm.provider "virtualbox" do |v|
v.customize ["modifyvm", :id, "--memory", 1536 ]
end
end
(1..N).each do |i|
config.vm.define "node-#{i}" do |node|
node.vm.box = IMAGE_NAME
node.vm.network "private_network", ip: "192.168.50.#{i + 10}"
node.vm.hostname = "node-#{i}"
node.vm.provision "ansible" do |ansible|
ansible.playbook = "kubernetes-setup/node-playbook.yml"
ansible.extra_vars = {
node_ip: "192.168.50.#{i + 10}",
}
end
node.vm.provider "virtualbox" do |v|
v.customize ["modifyvm", :id, "--memory", 2048 ]
end
end
end
end
./kubernetes-setup/master-playbook.yml
---
- hosts: all
become: true
tasks:
- name: Install packages that allow apt to be used over HTTPS
apt:
name: "{{ packages }}"
state: present
update_cache: yes
vars:
packages:
- apt-transport-https
- ca-certificates
- curl
- gnupg-agent
- software-properties-common
- name: Add an apt signing key for Docker
apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
state: present
- name: Add apt repository for stable version
apt_repository:
repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
state: present
- name: Install docker and its dependecies
apt:
name: "{{ packages }}"
state: present
update_cache: yes
vars:
packages:
- docker-ce
- docker-ce-cli
- containerd.io
notify:
- docker status
- name: Add vagrant user to docker group
user:
name: vagrant
group: docker
- name: Remove swapfile from /etc/fstab
mount:
name: "{{ item }}"
fstype: swap
state: absent
with_items:
- swap
- none
- name: Disable swap
command: swapoff -a
when: ansible_swaptotal_mb > 0
- name: Add an apt signing key for Kubernetes
apt_key:
url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
state: present
- name: Adding apt repository for Kubernetes
apt_repository:
repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
state: present
filename: kubernetes.list
- name: Install Kubernetes binaries
apt:
name: "{{ packages }}"
state: present
update_cache: yes
vars:
packages:
- kubelet
- kubeadm
- kubectl
- name: Configure node ip
lineinfile:
create: yes
path: /etc/default/kubelet
line: KUBELET_EXTRA_ARGS=--node-ip={{ node_ip }}
- name: Restart kubelet
service:
name: kubelet
daemon_reload: yes
state: restarted
- name: Initialize the Kubernetes cluster using kubeadm
command: kubeadm init --apiserver-advertise-address="192.168.50.10" --apiserver-cert-extra-sans="192.168.50.10" --node-name k8s-master --pod-network-cidr=192.168.0.0/16
- name: Setup kubeconfig for vagrant user
command: "{{ item }}"
with_items:
- mkdir -p /home/vagrant/.kube
- cp -i /etc/kubernetes/admin.conf /home/vagrant/.kube/config
- chown vagrant:vagrant /home/vagrant/.kube/config
- name: Install calico pod network
become: false
command: kubectl create -f https://docs.projectcalico.org/manifests/calico.yaml
- name: Generate join command
command: kubeadm token create --print-join-command
register: join_command
- name: Copy join command to local file
become: false
local_action: copy content="{{ join_command.stdout_lines[0] }}" dest="./join-command"
handlers:
- name: docker status
service: name=docker state=started
./kubernetes-setup/node-playbook.yml
---
- hosts: all
become: true
tasks:
- name: Install packages that allow apt to be used over HTTPS
apt:
name: "{{ packages }}"
state: present
update_cache: yes
vars:
packages:
- apt-transport-https
- ca-certificates
- curl
- gnupg-agent
- software-properties-common
- name: Add an apt signing key for Docker
apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
state: present
- name: Add apt repository for stable version
apt_repository:
repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
state: present
- name: Install docker and its dependecies
apt:
name: "{{ packages }}"
state: present
update_cache: yes
vars:
packages:
- docker-ce
- docker-ce-cli
- containerd.io
notify:
- docker status
- name: Add vagrant user to docker group
user:
name: vagrant
group: docker
- name: Remove swapfile from /etc/fstab
mount:
name: "{{ item }}"
fstype: swap
state: absent
with_items:
- swap
- none
- name: Disable swap
command: swapoff -a
when: ansible_swaptotal_mb > 0
- name: Add an apt signing key for Kubernetes
apt_key:
url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
state: present
- name: Adding apt repository for Kubernetes
apt_repository:
repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
state: present
filename: kubernetes.list
- name: Install Kubernetes binaries
apt:
name: "{{ packages }}"
state: present
update_cache: yes
vars:
packages:
- kubelet
- kubeadm
- kubectl
- name: Configure node ip
lineinfile:
create: yes
path: /etc/default/kubelet
line: KUBELET_EXTRA_ARGS=--node-ip={{ node_ip }}
- name: Restart kubelet
service:
name: kubelet
daemon_reload: yes
state: restarted
- name: Copy the join command to server location
copy: src=join-command dest=/tmp/join-command.sh mode=0777
- name: Join the node to cluster
command: sh /tmp/join-command.sh
handlers:
- name: docker status
service: name=docker state=started
実行
$ vagrant up
ログインして kubectl コマンドを打ってみる
% vagrant ssh k8s-master
Last login: Fri May 22 15:18:20 2020 from 10.0.2.2
vagrant@k8s-master:~$ kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master Ready master 13m v1.18.3
node-1 Ready <none> 6m58s v1.18.3
node-2 Ready <none> 2m21s v1.18.3
vagrant@k8s-master:~$ kubectl get pod --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-789f6df884-t476z 1/1 Running 0 12m
kube-system calico-node-8qqpk 1/1 Running 0 7m
kube-system calico-node-jnb8w 1/1 Running 0 2m23s
kube-system calico-node-lss4w 1/1 Running 0 12m
kube-system coredns-66bff467f8-82l2z 1/1 Running 0 12m
kube-system coredns-66bff467f8-sz28s 1/1 Running 0 12m
kube-system etcd-k8s-master 1/1 Running 0 13m
kube-system kube-apiserver-k8s-master 1/1 Running 0 13m
kube-system kube-controller-manager-k8s-master 1/1 Running 1 13m
kube-system kube-proxy-6gtft 1/1 Running 0 2m23s
kube-system kube-proxy-g4dgs 1/1 Running 0 12m
kube-system kube-proxy-j6drw 1/1 Running 0 7m
kube-system kube-scheduler-k8s-master 1/1 Running 2 13m
vagrant@k8s-master:~$
無事動いているようです！
kubeadm 便利..
対応したエラーと詳細
(1) Configure node ip
TASK [Configure node ip] *******************************************************
fatal: [k8s-master]: FAILED! => {"changed": false, "msg": "Destination /etc/default/kubelet does not exist !", "rc": 257}
/etc/default/kubelet が存在しなかった。
/etc/default/kubelet は kubelet の Environment file だそうです。
The kubelet drop-in file for systemd
-> create するように修正
(2) Install calico pod network
TASK [Install calico pod network] **********************************************
fatal: [k8s-master]: FAILED! => {"changed": true, "cmd": ["kubectl", "create", "-f", "https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml"], "delta": "0:00:01.641120", "end": "2020-05-22 14:12:14.999861", "msg": "non-zero return code", "rc": 1, "start": "2020-05-22 14:12:13.358741", "stderr": "unable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"DaemonSet\" in version \"extensions/v1beta1\"\nunable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"Deployment\" in version \"extensions/v1beta1\"", "stderr_lines": ["unable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"DaemonSet\" in version \"extensions/v1beta1\"", "unable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"Deployment\" in version \"extensions/v1beta1\""], "stdout": "configmap/calico-config created\nsecret/calico-etcd-secrets created\nserviceaccount/calico-node created\nserviceaccount/calico-kube-controllers created\nclusterrole.rbac.authorization.k8s.io/calico-kube-controllers created\nclusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created\nclusterrole.rbac.authorization.k8s.io/calico-node created\nclusterrolebinding.rbac.authorization.k8s.io/calico-node created", "stdout_lines": ["configmap/calico-config created", "secret/calico-etcd-secrets created", "serviceaccount/calico-node created", "serviceaccount/calico-kube-controllers created", "clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created", "clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created", "clusterrole.rbac.authorization.k8s.io/calico-node created", "clusterrolebinding.rbac.authorization.k8s.io/calico-node created"]}
どうやら今回の Kubernetes のバージョンで扱えなくなった記述がされているようでした。
apiVersion: extensions/v1beta1 は kuberenetes 1.16 で削除された模様。
Deprecated APIs Removed In 1.16: Here’s What You Need To Know
-> 最新の yaml に変更
(3) Join the node to cluster
TASK [Join the node to cluster] ************************************************
fatal: [node-1]: FAILED! => {"changed": true, "cmd": ["sh", "/tmp/join-command.sh"], "delta": "0:05:08.334251", "end": "2020-05-22 15:06:29.580596", "msg": "non-zero return code", "rc": 1, "start": "2020-05-22 15:01:21.246345", "stderr": "W0522 15:01:21.328886 26793 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.\n\t[WARNING IsDockerSystemdCheck]: detected \"cgroupfs\" as the Docker cgroup driver. The recommended driver is \"systemd\". Please follow the guide at https://kubernetes.io/docs/setup/cri/\nerror execution phase preflight: couldn't validate the identity of the API Server: Get https://192.168.50.10:6443/api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\nTo see the stack trace of this error execute with --v=5 or higher", "stderr_lines": ["W0522 15:01:21.328886 26793 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.", "\t[WARNING IsDockerSystemdCheck]: detected \"cgroupfs\" as the Docker cgroup driver. The recommended driver is \"systemd\". Please follow the guide at https://kubernetes.io/docs/setup/cri/", "error execution phase preflight: couldn't validate the identity of the API Server: Get https://192.168.50.10:6443/api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)", "To see the stack trace of this error execute with --v=5 or higher"], "stdout": "[preflight] Running pre-flight checks", "stdout_lines": ["[preflight] Running pre-flight checks"]}
これはよくわかりませんでした。。
全く同じことをしている方がいらっしゃったので参考にさせて頂きました。
AnsibleとVagrantでKubernetesをセットアップする
-> master-playbook.yml の [Copy join command to local file] を 非 root 権限で実行