KubernetesのMetalLBとBIRD2を接続してIPv4 IPv6対応のLoadBalancerを構築

Last updated at 2025-05-20Posted at 2025-05-20

目的

LinodeのVPSにアタッチしたGlobal IPv6 /56をKubernetesのLoadBalancerで利用できるようにする。

構成概要

Linode VPS

OS: ubuntu 22
Global IPv6 /56 をVPSにアタッチ
（例：2600:3c18:8000:8000::/56）
iptablesで 172.16.0.0/12 fd00::/8 をNAT
BGPで 0.0.0.0/0 ::/0 を AS65001 として広報
BGPの 0.0.0.0/0 ::/0 をkernelに取り込まない
（既存のdefault routeを上書きしないため）
VXLAN over Wireguard で Local BGP Router と接続

Local BGP Router

OS: debian 11
BGPで 172.16.1.0/24 fd00:16::/64 community=65003:1 を AS65002 として広報

Kubernetes

OS: debian 11
Kubernetes: 1.33.1 (実証時)
BGPで 2600:3c18:8000:8000::/56 に community=65003:1 を付与して AS65003 として広報

手順

Linode VPS

1. Install

sudo apt-get install -y resolvconf wireguard iptables networkd-dispatcher openvswitch-switch bird2

2. VXLAN over Wireguard

2-1. 編集

※ 鍵、公開鍵を挿入してください
/etc/netplan/99-wg0.yaml

network:
  version: 2

  tunnels:

    wg0:
      mode: wireguard
      port: 51820
      key: ＜鍵を挿入＞
      addresses:
      - 172.18.1.1/24
      peers:
      - allowed-ips:
        - 172.18.1.129/24
        keys:
          public: ＜公開鍵を挿入＞
        keepalive: 25
      mtu: 1420

    wg0@vxlan0:
      mode: vxlan
      local: 172.18.1.1
      remote: 172.18.1.129
      id: 100
      port: 4789
      link: wg0

  bridges:

    wg0@br0:
      addresses:
      - 172.17.1.1/24
      - fd00:17::1/64
      interfaces:
      - wg0@vxlan0

2-2. 適用する

sudo netplan apply

3. iptables

3-1. 編集

/etc/networkd-dispatcher/routable.d/99-wg0.sh

#!/bin/bash

set -x

if [ "$IFACE" == "wg0" ]; then
        # Refresh
        /etc/networkd-dispatcher/degraded.d/99-wg0.sh
        # IPv4
        iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
        iptables -A FORWARD -i wg0@br0 -o eth0 -j ACCEPT
        iptables -t nat -A POSTROUTING -s 172.16.0.0/12 -o eth0 -j MASQUERADE
        # IPv6
        ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
        ip6tables -A FORWARD -i wg0@br0 -o eth0 -j ACCEPT
        ip6tables -t nat -A POSTROUTING -s fd00::/8 -o eth0 -j MASQUERADE
fi

/etc/networkd-dispatcher/degraded.d/99-wg0.sh

#!/bin/bash

set -x

if [ "$IFACE" == "wg0" ]; then
        # IPv4
        iptables -D FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
        iptables -D FORWARD -i wg0@br0 -o eth0 -j ACCEPT
        iptables -t nat -D POSTROUTING -s 172.16.0.0/12 -o eth0 -j MASQUERADE
        # IPv6
        ip6tables -D FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
        ip6tables -D FORWARD -i wg0@br0 -o eth0 -j ACCEPT
        ip6tables -t nat -D POSTROUTING -s fd00::/8 -o eth0 -j MASQUERADE
fi

3-2. 適用

sudo systemctl enable --now networkd-dispatcher
sudo systemctl restart networkd-dispatcher

4. BIRD2

4-1. バックアップして起動する

sudo cp -p /etc/bird/bird.conf /etc/bird/bird.conf.bak
sudo systemctl enable --now bird
cat <<EOF | sudo tee /etc/sysctl.d/iptables.conf
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1
EOF
sudo sysctl --system

4-2. 編集

/etc/bird/bird.conf

log syslog all;
router id 172.17.1.1;

protocol device {
}

protocol direct {
	disabled;		# Disable by default
	ipv4;			# Connect to default IPv4 table
	ipv6;			# ... and to default IPv6 table
}

filter export_exclude_default4 {
	if net = 0.0.0.0/0 then {
		reject;
	}
	accept;
}
protocol kernel {
	persist;
	ipv4 {
		export filter export_exclude_default4;	# Export to protocol. default is export none
	};
}

filter export_exclude_default6 {
	if net = ::/0 then {
		reject;
	}
	accept;
}
protocol kernel {
	persist;
	ipv6 {
		export filter export_exclude_default6;
	};
}

protocol static static_default {
	ipv4;
	route 0.0.0.0/0 via "lo";
}
protocol static static_default6 {
	ipv6;
	route ::/0 via "lo";
}

protocol bgp k8szone {
	local as 65001;
	neighbor 172.17.1.129 as 65002;
	password "abcdefg1234567890";
	ipv4 {
		import all;
		export where net ~ [ 0.0.0.0/0 ];
	};
	ipv6 {
		import all;
		export where net ~ [ ::/0 ];
	};
}

4-3. 適用

sudo birdc configure

Local BGP Router

1. Install

sudo apt-get install -y resolvconf wireguard iptables networkd-dispatcher openvswitch-switch bird2

2. VXLAN over Wireguard

2-1. 編集

※ LinodeのIP、鍵、公開鍵を挿入してください
/etc/netplan/99-wg0.yaml

network:
  version: 2

  ethernets:

    eth0:
      addresses:
      - 192.168.1.49/24
      routes:
      - to: ＜LinodeのIPを挿入＞
        via: 192.168.1.1

    eth1:
      addresses:
      - 172.16.1.1/24
      - fd00:16::1/64

  tunnels:

    wg0:
      mode: wireguard
      key: ＜鍵を挿入＞
      addresses:
      - 172.18.1.129/24
      peers:
      - endpoint: ＜LinodeのIPを挿入＞:51820
        allowed-ips:
        - 172.18.1.1/24
        keys:
          public: ＜公開鍵を挿入＞
        keepalive: 20
      mtu: 1420

    wg0@vxlan0:
      mode: vxlan
      local: 172.18.1.129
      remote: 172.18.1.1
      id: 100
      port: 4789
      link: wg0

  bridges:

    wg0@br0:
      addresses:
      - 172.17.1.129/24
      - fd00:17::8001/64
      interfaces:
      - wg0@vxlan0
      nameservers:
        addresses:
        - 1.1.1.1
        - 1.0.0.1
        - 2606:4700:4700::1111
        - 2606:4700:4700::1001

2-2. 適用

cat <<EOF | sudo tee /etc/sysctl.d/iptables.conf
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1
EOF
sudo sysctl --system
sudo netplan apply

3. BIRD2

3-1. バックアップして起動する

sudo cp -p /etc/bird/bird.conf /etc/bird/bird.conf.bak
sudo systemctl enable --now bird
cat <<EOF | sudo tee /etc/sysctl.d/iptables.conf
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1
EOF
sudo sysctl --system

3-2. 編集

/etc/bird/bird.conf

log syslog all;
router id 172.17.1.129;

protocol device {
}
protocol direct {
	disabled;
	ipv4;
	ipv6;
}

protocol kernel {
	persist;
	ipv4 {
	      export all;
	};
}
protocol kernel {
	persist;
	ipv6 { export all; };
}

protocol static {
	ipv4;
	route 172.16.1.0/24 via "eth1";
}
protocol static {
	ipv6;
	route fd00:16::/64 via "eth1";
}

filter export_to_linode {
	# Kubernetes MetalLB community
	if (65003, 1) ~ bgp_community then {
		accept;
	}
	# Cluster Node Network
	if net ~ [ 172.16.1.0/24 ] then {
		accept;
	}
	if net ~ [ fd00:16::/64 ] then {
		accept;
	}
	reject;
}
protocol bgp linode {
	local as 65002;
	neighbor 172.17.1.1 as 65001;
	password "abcdefg1234567890";
	ipv4 {
		import all;
		export filter export_to_linode;
	};
	ipv6 {
		import all;
		export filter export_to_linode;
	};
}

template bgp metallb_template {
	local as 65002;
	password "1234567890abcdefg";
	ipv4 {
		import all;
		export all;
	};
	ipv6 {
		import all;
		export all;
	};
}
protocol bgp k8s_w001 from metallb_template {
	neighbor 172.16.1.129 as 65003;
}
protocol bgp k8s_w002 from metallb_template {
	neighbor 172.16.1.130 as 65003;
}

3-3. 適用

sudo birdc configure

Kubernetes

1. Dual Stack Cluster 構築

2. cilium

2-1. CLI をインストール

2-2. デプロイ

cilium install \
        --namespace kube-system \
        --set ipv6.enabled=true

3. MetalLB

3-1. FRR modeでインストール

3-2. 設定

~/metallb-config.yaml

apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  namespace: metallb-system
  name: private-ipv4-pool
spec:
  avoidBuggyIPs: true
  addresses:
  - 172.16.129.0/24
---
apiVersion: metallb.io/v1beta1
kind: BGPAdvertisement
metadata:
  namespace: metallb-system
  name: bgp-advert-v4
spec:
  ipAddressPools:
  - private-ipv4-pool
  aggregationLength: 24
  communities:
  - 65003:1
---
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  namespace: metallb-system
  name: global-ipv6-pool
spec:
  avoidBuggyIPs: true
  addresses:
  - 2600:3c18:8000:8000::/56
---
apiVersion: metallb.io/v1beta1
kind: BGPAdvertisement
metadata:
  namespace: metallb-system
  name: bgp-advert-v6
spec:
  ipAddressPools:
  - global-ipv6-pool
  communities:
  - 65003:1
---
apiVersion: metallb.io/v1beta2
kind: BGPPeer
metadata:
  namespace: metallb-system
  name: peer-k8szone-v4
spec:
  myASN: 65003
  peerASN: 65002
  peerAddress: 172.16.1.1
  password: 1234567890abcdefg
  nodeSelectors:
  - matchExpressions:
    - key: kubernetes.io/hostname
      operator: In
      values:
      - worker001
      - worker002

apply

kubectl -n metallb-system apply -f ~/metallb-config.yaml

You get articles that match your needs
You can efficiently read back useful information
You can use dark theme

What you can do with signing up