SRv6の実験用にClosネットワークを作りたかったので，ついでに記事にしました．

Closネットワークトポロジ

Closネットワークトポロジは，データセンターよく用いられる水平方向のスケーラビリティに優れたネットワークトポロジです．Closトポロジには，Leaf-Spineトポロジなどがあります．この記事は，3層のClosネットワークを，ネームスペースで仮想的なネットワークを作成するツールであるMininetとルーティングプロトコルスイートであるFRRoutingを使って，Closネットワークを構築します．

実行環境

VirtualBoxとVagrantでUbuntuを立てて，そこで動かしました．
基本的には，MininetとFRRoutingをインストールすれば動くと思います．

詳しい環境はGithubのVagrantfileを見てください．また，Githubには今回使用したコードも載せてます．

作りたいネットワーク

今回作成するネットワークは図のような3層のClosネットワークです．BGPをそれぞれのノードで実行します．ルータのアイコンになっているところでは，FRRoutingを動かしています．このネットワークは，SRv6の実験用に構築したのでアドレスの割り当てに違和感があるかも知れませんが問題なく動作します．

実装

今回の実装は以下のようなコードになりましたが，長いので概要だけ説明します．基本的にはMininetのインターフェースで結線して，Vtyshを通してFRRoutingの設定をしています．今回のCLOSネットワークでは，規則的な部分が多いので，ある程度共通した動作はまとめるようにしています．

(MininetとFRRoutingがインストールされていれば，コピペで動くはず．．．)

net_clos.py

from mininet.cli import CLI
from mininet.net import Mininet
from mininet.log import setLogLevel
from mininet.node import Node

daemons = """
zebra=yes
bgpd=yes

vtysh_enable=yes
zebra_options=" -s 90000000 --daemon -A 127.0.0.1"
bgpd_options="   --daemon -A 127.0.0.1"
"""

vtysh = """
hostname {name}
service integrated-vtysh-config
"""

super_spine_conf = """\
enable
configure terminal
router bgp 65000
  bgp router-id {router_id}
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  neighbor CLOS peer-group
  neighbor CLOS remote-as external
  neighbor CLOS capability extended-nexthop
  neighbor CLOS bfd
  neighbor {ss_name}_s1 interface peer-group CLOS
  neighbor {ss_name}_s2 interface peer-group CLOS
  neighbor {ss_name}_s3 interface peer-group CLOS
  neighbor {ss_name}_s4 interface peer-group CLOS
  neighbor {ss_name}_s1 capability extended-nexthop
  neighbor {ss_name}_s2 capability extended-nexthop
  neighbor {ss_name}_s3 capability extended-nexthop
  neighbor {ss_name}_s4 capability extended-nexthop
  address-family ipv6 unicast
    redistribute connected
    neighbor CLOS activate
  exit-address-family
"""

spine_conf = """\
enable
configure terminal
router bgp {as_number}
  bgp router-id {router_id}
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  bgp bestpath as-path multipath-relax
  neighbor CLOS peer-group
  neighbor CLOS remote-as external
  neighbor CLOS bfd
  neighbor CLOS capability extended-nexthop
  neighbor {s_name}_ss1 interface peer-group CLOS
  neighbor {s_name}_ss2 interface peer-group CLOS
  neighbor {s_name}_ss3 interface peer-group CLOS
  neighbor {s_name}_ss4 interface peer-group CLOS
  neighbor {s_name}_{l_name1} interface peer-group CLOS
  neighbor {s_name}_{l_name2} interface peer-group CLOS
  neighbor {s_name}_ss1 capability extended-nexthop
  neighbor {s_name}_ss2 capability extended-nexthop
  neighbor {s_name}_ss3 capability extended-nexthop
  neighbor {s_name}_ss4 capability extended-nexthop
  neighbor {s_name}_{l_name1} capability extended-nexthop
  neighbor {s_name}_{l_name2} capability extended-nexthop
  address-family ipv6 unicast
    redistribute connected
    neighbor CLOS activate
  exit-address-family
"""

leaf_conf = """\
enable
configure terminal
router bgp {as_number}
  bgp router-id {router_id}
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  bgp bestpath as-path multipath-relax
  neighbor CLOS peer-group
  neighbor CLOS remote-as external
  neighbor CLOS bfd
  neighbor CLOS capability extended-nexthop
  neighbor {l_name}_{s_name1} interface peer-group CLOS
  neighbor {l_name}_{s_name2} interface peer-group CLOS
  neighbor {l_name}_{s_name1} capability extended-nexthop
  neighbor {l_name}_{s_name2} capability extended-nexthop
  address-family ipv6 unicast
    redistribute connected
    neighbor CLOS activate
  exit-address-family
"""

no_ipv6_nd = """\
enable
configure terminal
interface {}
no ipv6 nd suppress-ra
"""

class BaseNode(Node):

    def __init__(self, name, **params):
        super().__init__(name, **params)

    def config(self, **params):
        # self.cmd("hostname " + self.name)
        self.cmd("ifconfig lo up")
        self.cmd("sysctl -w net.ipv4.ip_forward=1")
        self.cmd("sysctl -w net.ipv6.conf.all.forwarding=1")


class FRR(BaseNode):
    """FRR Node"""

    PrivateDirs = ["/etc/frr", "/var/run/frr"]

    def __init__(self, name, inNamespace=True, **params):
        params.setdefault("privateDirs", [])
        params["privateDirs"].extend(self.PrivateDirs)
        super().__init__(name, inNamespace=inNamespace, **params)
        
    def config(self, **params):
        super().config(**params)
        self.start_frr_service()

    def start_frr_service(self):
        """start FRR"""
        self.set_conf("/etc/frr/daemons", daemons)
        self.set_conf("/etc/frr/vtysh.conf", vtysh.format(name=self.name))
        self.set_conf("/etc/frr/frr.conf", "")
        print(self.cmd("/usr/lib/frr/frrinit.sh start"))

    def set_conf(self, file, conf):
        """set frr config"""
        self.cmd("""\
cat << 'EOF' | tee {}
{}
EOF""".format(file, conf))

    def vtysh_cmd(self, cmd=""):
        """exec vtysh commands"""
        cmds = cmd.split("\n")
        vtysh_cmd = "vtysh"
        for c in cmds:
            vtysh_cmd += " -c \"{}\"".format(c)
        return self.cmd(vtysh_cmd)
    

class Leaf(FRR):
    pass

class Spine(FRR):
    pass

class SuperSpine(FRR):
    pass


def main():
    setLogLevel("info")
    net = Mininet()

    l1 = net.addHost("l1", cls=Leaf)
    l2 = net.addHost("l2", cls=Leaf)
    l3 = net.addHost("l3", cls=Leaf)
    l4 = net.addHost("l4", cls=Leaf)
    
    s1 = net.addHost("s1", cls=Spine)
    s2 = net.addHost("s2", cls=Spine)
    s3 = net.addHost("s3", cls=Spine)
    s4 = net.addHost("s4", cls=Spine)
    
    ss1 = net.addHost("ss1", cls=SuperSpine)
    ss2 = net.addHost("ss2", cls=SuperSpine)
    ss3 = net.addHost("ss3", cls=SuperSpine)
    ss4 = net.addHost("ss4", cls=SuperSpine)

    # host
    h1 = net.addHost("h1", ip=None)
    h2 = net.addHost("h2", ip=None)
    h3 = net.addHost("h3", ip=None)
    h4 = net.addHost("h4", ip=None)
    h5 = net.addHost("h5", ip=None)
    h6 = net.addHost("h6", ip=None)
    h7 = net.addHost("h7", ip=None)
    h8 = net.addHost("h8", ip=None)
    
    def set_link(n1, n2, block):
        intf1 = str(n1)+"_"+str(n2)
        intf2 = str(n2)+"_"+str(n1)
        net.addLink(n1, n2, intfName1=intf1, intfName2=intf2)
        
        ipv6_1 = "fc00::{}:1:0:0/80".format(block)
        ipv6_2 = "fc00::{}:2:0:0/80".format(block)
        n1.cmd("ip -6 addr add {} dev {}".format(ipv6_1, intf1))
        n2.cmd("ip -6 addr add {} dev {}".format(ipv6_2, intf2))
    
    def set_link_super_spine(block):
        block = int(block)
        sspines = [n for n in net.nameToNode.values() if isinstance(n, SuperSpine)]
        spines = [n for n in net.nameToNode.values() if isinstance(n, Spine)]
        for ss in sspines:
            for s in spines:
                block = block + 1
                set_link(ss, s, block)
    
    def set_link_pod(s1, s2, l1, l2, block):
        block = int(block)
        for s in [s1, s2]:
            for l in [l1, l2]:
                block = block + 1
                set_link(s, l, block)
    
    def set_link_hosts(r, h1, h2, block):
        
        def set_link_host(r, n, block):
            ipv6_1 = "fc00::{}:1:0:0/80".format(block)
            ipv6_2 = "fc00::{}:2:0:0/80".format(block)
            ipv4_1 = "192.168.{}.1/24".format(block)
            ipv4_2 = "192.168.{}.2/24".format(block)
            intf1 = str(r)+"_"+str(n)
            intf2 = str(n)+"_"+str(r)
            net.addLink(r, n, 
                        intfName1=intf1, params1={"ip": ipv4_1},
                        intfName2=intf2, params2={"ip": ipv4_2})
            r.cmd("ip -6 addr add {} dev {}".format(ipv6_1, intf1))
            n.cmd("ip -6 addr add {} dev {}".format(ipv6_2, intf2))
            
            n.cmd("ip route add default dev {} via {}".format(intf2, ipv4_1.split("/")[0]))
            n.cmd("ip -6 route add default dev {} via {}".format(intf2, ipv6_1.split("/")[0]))
            
        set_link_host(r, h1, block)
        set_link_host(r, h2, int(block)+1)
    
    
    set_link_super_spine("300")
    
    set_link_pod(s1, s2, l1, l2, "200")
    set_link_pod(s3, s4, l3, l4, "210")
    
    set_link_hosts(l1, h1, h2, "110")
    set_link_hosts(l2, h3, h4, "120")
    set_link_hosts(l3, h5, h6, "130")
    set_link_hosts(l4, h7, h8, "140")
    
    net.start()
    
    
    
    # frr setup
    def set_frr_superspine(ss, router_id):
        ss.vtysh_cmd(super_spine_conf.format(
            router_id=router_id,
            ss_name=str(ss)
        ))
        
    def set_frr_spine(s, router_id, as_number, l1, l2):
        s.vtysh_cmd(spine_conf.format(
            as_number=as_number,
            router_id=router_id,
            s_name=str(s),
            l_name1=str(l1),
            l_name2=str(l2)
        ))
        
    def set_frr_leaf(l, router_id, as_number, s1, s2):
        l.vtysh_cmd(leaf_conf.format(
            as_number=as_number,
            router_id=router_id,
            l_name=str(l),
            s_name1=str(s1),
            s_name2=str(s2)
        ))
    
    set_frr_superspine(ss1, "1.1.1.1")
    set_frr_superspine(ss2, "1.1.1.2")
    set_frr_superspine(ss3, "1.1.1.3")
    set_frr_superspine(ss4, "1.1.1.4")
    
    set_frr_spine(s1, "2.2.2.1", 65103, l1, l2)
    set_frr_spine(s2, "2.2.2.2", 65104, l1, l2)
    set_frr_spine(s3, "2.2.2.3", 65203, l3, l4)
    set_frr_spine(s4, "2.2.2.4", 65204, l3, l4)
    
    set_frr_leaf(l1, "3.3.3.1", 65101, s1, s2)
    set_frr_leaf(l2, "3.3.3.2", 65102, s1, s2)
    set_frr_leaf(l3, "3.3.3.3", 65201, s3, s4)
    set_frr_leaf(l4, "3.3.3.4", 65202, s3, s4)
    
    
    CLI(net)
    net.stop()


if __name__ == "__main__":
    main()

SuperSpineノードのBGP設定を軽く説明します．まず，BGPでピアのアドレスを指定するのは面倒なので，BGP Unnumberedの機能を使います．GLOSというグループを使ってインターフェースの設定を行います．また，コネクテッドルートを広報するように設定します．

super_spine_conf = """\
enable
configure terminal
router bgp 65000
  bgp router-id {router_id}
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  neighbor CLOS peer-group
  neighbor CLOS remote-as external
  neighbor CLOS capability extended-nexthop
  neighbor CLOS bfd
  neighbor {ss_name}_s1 interface peer-group CLOS
  neighbor {ss_name}_s2 interface peer-group CLOS
  neighbor {ss_name}_s3 interface peer-group CLOS
  neighbor {ss_name}_s4 interface peer-group CLOS
  neighbor {ss_name}_s1 capability extended-nexthop
  neighbor {ss_name}_s2 capability extended-nexthop
  neighbor {ss_name}_s3 capability extended-nexthop
  neighbor {ss_name}_s4 capability extended-nexthop
  address-family ipv6 unicast
    redistribute connected
    neighbor CLOS activate
  exit-address-family
"""

実行

実行です．普通にpythonで実行すれば，ネットワークが構築されます．

$ sudo python3 net_clos.py 
*** Configuring hosts
l1  * Started watchfrr

l2  * Started watchfrr

l3  * Started watchfrr

l4  * Started watchfrr

s1  * Started watchfrr

s2  * Started watchfrr

s3  * Started watchfrr

s4  * Started watchfrr

ss1  * Started watchfrr

ss2  * Started watchfrr

ss3  * Started watchfrr

ss4  * Started watchfrr

h1 h2 h3 h4 h5 h6 h7 h8 
*** Starting controller

*** Starting 0 switches

*** Starting CLI:
mininet>

実行後，Leaf1でルーティングテーブルを確認してみます．多すぎてわかりませんが，BGPのルートがデプロイされています．

$ ip -6 route
fc00::110:1:0:0/96 dev l1_h1 proto kernel metric 256 pref medium
fc00::111:1:0:0/96 dev l1_h2 proto kernel metric 256 pref medium
fc00::120:1:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::121:1:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::130:1:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::131:1:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::140:1:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::141:1:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::201:1:0:0/96 nhid 17 via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 proto bgp metric 20 pref medium
fc00::201:2:0:0/96 dev l1_s1 proto kernel metric 256 pref medium
fc00::202:1:0:0/96 nhid 17 via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 proto bgp metric 20 pref medium
fc00::202:2:0:0/96 nhid 22 proto bgp metric 20 pref medium
        nexthop via fe80::4ff:f2ff:fe6f:d5cf dev l1_s1 weight 1 
        nexthop via fe80::846a:18ff:fe14:a65 dev l1_s2 weight 1 
fc00::203:1:0:0/96 nhid 21 via fe80::846a:18ff:fe14:a65 dev l1_s2 proto bgp metric 20 pref medium

    (省略)

動作確認

動作確認として，H1からH8へPingします．

Pingが通りました．

CPU使用率とメモリ使用量

これぐらいのノードを立てると，どれぐらい負荷がかかるかが気になったので，裏でCPU使用率とメモリ使用量を測ってみました．一応，今回使用した環境です．

	ホスト	ゲスト
OS	Windows 10	Ubuntu 20.04
CPU	Intel(R) Core(TM) i7-11800H	1コア割り当て
Memory	16GB	2G割り当て

CPU使用率とメモリ使用量を1秒毎に取るようなプログラムを動作させて測りました．厳密に測ってないので，参考程度にしてください．Mininet起動 -> Pingを一度実行 -> 終了という流れでの結果です．

やはり，Mininet起動時がだいぶ負荷が高いことがわかります．流石に，1コア割り当てはきつかったようです．．．．
逆にメモリは2Gで十分なようです．CPUは最低2コアぐらいは割り当てたほうが良さそうです．

おわりに

とりあえず，CLOSネットワークを組めたのでこれから色々と遊べそうです．VXLANとかSRv6とかを動作させる予定です．

参考