BeeGFSを最小構成で構築しよう

date: 2024/04/14

Version

Ubuntu: 22.04.4 LTS
BeeGFS: 7.4.3

Architecture

node	ID	IP	role	storageID
node1	01	172.16.0.101	management and monitering
node2	02	172.16.0.102	metadata
node3		172.16.0.103	client
node4	100	172.16.0.104	storage	1000
node5		172.16.0.105	moniteringDB and moniteringUI

Install

各ノード共通操作

$ sudo apt update && sudo apt upgrade -y
$ sudo wget -P /etc/apt/sources.list.d/ https://www.beegfs.io/release/beegfs_7.4.3/dists/beegfs-jammy.list
$ sudo wget -P /etc/apt/keyrings/ https://www.beegfs.io/release/beegfs_7.4.3/gpg/GPG-KEY-beegfs
$ tee diff.patch << "EOF"
--- beegfs-jammy.list   2024-03-21 19:35:36.000000000 +0900
+++ /etc/apt/sources.list.d/beegfs-jammy.list   2024-04-14 00:49:43.260670659 +0900
@@ -2,4 +2,4 @@
 # to retrieve early updates. Replace username/password with your account for
 # the BeeGFS customer login area.
 #deb  https://username:password@www.beegfs.io/login/release/beegfs_7.4.3 jammy non-free
-deb  https://www.beegfs.io/release/beegfs_7.4.3  jammy non-free
+deb [signed-by=/etc/apt/keyrings/GPG-KEY-beegfs] https://www.beegfs.io/release/beegfs_7.4.3  jammy non-free
EOF
$ sudo patch -u -f /etc/apt/sources.list.d/beegfs-jammy.list < diff.patch
$ sudo apt update

node1

$ curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
$ echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
$ echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
$ sudo apt update
$ sudo apt install -y beegfs-mgmtd beegfs-mon telegraf

node2

$ curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
$ echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
$ echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
$ sudo apt update
$ sudo apt install -y beegfs-meta telegraf

node3

$ sudo apt install -y beegfs-client beegfs-helperd beegfs-utils

node4

$ curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
$ echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
$ echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
$ sudo apt update
$ sudo apt install -y beegfs-storage telegraf

node5

$ sudo apt install ca-certificates curl gnupg lsb-release
$ sudo mkdir -p /etc/apt/keyrings
$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
$ sudo chmod a+r /etc/apt/keyrings/docker.gpg
$ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
$ sudo apt update
$ sudo apt install -y beegfs-mon-grafana docker-ce docker-ce-cli containerd.io

Construction

node1

共有秘密鍵を作成し、共有

$ sudo dd if=/dev/random of=/etc/beegfs/connauthfile bs=128 count=1
$ sudo chmod 400 /etc/beegfs/connauthfile
$ scp /etc/beegfs/connauthfile 172.16.0.102:~
$ scp /etc/beegfs/connauthfile 172.16.0.103:~
$ scp /etc/beegfs/connauthfile 172.16.0.104:~
$ sudo chown root:root /etc/beegfs/connauthfile

configの設定と起動

$ sudo mkdir -p /beegfs/beegfs_mgmtd
$ sudo /opt/beegfs/sbin/beegfs-setup-mgmtd -p /beegfs/beegfs_mgmtd
$ tee config.patch << "EOF"
--- beegfs-mgmtd.conf   2024-04-14 01:41:06.079813566 +0900
+++ /etc/beegfs/beegfs-mgmtd.conf       2024-04-14 01:41:40.852984444 +0900
@@ -25,7 +25,7 @@
 # --- Section 1.2: [Advanced Settings] ---
 #

-connAuthFile                           =
+connAuthFile                           = /etc/beegfs/connauthfile
 connDisableAuthentication              = false
 connBacklogTCP                         = 128
 connInterfacesFile                     =
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-mgmtd.conf < config.patch
$ sudo systemctl start beegfs-mgmtd

node2

configの設定と起動

$ sudo mv connauthfile /etc/beegfs/
$ sudo chown root:root /etc/beegfs/connauthfile
$ sudo mkdir -p /beegfs/beegfs-meta
$ sudo /opt/beegfs/sbin/beegfs-setup-meta -p /beegfs/beegfs-meta/ -s 2 -m 172.16.0.101
$ tee config.patch << "EOF"
--- /etc/beegfs/beegfs-meta.conf        2024-04-14 01:54:09.510481368 +0900
+++ beegfs-meta.conf    2024-04-14 02:00:32.387115009 +0900
@@ -25,7 +25,7 @@
 # --- Section 1.2: [Advanced Settings] ---
 #

-connAuthFile                 =
+connAuthFile                 = /etc/beegfs/connauthfile
 connDisableAuthentication    = false
 connBacklogTCP               = 128
 connFallbackExpirationSecs   = 900
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-meta.conf < config.patch
$ sudo systemctl start beegfs-meta

node3

configの設定と起動

$ sudo mv connauthfile /etc/beegfs/
$ sudo /opt/beegfs/sbin/beegfs-setup-client -m 172.16.0.101
$ tee client-config.patch << "EOF"
--- beegfs-client.conf  2024-04-14 02:14:22.258770847 +0900
+++ /etc/beegfs/beegfs-client.conf      2024-04-14 02:14:51.831192792 +0900
@@ -21,7 +21,7 @@
 # --- Section 1.2: [Advanced Settings] ---
 #

-connAuthFile                  =
+connAuthFile                  = /etc/beegfs/connauthfile
 connDisableAuthentication     = false
 connClientPortUDP             = 8004
 connHelperdPortTCP            = 8006
@@ -55,7 +55,8 @@
 quotaEnabled                  = false

 sysCreateHardlinksAsSymlinks  = false
-sysMountSanityCheckMS         = 11000
+#sysMountSanityCheckMS         = 11000
+sysMountSanityCheckMS         = 0
 sysSessionCheckOnClose        = false
 sysSyncOnClose                = false
 sysTargetOfflineTimeoutSecs   = 900
EOF
$ tee helper-config.patch << "EOF"
--- beegfs-helperd.conf 2024-04-14 02:11:31.076766643 +0900
+++ /etc/beegfs/beegfs-helperd.conf     2024-04-14 02:10:03.983960396 +0900
@@ -14,7 +14,7 @@
 # --- Section 1: [Settings] ---
 #

-connAuthFile       =
+connAuthFile       = /etc/beegfs/connauthfile
 connDisableAuthentication = false
 connHelperdPortTCP = 8006
 connPortShift      = 0
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-client.conf < client-config.patch
$ sudo patch -u -f /etc/beegfs/beegfs-helperd.conf < helper-config.patch
$ sudo systemctl start beegfs-helperd
$ sudo systemctl start beegfs-client

node4

configの設定と起動

$ sudo mv connauthfile /etc/beegfs/
$ sudo chown root:root /etc/beegfs/connauthfile
$ sudo mkdir -p /beegfs/beegfs-storage
$ sudo /opt/beegfs/sbin/beegfs-setup-storage -i 1000 -s 100 -m 172.16.0.101 -p /beegfs/beegfs-storage/
$ tee config.patch << "EOF"
--- /etc/beegfs/beegfs-storage.conf     2024-04-14 02:23:02.003654373 +0900
+++ beegfs-storage.conf 2024-04-14 02:23:54.208972205 +0900
@@ -25,7 +25,7 @@
 # --- Section 1.2: [Advanced Settings] ---
 #

-connAuthFile                 =
+connAuthFile                 = /etc/beegfs/connauthfile
 connDisableAuthentication    = false
 connBacklogTCP               = 128
 connInterfacesFile           =
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-storage.conf < config.patch
$ sudo systemctl start beegfs-storage

ストレージの利用 clientでのみ可能

$ mv file /mnt/beegfs/

サーバ状況の確認コマンド clientでのみ可能(beegfs-utilsをインストールしたnodeでのみ可能)

$ beegfs-check-servers # 各nodeの状況が確認できる
$ beegfs-ctl --listtargets --longnodes --state --spaceinfo # storageNodeの状況が確認できる
$ beegfs-ctl --getentryinfo /mnt/beegfs # storageの状況が確認できる

システム監視

node5

$ sudo docker run -d -p 8086:8086 \
  -v "$PWD/data:/var/lib/influxdb2" \
  -v "$PWD/config:/etc/influxdb2" \
  -e DOCKER_INFLUXDB_INIT_MODE=setup \
  -e DOCKER_INFLUXDB_INIT_USERNAME=<USERNAME> \
  -e DOCKER_INFLUXDB_INIT_PASSWORD=<PASSWORD> \
  -e DOCKER_INFLUXDB_INIT_ORG=<ORG_NAME> \
  -e DOCKER_INFLUXDB_INIT_BUCKET=<BUCKET_NAME> \
  influxdb:2
$ sudo docker run -d --name=grafana -p 3000:3000 grafana/grafana

http://172.16.0.105:8006でinfluxdbのTokenを入手する。新しいものを作成しても良い。
http://172.16.0.105:3000でDataSourceの画面でinfluxdbを登録する。URIをlocalhostにはしないほうが良い。localhostで通信したけらばおそらくDocker周りの設定を弄れば可能。

node1
下記で作成しているconfig.patchとauth.patchは元の設定ファイルをコピーし忘れたので、うろ覚えで元のファイル作成してdiffを取ったもの。ERRORを読めば対処可能と思う。
また、ここで設定するUSERNAMEとPASSWORD,ORG_NAME,TOKENはinfluxDBで作成、取得したもの。

$ sudo tee config.patch << "EOF"
--- beegfs-mon.conf     2024-04-16 09:41:28.460602527 +0900
+++ /etc/beegfs/beegfs-mon.conf 2024-04-14 04:07:03.634357379 +0900
@@ -14,25 +14,25 @@
 # --- Section 1.1: [Basic Settings] ---
 #

-sysMgmtdHost                 =
+sysMgmtdHost                 = 172.16.0.101

 #
 # --- Section 1.2: [Advanced Settings] ---
 #

-dbType                       = influxdb
-dbHostName                   = localhost
+dbType                       = influxdb2
+dbHostName                   = 172.16.0.105
 dbHostPort                   = 8086
 dbAuthFile                   =

 # used by influxdb only
-dbDatabase                   =
+dbDatabase                   = beegfs_mon
 dbMaxPointsPerRequest        = 5000
 dbSetRetentionPolicy         = true
 dbRetentionDuration          = 1d

 # used by influxdb V2 only
-dbBucket                     =
+dbBucket                     = beegfs_bucket

 # used by cassandra only
 cassandraMaxInsertsPerBatch  = 25
@@ -52,7 +52,7 @@
 connMgmtdPortUDP             = 8008
 connPortShift                = 0

-connAuthFile                 =
+connAuthFile                 = /etc/beegfs/connauthfile
 connDisableAuthentication    = false
 connFallbackExpirationSecs   = 900
 connMaxInternodeNum          = 3
EOF
$ sudo tee auth.patch << "EOF"
--- beegfs-mon.auth     2024-04-16 09:41:48.673042844 +0900
+++ /etc/beegfs/beegfs-mon.auth 2024-04-14 04:04:03.444672259 +0900
@@ -1,9 +1,9 @@
 # This file configures the credentials needed to connect to your monitoring database instance.
 # This currently only works with InfluxDB.

-username =
-password =
+username = <USERNAME>
+password = <PASSWORD>

 # used by influxdb V2 only
-organization =
-token =
+organization = <ORG_NAME>
+token = <TOKEN>
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-mon.conf < config.patch
$ sudo patch -u -f /etc/beegfs/beegfs-mon.auth < auth.patch

node5以外の各ノード
"your_~~~"のところは上記でinfluxdbを立てたときにコマンドに入力したものです。
"your_influxdb_token"はinfluxdbのUIから入手してください。

$ sudo tee /etc/telegraf.d/beegfs_mon_telegraf.conf << "EOF"
[[outputs.influxdb_v2]]
urls = ["http://172.16.0.105:8086"] # Replace with the actual InfluxDB URL
token = "your_influxdb_token" # Replace with your InfluxDB 2.x token
organization = "your_organization" # Replace with your InfluxDB 2.x organization
bucket = "your_bucket" # Replace with your InfluxDB 2.x bucket

[[inputs.cpu]]
percpu = true
totalcpu = true
collect_cpu_time = false
report_active = false
core_tags = false

[[inputs.disk]]
ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]

[[inputs.diskio]]
[[inputs.mem]]
[[inputs.processes]]
[[inputs.system]]
EOF
$ sudo systemctl start telegraf

influxdbのUIのDataExplorerにデータが来ているのが確認できる。
DataExplorerでscriptが作成できるので、それをGrafanaのDashboardに入れるとDataが表示されるためGrafanaにもデータが来ているのが確認できる。

一応Beegfsが提供しているGrafanaのDashboardをインストールする方法を示す。
しかし、私の環境ではinfluxdbのversionの違いからかNo Dataになった。

node5

$ /opt/beegfs/scripts/grafana/import-dashboards default

参考文献