BeeGFSを最小構成で構築しよう
date: 2024/04/14
Version
Ubuntu: 22.04.4 LTS
BeeGFS: 7.4.3
Architecture
node | ID | IP | role | storageID |
---|---|---|---|---|
node1 | 01 | 172.16.0.101 | management and monitering | |
node2 | 02 | 172.16.0.102 | metadata | |
node3 | 172.16.0.103 | client | ||
node4 | 100 | 172.16.0.104 | storage | 1000 |
node5 | 172.16.0.105 | moniteringDB and moniteringUI |
Install
各ノード共通操作
$ sudo apt update && sudo apt upgrade -y
$ sudo wget -P /etc/apt/sources.list.d/ https://www.beegfs.io/release/beegfs_7.4.3/dists/beegfs-jammy.list
$ sudo wget -P /etc/apt/keyrings/ https://www.beegfs.io/release/beegfs_7.4.3/gpg/GPG-KEY-beegfs
$ tee diff.patch << "EOF"
--- beegfs-jammy.list 2024-03-21 19:35:36.000000000 +0900
+++ /etc/apt/sources.list.d/beegfs-jammy.list 2024-04-14 00:49:43.260670659 +0900
@@ -2,4 +2,4 @@
# to retrieve early updates. Replace username/password with your account for
# the BeeGFS customer login area.
#deb https://username:password@www.beegfs.io/login/release/beegfs_7.4.3 jammy non-free
-deb https://www.beegfs.io/release/beegfs_7.4.3 jammy non-free
+deb [signed-by=/etc/apt/keyrings/GPG-KEY-beegfs] https://www.beegfs.io/release/beegfs_7.4.3 jammy non-free
EOF
$ sudo patch -u -f /etc/apt/sources.list.d/beegfs-jammy.list < diff.patch
$ sudo apt update
node1
$ curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
$ echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
$ echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
$ sudo apt update
$ sudo apt install -y beegfs-mgmtd beegfs-mon telegraf
node2
$ curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
$ echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
$ echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
$ sudo apt update
$ sudo apt install -y beegfs-meta telegraf
node3
$ sudo apt install -y beegfs-client beegfs-helperd beegfs-utils
node4
$ curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
$ echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
$ echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
$ sudo apt update
$ sudo apt install -y beegfs-storage telegraf
node5
$ sudo apt install ca-certificates curl gnupg lsb-release
$ sudo mkdir -p /etc/apt/keyrings
$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
$ sudo chmod a+r /etc/apt/keyrings/docker.gpg
$ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
$ sudo apt update
$ sudo apt install -y beegfs-mon-grafana docker-ce docker-ce-cli containerd.io
Construction
node1
共有秘密鍵を作成し、共有
$ sudo dd if=/dev/random of=/etc/beegfs/connauthfile bs=128 count=1
$ sudo chmod 400 /etc/beegfs/connauthfile
$ scp /etc/beegfs/connauthfile 172.16.0.102:~
$ scp /etc/beegfs/connauthfile 172.16.0.103:~
$ scp /etc/beegfs/connauthfile 172.16.0.104:~
$ sudo chown root:root /etc/beegfs/connauthfile
configの設定と起動
$ sudo mkdir -p /beegfs/beegfs_mgmtd
$ sudo /opt/beegfs/sbin/beegfs-setup-mgmtd -p /beegfs/beegfs_mgmtd
$ tee config.patch << "EOF"
--- beegfs-mgmtd.conf 2024-04-14 01:41:06.079813566 +0900
+++ /etc/beegfs/beegfs-mgmtd.conf 2024-04-14 01:41:40.852984444 +0900
@@ -25,7 +25,7 @@
# --- Section 1.2: [Advanced Settings] ---
#
-connAuthFile =
+connAuthFile = /etc/beegfs/connauthfile
connDisableAuthentication = false
connBacklogTCP = 128
connInterfacesFile =
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-mgmtd.conf < config.patch
$ sudo systemctl start beegfs-mgmtd
node2
configの設定と起動
$ sudo mv connauthfile /etc/beegfs/
$ sudo chown root:root /etc/beegfs/connauthfile
$ sudo mkdir -p /beegfs/beegfs-meta
$ sudo /opt/beegfs/sbin/beegfs-setup-meta -p /beegfs/beegfs-meta/ -s 2 -m 172.16.0.101
$ tee config.patch << "EOF"
--- /etc/beegfs/beegfs-meta.conf 2024-04-14 01:54:09.510481368 +0900
+++ beegfs-meta.conf 2024-04-14 02:00:32.387115009 +0900
@@ -25,7 +25,7 @@
# --- Section 1.2: [Advanced Settings] ---
#
-connAuthFile =
+connAuthFile = /etc/beegfs/connauthfile
connDisableAuthentication = false
connBacklogTCP = 128
connFallbackExpirationSecs = 900
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-meta.conf < config.patch
$ sudo systemctl start beegfs-meta
node3
configの設定と起動
$ sudo mv connauthfile /etc/beegfs/
$ sudo /opt/beegfs/sbin/beegfs-setup-client -m 172.16.0.101
$ tee client-config.patch << "EOF"
--- beegfs-client.conf 2024-04-14 02:14:22.258770847 +0900
+++ /etc/beegfs/beegfs-client.conf 2024-04-14 02:14:51.831192792 +0900
@@ -21,7 +21,7 @@
# --- Section 1.2: [Advanced Settings] ---
#
-connAuthFile =
+connAuthFile = /etc/beegfs/connauthfile
connDisableAuthentication = false
connClientPortUDP = 8004
connHelperdPortTCP = 8006
@@ -55,7 +55,8 @@
quotaEnabled = false
sysCreateHardlinksAsSymlinks = false
-sysMountSanityCheckMS = 11000
+#sysMountSanityCheckMS = 11000
+sysMountSanityCheckMS = 0
sysSessionCheckOnClose = false
sysSyncOnClose = false
sysTargetOfflineTimeoutSecs = 900
EOF
$ tee helper-config.patch << "EOF"
--- beegfs-helperd.conf 2024-04-14 02:11:31.076766643 +0900
+++ /etc/beegfs/beegfs-helperd.conf 2024-04-14 02:10:03.983960396 +0900
@@ -14,7 +14,7 @@
# --- Section 1: [Settings] ---
#
-connAuthFile =
+connAuthFile = /etc/beegfs/connauthfile
connDisableAuthentication = false
connHelperdPortTCP = 8006
connPortShift = 0
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-client.conf < client-config.patch
$ sudo patch -u -f /etc/beegfs/beegfs-helperd.conf < helper-config.patch
$ sudo systemctl start beegfs-helperd
$ sudo systemctl start beegfs-client
node4
configの設定と起動
$ sudo mv connauthfile /etc/beegfs/
$ sudo chown root:root /etc/beegfs/connauthfile
$ sudo mkdir -p /beegfs/beegfs-storage
$ sudo /opt/beegfs/sbin/beegfs-setup-storage -i 1000 -s 100 -m 172.16.0.101 -p /beegfs/beegfs-storage/
$ tee config.patch << "EOF"
--- /etc/beegfs/beegfs-storage.conf 2024-04-14 02:23:02.003654373 +0900
+++ beegfs-storage.conf 2024-04-14 02:23:54.208972205 +0900
@@ -25,7 +25,7 @@
# --- Section 1.2: [Advanced Settings] ---
#
-connAuthFile =
+connAuthFile = /etc/beegfs/connauthfile
connDisableAuthentication = false
connBacklogTCP = 128
connInterfacesFile =
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-storage.conf < config.patch
$ sudo systemctl start beegfs-storage
ストレージの利用 clientでのみ可能
$ mv file /mnt/beegfs/
サーバ状況の確認コマンド clientでのみ可能(beegfs-utilsをインストールしたnodeでのみ可能)
$ beegfs-check-servers # 各nodeの状況が確認できる
$ beegfs-ctl --listtargets --longnodes --state --spaceinfo # storageNodeの状況が確認できる
$ beegfs-ctl --getentryinfo /mnt/beegfs # storageの状況が確認できる
システム監視
node5
$ sudo docker run -d -p 8086:8086 \
-v "$PWD/data:/var/lib/influxdb2" \
-v "$PWD/config:/etc/influxdb2" \
-e DOCKER_INFLUXDB_INIT_MODE=setup \
-e DOCKER_INFLUXDB_INIT_USERNAME=<USERNAME> \
-e DOCKER_INFLUXDB_INIT_PASSWORD=<PASSWORD> \
-e DOCKER_INFLUXDB_INIT_ORG=<ORG_NAME> \
-e DOCKER_INFLUXDB_INIT_BUCKET=<BUCKET_NAME> \
influxdb:2
$ sudo docker run -d --name=grafana -p 3000:3000 grafana/grafana
http://172.16.0.105:8006
でinfluxdbのTokenを入手する。新しいものを作成しても良い。
http://172.16.0.105:3000
でDataSource
の画面でinfluxdbを登録する。URIをlocalhostにはしないほうが良い。localhostで通信したけらばおそらくDocker周りの設定を弄れば可能。
node1
下記で作成しているconfig.patchとauth.patchは元の設定ファイルをコピーし忘れたので、うろ覚えで元のファイル作成してdiffを取ったもの。ERRORを読めば対処可能と思う。
また、ここで設定するUSERNAME
とPASSWORD
,ORG_NAME
,TOKEN
はinfluxDBで作成、取得したもの。
$ sudo tee config.patch << "EOF"
--- beegfs-mon.conf 2024-04-16 09:41:28.460602527 +0900
+++ /etc/beegfs/beegfs-mon.conf 2024-04-14 04:07:03.634357379 +0900
@@ -14,25 +14,25 @@
# --- Section 1.1: [Basic Settings] ---
#
-sysMgmtdHost =
+sysMgmtdHost = 172.16.0.101
#
# --- Section 1.2: [Advanced Settings] ---
#
-dbType = influxdb
-dbHostName = localhost
+dbType = influxdb2
+dbHostName = 172.16.0.105
dbHostPort = 8086
dbAuthFile =
# used by influxdb only
-dbDatabase =
+dbDatabase = beegfs_mon
dbMaxPointsPerRequest = 5000
dbSetRetentionPolicy = true
dbRetentionDuration = 1d
# used by influxdb V2 only
-dbBucket =
+dbBucket = beegfs_bucket
# used by cassandra only
cassandraMaxInsertsPerBatch = 25
@@ -52,7 +52,7 @@
connMgmtdPortUDP = 8008
connPortShift = 0
-connAuthFile =
+connAuthFile = /etc/beegfs/connauthfile
connDisableAuthentication = false
connFallbackExpirationSecs = 900
connMaxInternodeNum = 3
EOF
$ sudo tee auth.patch << "EOF"
--- beegfs-mon.auth 2024-04-16 09:41:48.673042844 +0900
+++ /etc/beegfs/beegfs-mon.auth 2024-04-14 04:04:03.444672259 +0900
@@ -1,9 +1,9 @@
# This file configures the credentials needed to connect to your monitoring database instance.
# This currently only works with InfluxDB.
-username =
-password =
+username = <USERNAME>
+password = <PASSWORD>
# used by influxdb V2 only
-organization =
-token =
+organization = <ORG_NAME>
+token = <TOKEN>
EOF
$ sudo patch -u -f /etc/beegfs/beegfs-mon.conf < config.patch
$ sudo patch -u -f /etc/beegfs/beegfs-mon.auth < auth.patch
node5以外の各ノード
"your_~~~"のところは上記でinfluxdbを立てたときにコマンドに入力したものです。
"your_influxdb_token"はinfluxdbのUIから入手してください。
$ sudo tee /etc/telegraf.d/beegfs_mon_telegraf.conf << "EOF"
[[outputs.influxdb_v2]]
urls = ["http://172.16.0.105:8086"] # Replace with the actual InfluxDB URL
token = "your_influxdb_token" # Replace with your InfluxDB 2.x token
organization = "your_organization" # Replace with your InfluxDB 2.x organization
bucket = "your_bucket" # Replace with your InfluxDB 2.x bucket
[[inputs.cpu]]
percpu = true
totalcpu = true
collect_cpu_time = false
report_active = false
core_tags = false
[[inputs.disk]]
ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.diskio]]
[[inputs.mem]]
[[inputs.processes]]
[[inputs.system]]
EOF
$ sudo systemctl start telegraf
influxdbのUIのDataExplorerにデータが来ているのが確認できる。
DataExplorerでscriptが作成できるので、それをGrafanaのDashboardに入れるとDataが表示されるためGrafanaにもデータが来ているのが確認できる。
一応Beegfsが提供しているGrafanaのDashboardをインストールする方法を示す。
しかし、私の環境ではinfluxdbのversionの違いからかNo Data
になった。
node5
$ /opt/beegfs/scripts/grafana/import-dashboards default
参考文献