proxmox5ではcorosyncが最初から入っている。
drbd8とpacemakerも簡単にインストール出来た。
drbd9にしようとするとkernelモジュール作成の必要があり時間がかかるのと、drbdmanageがうまく動かないところで詰まった。
fdisk
fdisk /dev/vdb
n p enter enter enter
t 8e
w
drbd用にボリューム用意
pvcreate /dev/vdb1
vgcreate vg1 /dev/vdb1
lvcreate --name mysql-drbd --size 3.99g vg1
/etc/hosts
192.168.100.55 proxmox55
192.168.100.56 proxmox56
apt update;
apt-get install -y \
drbd-utils pacemaker crmsh
/etc/drbd.d/mysqldrbd.res
resource mysqldrbd {
protocol C;
handlers {
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger; halt -f";
fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
}
startup {
degr-wfc-timeout 120; # 2 minutes.
outdated-wfc-timeout 2; #2 seconds
}
disk {
on-io-error detach;
}
net {
cram-hmac-alg "sha1";
shared-secret "mysqldrbd";
after-sb-0pri disconnect;
after-sb-1pri disconnect;
after-sb-2pri disconnect;
rr-conflict disconnect;
}
syncer {
rate 50M;
al-extents 257;
on-no-data-accessible io-error;
}
on proxmox55 {
device /dev/drbd0;
disk /dev/vg1/mysql-drbd;
address 192.168.100.55:7788;
meta-disk internal;
}
on proxmox56 {
device /dev/drbd0;
disk /dev/vg1/mysql-drbd;
address 192.168.100.56:7788;
meta-disk internal;
}
}
- OS一度再起動して以下実施
modprobe drbd
drbdadm create-md mysqldrbd
# (両方のサーバーでほぼ同時に実行しないとコマンドが完了しない)
service drbd start
# (一台で実施)primaryに指定
drbdadm -- --overwrite-data-of-peer primary all
同期状況の確認(UpToDate/UpToDateになれば同期完了)
drbd-overview
0:mysqldrbd/0 SyncTarget Secondary/Primary Inconsistent/UpToDate
[=>..................] sync'ed: 12.0% (3688284/4185948)K
mkfs.ext4 /dev/drbd0
mkdir /var/lib/mysql
mount /dev/drbd0 /var/lib/mysql/
# primaryをsecondaryに切り替える
umount /var/lib/mysql
drbdadm secondary mysqldrbd
# standby側で実施
drbdadm primary mysqldrbd
mkdir /var/lib/mysql
mount /dev/drbd0 /var/lib/mysql/
# もとに戻す
umount /var/lib/mysql
drbdadm secondary mysqldrbd
drbdadm primary mysqldrbd
corosync pacemaker
corosync-keygen -l
scp /etc/corosync/authkey root@proxmox56:/etc/corosync/
/etc/corosync/corosync.conf
totem {
version: 2
# How long before declaring a token lost (ms)
token: 3000
# How many token retransmits before forming a new configuration
token_retransmits_before_loss_const: 10
# How long to wait for join messages in the membership protocol (ms)
join: 60
# How long to wait for consensus to be achieved before starting a new round of membership configuration (ms)
consensus: 3600
# Turn off the virtual synchrony filter
vsftype: none
# Number of messages that may be sent by one processor on receipt of the token
max_messages: 20
# Limit generated nodeids to 31-bits (positive signed integers)
clear_node_high_bit: yes
# Disable encryption
#secauth: off
secauth: on
# How many threads to use for encryption/decryption
threads: 0
# Optionally assign a fixed node id (integer)
# nodeid: 1234
# This specifies the mode of redundant ring, which may be none, active, or passive.
rrp_mode: none
interface {
# The following values need to be set based on your environment
ringnumber: 0
bindnetaddr: 192.168.100.0
#bindnetaddr: 127.0.0.1
mcastaddr: 226.94.1.1
mcastport: 5405
}
}
amf {
mode: disabled
}
quorum {
# Quorum for the Pacemaker Cluster Resource Manager
provider: corosync_votequorum
expected_votes: 1
}
aisexec {
user: root
group: root
}
logging {
fileline: off
to_stderr: yes
to_logfile: no
to_syslog: yes
syslog_facility: daemon
debug: off
timestamp: on
logger_subsys {
subsys: AMF
debug: off
tags: enter|leave|trace1|trace2|trace3|trace4|trace6
}
}
scp /etc/corosync/corosync.conf root@proxmox56:/etc/corosync/
/etc/default/corosync
START=yes
scp /etc/default/corosync root@proxmox56:/etc/default/
mkdir /etc/corosync/service.d
/etc/corosync/service.d/pcmk
service {
# Load the Pacemaker Cluster Resource Manager
name: pacemaker
ver: 1
}
scp -r /etc/corosync/service.d/ root@proxmox56:/etc/corosync/
crm
crm status
mysql
wget https://github.com/tukiyo/mysql4.0php5.2/releases/download/debian9/opt-mysql4_4.0.30-1_amd64.deb
dpkg -i opt-mysql4_4.0.30-1_amd64.deb
/etc/my.cnf
[mysqld]
socket=/var/run/mysqld/mysqld.sock2
datadir=/opt/mysql/var/
default-character-set=sjis
max_allowed_packet=512M # for insert big data
log=/tmp/query.log
log-long-format
log_slow_queries=/tmp/mysql-slow.log
long_query_time = 1
#log-queries-not-using-indexes
query_cache_limit=16M
query_cache_size=256M
basedir=/opt/mysql
[client]
socket=/var/run/mysqld/mysqld.sock2
scp /etc/my.cnf root@proxmox56:/etc/
mysqlアカウント作成
addgroup --system mysql
adduser --system \
--group \
--no-create-home \
--home /opt/mysql \
--shell /sbin/nologin \
--disabled-password \
--disabled-login \
mysql
chown -R mysql:mysql /var/lib/mysql/
mkdir -p /var/run/mysqld/
chown -R mysql:mysql /var/run/mysqld/
# 以下片方だけで実施
drbdadm primary mysqldrbd
mount /dev/drbd0 /var/lib/mysql
mkdir -p /var/lib/mysql/data
chown -R mysql:mysql /var/lib/mysql/data
chmod 700 /var/lib/mysql/data
# mysql_install_dbが/usr/bin/hostnameを参照しているため
ln -s /bin/hostname /usr/bin/hostname
/opt/mysql/bin/mysql_install_db --no-defaults --datadir=/var/lib/mysql/data --user=mysql
crmshで設定
設定前のためエラー出力を確認
crm_verify -L -V
error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
Errors found during check: config not valid
片方のノードだけで実行
crm configure node proxmox55 attributes standby=off
crm configure node proxmox56 attributes standby=off
crm configure property stonith-enabled=false
crm configure property no-quorum-policy=ignore
crm configure rsc_defaults resource-stickiness=100
crm configure primitive p_drbd_mysqldrbd ocf:linbit:drbd params drbd_resource="mysqldrbd" op monitor interval="15s"
crm configure ms ms_drbd_mysqldrbd p_drbd_mysqldrbd meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
crm configure primitive p_fs_mysqldrbd ocf:heartbeat:Filesystem params device="/dev/drbd0" directory="/var/lib/mysql" fstype="ext4"
crm configure primitive p_ip_mysqldrbd ocf:heartbeat:IPaddr2 params ip="192.168.100.57" cidr_netmask="24" nic="vmbr0" iflabel="0"
crm configure primitive p_mysqldrbd ocf:heartbeat:mysql params binary="/opt/mysql/bin/mysqld_safe" client_binary="/opt/mysql/bin/mysql" config="/etc/my.cnf" datadir="/var/lib/mysql/data" pid="/var/run/mysqld/mysql.pid" socket="/var/run/mysqld/mysqld.sock2" user="mysql" group="mysql" op monitor interval="60s" meta target-role="Started"
crm configure group g_mysqldrbd p_fs_mysqldrbd p_ip_mysqldrbd p_mysqldrbd
crm configure colocation c_mysql_on_drbd inf: g_mysqldrbd ms_drbd_mysqldrbd:Master
crm configure order o_drbd_before_mysql inf: ms_drbd_mysqldrbd:promote g_mysqldrbd:start
sh crm.sh
WARNING: p_drbd_mysqldrbd: default timeout 20s for start is smaller than the advised 240
WARNING: p_drbd_mysqldrbd: default timeout 20s for stop is smaller than the advised 100
WARNING: p_drbd_mysqldrbd: action 'monitor' not found in Resource Agent meta-data
WARNING: p_fs_mysqldrbd: default timeout 20s for start is smaller than the advised 60
WARNING: p_fs_mysqldrbd: default timeout 20s for stop is smaller than the advised 60
出力が何もないことを確認。
crm_verify -L -V
- OS再起動。
状態確認
初回のみdrbd-overviewで同期完了を待つ(1時間はかかる。)
切り替えテスト
# 通常Masterのproxmox55をslaveにして、proxmox56をMasterにする
/usr/sbin/crm node standby proxmox55
# slaveにしたproxmox55をonlineにする
/usr/sbin/crm node online proxmox55