server1 master server4 高可用
serever 2 3 chunk
真机 client
具体查看上片文章
server1
yum install moosefs-cgi-3.0.103-1.rhsystemd.x86_64.rpm moosefs-cgiserv-3.0.103-1.rhsystemd.x86_64.rpm moosefs-cli-3.0.103-1.rhsystemd.x86_64.rpm moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm -y
server2 3
yum install -y moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm
server4
yum install moosefs-master-3.0.103-1.rhsystemd.x86_64
[root@server1 mfs]# du -sh /var/lib/mfs
3.6M /var/lib/mfs
server1 4
yum install -y pacemaker corosync pcs
yum.repo
[HighAvailability]
name=HighAvailability
baseurl=http://172.25.11.250/rhel7.3/addons/HighAvailability
gpgcheck=0
[ResilientStorage]
name=HighAvailability
baseurl=http://172.25.11.250/rhel7.3/addons/ResilientStorage
gpgcheck=0
做好免密 1->4
ssh-copy-id server4 server1
systemctl isolate multiuser 关闭图形
systemcctl isolate graphical 启动图形 相当于 init 3 4
systemctl start enalbe pcsd
[root@server1 mfs]# systemctl start pcsd
[root@server1 mfs]# systemctl enable pcsd
passwd hacluster
server1上
pcs cluster auth server1 server4
[root@server1 mfs]# pcs cluster auth server1 server4
Username: hacluster
Password:
server4: Authorized
server1: Authorized
pcs cluster setup --name mycluster server1 server4
pcs cluster start server1 server4
[root@server1 mfs]# pcs status corosync
Membership information
----------------------
Nodeid Votes Name
1 1 server1 (local)
2 1 server4
journalctl | grep -i error
pcs property set stonith-enabled=false
crm_verify -L -V 主备集群
pcs cluster start server1 server4
pcs cluster start --all
[root@server1 mfs]# pcs status
Cluster name: mycluster
Stack: corosync
Current DC: server1 (version 1.1.15-11.el7-e174ec8) - partition with quorum
Last updated: Fri Apr 5 14:16:28 2019 Last change: Fri Apr 5 14:15:31 2019 by root via cibadmin on server1
2 nodes and 0 resources configured
Online: [ server1 server4 ]
No resources
Daemon Status:
corosync: active/disabled
pacemaker: active/disabled
pcsd: active/enabled
[root@server1 mfs]# corosync-cfgtool -s
Printing ring status.
Local node ID 1
RING ID 0
id = 172.25.11.1
status = ring 0 active with no faults
[root@server4 3.0.103]# corosync-cfgtool -s
Printing ring status.
Local node ID 2
RING ID 0
id = 172.25.11.4
status = ring 0 active with no faults
[root@server1 mfs]# cat /etc/corosync/corosync.conf
totem {
version: 2
secauth: off
cluster_name: mycluster
transport: udpu
}
nodelist {
node {
ring0_addr: server1
nodeid: 1
}
node {
ring0_addr: server4
nodeid: 2
}
}
client
[root@foundation11 mfs]# umount /mnt/mfs/mfsmeta
[root@foundation11 mfs]# umount /mnt/mfs
server1-4
systemctl stop moosefs-chunkserver
systemctl stop moosefs-master
修改hosts 为对应的虚拟IP mfsmaster
172.25.11.100 mfsmaster
######
server2
yum install targetcli
systemctl start targetcli
tatgetcli
create my_disk1 /dev/vdb
create iqn.2019-04.com.example:server3
tpg1/luns create
server1
yum install iscsi-* -y
iscsiadm -m discovery -t st -p 172.25.11.3
systemctl restart iscsid
iscsiadm -m node -o delete
iscsiadm -m node -l
cat /etc/iscsi/initiatorname.iscsi
server1
mkfs.xfs /dev/sdb
mount /dev/sdb /mnt
cd /var/lib/mfs/
cp -p * /mnt/
chown mfs.mfs /mnt
systemctl start moosefs-master
再停
server4
yum install iscsi-* -y
cat /etc/iscsi/initiatorname.iscsi
iscsiadm -m discovery -t st -p 172.25.11.3
iscsiadm -m node -l
mount /dev/sdb /var/lib/mfs
systemctl start moosefs-master stop
server1
挂载iscsi设备
pcs resource create vip ocf:heartbeat:IPaddr2 ip=172.25.11.100 cidr_netmask=32 op monitor interval=30s
pcs resource create mfsdata ocf:heartbeat:Filesystem device=/dev/sdb directory=/var/lib/mfs fstype=xfs op monitor interval=30s
pcs resource create mfsd systemd:moosefs-master op monitor interval=1min
pcs resource group add mfsgroup vip mfsdata mfsd
pcs cluster stop server1 开始迁移
server2 3
systemctl start moosefs-chunkserver
client
mfsmount
cd /mnt/mfs
crm_mon
Stack: corosync
Current DC: server1 (version 1.1.15-11.el7-e174ec8) - partition with quorum
Last updated: Fri Apr 5 16:55:45 2019 Last change: Fri Apr 5 16:55:42 2019 by root via cibadmin on serv
er1
2 nodes and 3 resources configured
Online: [ server1 server4 ]
Active resources:
Resource Group: mfsgroup
vip (ocf::heartbeat:IPaddr2): Started server1
mfsd (systemd:moosefs-master): Started server1
mfsdata (ocf::heartbeat:Filesystem): Started server4
通过fence 解决内核错误无法自动关机
client 真机
echo c > /proc/sysrq-trigger
server1 4
yum list fence_virtd
yum install fence-virtd -y fence-virt
cd /etc/cluster/
dd if=/dev/urandom of=fence_xvm.key bs=128 count=1
fence_virtd -c
添加网卡选择如果默认不是br0的时候不要全回车
systemctl start fence_virtd
server1 4
mkdir /etc/cluster
scp fence_xvm.key server4:/etc/cluster
pcs stonith create vmfence fence_xvm pcmk_host_map="server1:server1;server4:server4" op monitor interval=1min
pcs property set stonith-enabled=true
map是"主机名：域名"的映射
[root@foundation11 mfs]# virsh list
Id Name State
----------------------------------------------------
17 server1 running
22 server3 running
27 server4 running
crm_verify -L -V 校验是否有错误
fence_xvm -H server4
会自动重启server4
systemctl status pcsd
pcs cluster start server4
crm_mon
Stack: corosync
Current DC: server1 (version 1.1.15-11.el7-e174ec8) - partition with quorum
Last updated: Fri Apr 5 17:19:15 2019 Last change: Fri Apr 5 17:18:58 2019 by root via cibadmin on serv
er1
2 nodes and 4 resources configured
Online: [ server1 server4 ]
Active resources:
Resource Group: mfsgroup
vip (ocf::heartbeat:IPaddr2): Started server1
mfsd (systemd:moosefs-master): Started server1
mfsdata (ocf::heartbeat:Filesystem): Started server4
vmfence (stonith:fence_xvm): Started server4
测试的时候
server4
echo c > /proc/sysrq-trigger
会自动重起
cat /etc/fence_virt.conf
fence_virtd {
listener = "multicast";
backend = "libvirt";
module_path = "/usr/lib64/fence-virt";
}
listeners {
multicast {
key_file = "/etc/cluster/fence_xvm.key";
address = "225.0.0.12";
interface = "br0";
family = "ipv4";
port = "1229";
}
}
backends {
libvirt {
uri = "qemu:///system";
}
}