解释如上图read模式，当客户端访问mfsmaster时，mfsmaster会告诉客户端，它存储的数据在那里，然后客户端会去访问存放数据的主机，存放数据的主机返还客户端数据，上图大概就是这个个意思
上图时写的过程，客户端写入数据请求到mfs-master，mfs-master存到后端server根据客户端的存储服务器数两分配chunk servers，然后，mfs-master将这个信息转给客户端，客户端与chunk servers进行数据写入，存储
环境
redhat.7.3
server1 172.25.51.1 ##mfsmaster节点
server2 172.25.51.2 ##从节点，就是真正储存数据的节点
server3 172.25.52.3 ##通server2
server4 172.25.52.4 ##高可用用，作为master节点，前边实验不用
selinux=disable ,防火墙关闭,开机不启动NetworkManager关闭，开机不启动
一.安装mfs
官网：https://moosefs.com/
参考https://moosefs.com/download/
如果你没包的话参考上边的官网，配置yum源安装，这里已经下载好了
server1安装下边三个包，server1为master节点
rpm -ivh
moosefs-cgi-3.0.100-1.rhsystemd.x86_64.rpm
moosefs-cgiserv-3.0.100-1.rhsystemd.x86_64.rpm
moosefs-master-3.0.100-1.rhsystemd.x86_64.rpm安装完，自动创建mfs用户，数据储藏目录在/var/lib/mfs
[root@server1 mfs]# ll -d /var/lib/mfs/
drwxr-xr-x 2 mfs mfs 52 May 16 07:43 /var/lib/mfs/
[root@server1 mfs]# id mfs
uid=996(mfs) gid=992(mfs) groups=992(mfs)启动mfsmaster服务
[root@server1 mfs]# mfsmaster start
如下图所示，没有报错，ok，开启了三个端口9419 #metalogger 监听的端口地址(默认是9419)；9420 #用于chunkserver 连接的端口地址（默认是9420）；9421 #用于客户端挂接连接的端口地址（默认是9421）；，
启动cgi服务，主要用古web监控
[root@server1 mfs]# mfscgiserv
浏览器查看
http://172.25.51.1:9425/mfs.cgi这是由于没有解析，注意是mfs的解析
添加解析，server1
[root@server1 mfs]# vim /etc/hosts
172.25.51.1 server1 mfsmaster
浏览器再次查看
点击上边的加号或减号添加湖泊减去相应的信息（查看）
注意,mfsmaster这个解析从节点及一客户端短都要有，这里直接把server1的解析文件发给server2和server3以及server4
[root@server1 mfs]# scp /etc/hosts server2:/etc/
[root@server1 mfs]# scp /etc/hosts server3:/etc/
[root@server1 mfs]# scp /etc/hosts server4:/etc/
二.从节点部署
server2和server3安装包
[root@server2 3.0.100]# rpm -ivh moosefs-chunkserver-3.0.100-1.rhsystemd.x86_64.rpm
[root@server3 3.0.100]# rpm -ivh moosefs-chunkserver-3.0.100-1.rhsystemd.x86_64.rpm
server2从节点的配置
[root@server2 3.0.100]# cd
[root@server2 ~]# vim /etc/mfs/mfshdd.cfg ##这个文件全是注释只需要在最后一行写上这个目录，wq退出后重建该目录，修改所有人所有组为mfs
/mnt/chunk1
[root@server2 ~]# mkdir /mnt/chunk1
[root@server2 ~]# chown mfs.mfs /mnt/chunk1/
[root@server2 ~]# vim /etc/mfs/mfschunkserver.cfg ##这个文件页全是注释，取消下边这行注释，
MASTER_HOST = mfsmaster
server3结点的配置server2
[root@server3 ~]# vim /etc/mfs/mfshdd.cfg
/mnt/chunk2
[root@server3 ~]# mkdir /mnt/chunk2
[root@server3 ~]# chown mfs.mfs /mnt/chunk2
[root@server3 ~]# vim /etc/mfs/mfschunkserver.cfg
MASTER_HOST = mfsmaster
启动两个结点的mfschunkserver
[root@server2 ~]# mfschunkserver start
如下图没报错ok
server3同上
[root@server3 ~]# mfschunkserver start
浏览器：查看
http://172.25.254.1:9425
三.真机安装客户端
安装客户端
[root@foundation51 ~]# rpm -ivh moosefs-client-3.0.100-1.rhsystemd.x86_64.rpm
配置解析
[root@foundation51 ~]# vim /etc/hosts
172.25.51.1 server1 mfsmaster
[root@foundation51 ~]# ping mfsmaster
##测试下，可以ping通
配置文件，挂载
[root@foundation51 ~]# vim /etc/mfs/mfsmount.cfg
/mnt/mfs
[root@foundation51 ~]# mkdir /mnt/mfs
[root@foundation51 ~]# mfsmount ##挂载
[root@foundation51 ~]# df 查看
数据的存储
在挂载的目录下新建两个目录
[root@foundation51 ~]# mkdir /mnt/mfs/dir1
[root@foundation51 ~]# mkdir /mnt/mfs/dir2
查看由即太数据存储服务器（server2和server3）
[root@foundation51 ~]# mfsgetgoal /mnt/mfs/dir1
/mnt/mfs/dir1: 2 ##当然两个了
[root@foundation51 ~]# mfsgetgoal /mnt/mfs/dir2
/mnt/mfs/dir2: 2 ##当然两个了
设置存储几份就是说，在给几台后端存储服务器上同步数据，这里最大为2因为只有server2和server3
[root@foundation51 ~]# mfssetgoal -r 1 /mnt/mfs/dir1
/mnt/mfs/dir1:
inodes with goal changed: 1
inodes with goal not changed: 0
inodes with permission denied: 0
[root@foundation51 ~]# mfssetgoal -r 2 /mnt/mfs/dir2
/mnt/mfs/dir2:
inodes with goal changed: 0
inodes with goal not changed: 1
inodes with permission denied: 0
各仍一份文件到/mnt/mfs/dir1和/mnt/mfs/dir2中
查看文件信息
[root@foundation51 ~]# mfsfileinfo /mnt/mfs/dir1/passwd
/mnt/mfs/dir1/passwd:
chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
copy 1: 172.25.51.3:9422 (status:VALID)
[root@foundation51 ~]# mfsfileinfo /mnt/mfs/dir2/fstab
/mnt/mfs/dir2/fstab:
chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
copy 1: 172.25.51.2:9422 (status:VALID)
copy 2: 172.25.51.3:9422 (status:VALID
很显然fstab存了两份在server2和server3上各一份，二passwd存了一份在server3上
关闭server2
[root@server2 ~]# mfschunkserver stop
sending SIGTERM to lock owner (pid:2188)
waiting for termination terminated
对于fstasb这份文件来说不受影响，因为server2和server3都存了，关闭server2可以从server3读取
[root@foundation51 ~]# mfsfileinfo /mnt/mfs/dir2/fstab
/mnt/mfs/dir2/fstab:
chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
copy 1: 172.25.51.3:9422 (status:VALID)
[root@foundation51 ~]# mfsfileinfo /mnt/mfs/dir1/passwd
/mnt/mfs/dir1/passwd:
chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
copy 1: 172.25.51.3:9422 (status:VALID)
开启server2
[root@server2 ~]# mfschunkserver start
客户端恢复正常
关闭server3
[root@server3 mnt]# mfschunkserver stop
sending SIGTERM to lock owner (pid:2154)
waiting for termination terminated
客户端查看信息如下图
赶紧恢复过来进行下个实验
对于大文件，实行离散存储
[root@foundation51 ~]# cd /mnt/mfs/dir1/
[root@foundation51 dir1]# dd if=/dev/zero of=bigfile bs=1M count=200
[root@foundation51 dir1]# mfsfileinfo bigfile
----------------------------------------
[root@foundation51 dir1]# cd ../dir2/
[root@foundation51 dir2]# dd if=/dev/zero of=bigfile
[root@foundation51 dir2]# mfsfileinfo bigfile
删除两个目录的bigfile
[root@foundation51 dir2]# rm -fr bigfile
[root@foundation51 dir2]# ls
fstab
[root@foundation51 dir2]# cd ../dir1/
[root@foundation51 dir1]# rm -fr bigfile
[root@foundation51 dir1]# ls
passwd
删除文件缓存时间
[root@foundation51 dir1]# mfsgettrashtime .
.: 86400 ##单位秒
删除/mnt/mfs/dir1/passwd
[root@foundation51 dir1]# ls
passwd
[root@foundation51 dir1]# rm -fr passwd
[root@foundation51 dir1]# ls
接下来来进行元数据恢复
[root@foundation51 dir1]# mkdir /mnt/mfsmeta
[root@foundation51 dir1]# mfsmount -m /mnt/mfsmeta/
mfsmaster accepted connection with parameters: read-write,restricted_ip
##此处挂载的话，事实上，df查看不到
[root@foundation51 dir1]# cd /mnt/mfsmeta/trash
[root@foundation51 trash]# find -name *passwd*
./004/00000004|dir1|passwd
[root@foundation51 trash]# mv ./004/00000004\|dir1\|passwd undel/ ##注意转义，你也可以把整个文件强引用单引号
[root@foundation51 trash]# find -name *passwd* ##为空找不到
[root@foundation51 trash]# cd /mnt/mfs/dir1/
[root@foundation51 dir1]# ls ##找回来了
passwd
找回来了
补充
当mfs-master开启的时候在/var/lib/mfs/会有metadata.mfs.back这个文件，关闭的时候，没有
四.高可用
1.需要安装pacemaker corosync pcs，我用的时redhat7.3所以用pcs
宿主机设置iptables
[root@foundation51 dir1]# iptables -t nat -I POSTROUTING -s 172.25.51.0/24 -j MASQUERADE server1添加路由和dns
[root@server1 ~]# route add default gw 172.25.51.250
[root@server1 ~]# echo nameserver 114.114.114.114 >/etc/resolv.conf
[root@server1 ~]# ping www.baidu.com 
阿里云yum源
[root@server1 ~]# vim /etc/yum.repos.d/yum.repo
[centos]
name=centos
baseurl=https://mirrors.aliyun.com/centos/7/os/x86_64/
gpgcheck=0
安装
准备搭建第三方yum源
[root@server1 yum.repos.d]# vim /etc/yum.conf
keepcache=1 ##该为1,缓存下的包
[root@server1 yum.repos.d]# yum install pacemaker corosync pcs
搭建第三方yum源
[root@server1 ~]# cp /var/cache/yum/x86_64/7Server/centos/packages /mnt
[root@server1 ~]# scp -r /mnt/packages/ [email protected]:
[root@server4 ~]# createrepo -v packages/
[root@server4 ~]# vim /etc/yum.repos.d/yum.repo
[centos]
name=centos
baseurl=file:///root/packages
gpgcheck=0
[root@server4 ~]# yum clean all
[root@server4 ~]# yum repolist
[root@server4 ~]# yum install pacemaker corosync pcs一番折腾终于安装完毕
2.免密配置
[root@server1 mnt]# ssh-keygen
[root@server1 mnt]# ssh-copy-id server1
[root@server1 mnt]# scp -r /root/.ssh/ server4:~ 
测试下
ssh相互连接测试
3.启动服务
[root@server1 ~]# systemctl start pcsd.service
[root@server1 ~]# systemctl enable pcsd.service
[root@server4 ~]# systemctl start pcsd.service
[root@server4 ~]# systemctl enable pcsd.service
4.设置hacluster用户密码，当安装完三个包时，hacluster用户会自动创建，但是没密码，这里设置相同密码，必须相同**
[root@server1 ~]# ssh server1 -- 'echo redhat | passwd --stdin hacluster'
[root@server1 ~]# ssh server4 -- 'echo redhat | passwd --stdin hacluster'
5.集群认证
[root@server1 ~]# pcs cluster auth server1 server4
Username: hacluster
Password: redhat
server4: Authorized
server1: Authorized
6.给集群起个名字mycluster
[root@server1 ~]# pcs cluster setup --name mycluster server1 server4
7启动集群
[root@server1 ~]# pcs status cluster ##查看下关着
Error: cluster is not currently running on this node
[root@server1 ~]# pcs cluster start --all ##启动all标识两台主机server1和server4
server1: Starting Cluster...
server4: Starting Cluster...
[root@server1 ~]# pcs status cluster ##再次查看ok
8,查看集群信息的一些命令
[root@server1 ~]# corosync-cfgtool -s
[root@server1 ~]# pcs status corosync
[root@server1 ~]# pcs status
9.配置集群服务
[root@server1 ~]# yum install bash-* -y ##tab补全命令
[root@server4 ~]# yum install bash-* -y[root@server1 ~]# crm_verify -L -V ##会报一大堆error
error: unpack_resources: Resource start-up disabled since no STONITH resources have been defined
error: unpack_resources: Either configure some or disable STONITH with the stonith-enabled option
error: unpack_resources: NOTE: Clusters with shared data need STONITH to ensure data integrity
Errors found during check: config not valid
关闭stonith-enabled
[root@server1 ~]# pcs property set stonith-enabled=false
[root@server1 ~]# crm_verify -L -V
添加vip
[root@server1 ~]# pcs resource create VIP ocf:heartbeat:IPaddr2 ip=172.25.51.111 cidr_netmask=32 op monitor interval=30s
[root@server1 ~]# pcs status ##查看集群状态，server1和server4都在线，vip在server1
查看下vip是否在server1
[root@server1 ~]# ip addr
挂起server1,vip转移server4
[root@server1 ~]# pcs node standby
[root@server1 ~]# pcs status
[root@server4 ~]# ip addr
server1恢复过来vip不会在转移回来，除非server4挂了
[root@server1 ~]# pcs node unstandby
[root@server1 ~]# pcs status
[root@server1 ~]# pcs resource providers ##查看提供的资源
heartbeat
openstack
pacemaker
[root@server1 ~]# pcs resource standards ##标准
lsb
ocf
service
systemd
编写systemd启动脚本
[root@server1 ~]# cd /usr/lib/systemd/system
[root@server1 system]# vim mfsd.service
[Unit]
Decription=mfs
After=network.target
[Service]
Type=forking
ExecStart=/usr/sbin/mfsmaster -a
ExecStop=/usr/sbin/mfsmaster stop
PrivateTmp=true
[Install]
WantedBy=multi-user.target
[root@server1 system]# systemctl daemon-reload
[root@server1 system]# systemctl stop mfsd.service
[root@server1 system]# systemctl start mfsd.service
[root@server1 system]# systemctl enable mfsd.service
警策是没问题后传给server4
[root@server1 system]# scp mfsd.service server4:/usr/lib/systemd/system/测试的话可以铜鼓片查看进程或者看 ll /var/lib/mfs/有没有metadata.mfs这个文件，也可以通过客户端查看测试结果,在/mnt/mfs/中，server1关闭mfs-master客户端会卡住
server4安装mfs-master
[root@server4 3.0.100]# rpm -ivh moosefs-master-3.0.100-1.rhsystemd.x86_64.rpm
[root@server4 ~]# systemctl start mfsd
[root@server4 ~]# systemctl enable mfsd
server3增加个盘
查找下盘符vda-5G
[root@server3 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 10G 0 disk
├─sda1 8:1 0 1G 0 part /boot
└─sda2 8:2 0 9G 0 part
├─rhel-root 253:0 0 8G 0 lvm /
└─rhel-swap 253:1 0 1G 0 lvm [SWAP]
vda 252:0 0 5G 0 disk
安装targetcli
[root@server3 ~]# yum install targetcli -y
[root@server3 ~]# systemctl start target
[root@server3 ~]# targetcli
/> /backstores/block create server3.disk1 /dev/vda
/> iscsi/ create iqn.2018-05.com.example:server3
/> iscsi/iqn.2018-05.com.example:server3/tpg1/acls create iqn.2018-05.com.example:mfs
/> iscsi/iqn.2018-05.com.example:server3/tpg1/luns create /backstores/block/server3.disk1
/> exit
[root@server3 ~]# netstat -anltpp | grep 3260
3260开启
server4，server1安装iscsi
[root@server4 ~]# yum install iscsi-* -y
[root@server1 system]# yum install iscsi-* -y以下操作在server1上
发现设备
[root@server4 ~]# iscsiadm -m discovery -t st -p 172.25.51.3
172.25.51.3:3260,1 iqn.2018-05.com.example:server3
[root@server1 system]# iscsiadm -m discovery -t st -p 172.25.51.3
172.25.51.3:3260,1 iqn.2018-05.com.example:server3
链接设备
[root@server1 system]# vim /etc/iscsi/initiatorname.iscsi
InitiatorName=iqn.2018-05.com.example:mfs
[root@server1 system]# systemctl restart iscsid
[root@server1 system]# iscsiadm -m node -l
Logging in to [iface: default, target: iqn.2018-05.com.example:server3, portal: 172.25.51.3,3260] (multiple)
Login to [iface: default, target: iqn.2018-05.com.example:server3, portal: 172.25.51.3,3260] successful.
[root@server1 system]# lsblk ##查看盘符sdb
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 10G 0 disk
├─sda1 8:1 0 1G 0 part /boot
└─sda2 8:2 0 9G 0 part
├─rhel-root 253:0 0 8G 0 lvm /
└─rhel-swap 253:1 0 1G 0 lvm [SWAP]
sdb 8:16 0 5G 0 disk
分区，格式化ext4
[root@server1 system]# fdisk /dev/sdb
[root@server1 system]# mkfs.ext4 /dev/sdb1
挂载
[root@server1 system]# mount /dev/sdb1 /mnt/
[root@server1 system]# df[root@server1 ~]# cp -rp /var/lib/mfs/* /mnt/不是所有都得这样修改，主要是挂载后，/var/lib/mfs/下的这些文件所有人和所有组要为mfs
[root@server1 ~]# umount /mnt/
[root@server1 ~]# mount /dev/sdb1 /var/lib/mfs/
[root@server1 ~]# chown mfs.mfs /var/lib/mfs/*
[root@server1 ~]# chown root.root /var/lib/mfs/lost+found/ -R
server4操作
server4挂载
[root@server4 ~]# vim /etc/iscsi/initiatorname.iscsi
InitiatorName=iqn.2018-05.com.example:mfs
[root@server4 ~]# systemctl restart iscsid
[root@server4 ~]# iscsiadm -m node -l
[root@server4 ~]# blkid
[root@server4 ~]# lsblk
sda 8:0 0 10G 0 disk
├─sda1 8:1 0 1G 0 part /boot
└─sda2 8:2 0 9G 0 part
├─rhel-root 253:0 0 8G 0 lvm /
└─rhel-swap 253:1 0 1G 0 lvm [SWAP]
sdb 8:16 0 5G 0 disk
└─sdb1 8:17 0 5G 0 part
[root@server4 ~]# mount /dev/sdb1 /var/lib/mfs/
[root@server4 ~]# ll /mnt/
[root@server4 ~]# chown mfs.mfs /var/lib/mfs/*
[root@server4 ~]# chown root.root /var/lib/mfs/lost+found/
usermod -g mfs mfs ##这个后边可能还有麻烦添加磁盘,启动服务
[root@server1 ~]# pcs resource create mfsFS ocf:heartbeat:Filesystem device="/dev/sdb1" directory="/var/lib/mfs/" fstype="ext4"
[root@server1 ~]# pcs status
[root@server1 ~]# pcs resource create mfsd systemd:mfsd
[root@server1 ~]# pcs status添加组服务
[root@server1 ~]# pcs resource group add mfsgroup VIP mfsFS mfsd op monitor interval=20s timeout=10s
[root@server1 ~]# pcs status
添加解析,集群内所有主机，包括客户机，注意去除原先的解析，这个解析文件会从上往下依次读取
[root@server1 ~]# vim /etc/hosts
172.25.51.111 mfsmaster
[root@server2 ~]# vim /etc/hosts
172.25.51.111 mfsmaster
[root@server3 ~]# vim /etc/hosts
172.25.51.111 mfsmaster
[root@server4 ~]# vim /etc/hosts
172.25.51.111 mfsmaster
[kiosk@foundation51 ~]# vim /etc/hosts
172.25.51.111 mfsmaster
客户端此时可以正常使用
接下来，挂server1，这里上边的操作并没有导致server4正常启动mfs-master,原因是/var/lib/mfs/下的那一堆文件的所有人和所有组有问题，总之解决了
[root@server1 ~]# pcs cluster standby server1
[root@server1 ~]# pcs status
客户端正常访问
fence机制
[root@server1 ~]# yum install -y fence-virt.x86_64
[root@server4 ~]# yum install -y fence-virt.x86_64
查看可用的资源导入
[root@server1 ~]# stonith_admin -I
fence_xvm
fence_virt
2 devices found
[root@server1 ~]# stonith_admin -M -a fence_xvm
配置认证文件
[root@server1 ~]# mkdir /etc/cluster/
[root@server4 ~]# mkdir /etc/cluster/
##因为之前配好了，所以直接发，没有的话自行配置，参开之前的pacemaker
[root@foundation51 ~]# cd /etc/cluster/
[root@foundation51 cluster]# ls
fence_xvm.key
[root@foundation51 cluster]# scp fence_xvm.key server1:/etc/cluster/
[root@foundation51 cluster]# scp fence_xvm.key server4:/etc/cluster/
普通用户开启的时候输入密码了，途中没显示
添加服务
[root@server1 ~]# pcs property set stonith-enabled=true ##true
[root@server1 ~]# pcs stonith list
[root@server1 ~]# pcs stonith create vmfence fence_xvm pcmk_host_map="server1:server1;server4:server4" op monitor interval=30s
[root@server1 ~]# pcs status
[root@server1 ~]# pcs cluster unstandby server1
[root@server1 ~]# pcs status 
注意了，要变身了：使server4崩溃，server4会自动断电重启
[root@server4 ~]# echo c >/proc/sysrq-trigger
服务转移server1，客户端正常
重启后，由于没有设置开机自动启动集群服务，所以你得手动启动下
[root@server4 ~]# pcs cluster start --all
server1: Starting Cluster...
server4: Starting Cluster...
[root@server1 ~]# pcs status ##少等片刻，查看集群状态，fence机制到了server4上，服务正常