Moosefs distributed file system (1)

1. Introduction to MFS

2. MFS deployment

2.1 Deployment of the master master

Official website

[root@demo1 ~]#  curl "http://ppa.moosefs.com/MooseFS-3-el7.repo" > /etc/yum.repos.d/MooseFS.repo  #下载repo源
[root@demo1 yum.repos.d]# vim MooseFS.repo 
[root@demo1 yum.repos.d]# cat MooseFS.repo 
[MooseFS]
name=MooseFS $releasever - $basearch
baseurl=http://ppa.moosefs.com/moosefs-3/yum/el7
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
enabled=1
[root@demo1 yum.repos.d]# yum install -y moosefs-master  ##安装MFS master
[root@demo1 yum.repos.d]# yum install moosefs-cgi moosefs-cgiserv moosefs-cli -y  ##安装主服务器
[root@demo1 yum.repos.d]# cd /etc/mfs/
[root@demo1 mfs]# ls
mfsexports.cfg         mfsmaster.cfg         mfstopology.cfg
mfsexports.cfg.sample  mfsmaster.cfg.sample  mfstopology.cfg.sample
[root@demo1 mfs]# systemctl start moosefs-master
[root@demo1 mfs]# vim /etc/hosts   ##解析必须做
172.25.13.1 demo1  mfsmaster
172.25.13.2 demo2
172.25.13.3 demo3
[root@demo1 mfs]# systemctl start moosefs-cgiserv.service    ##网页监控
[root@demo1 mfs]# netstat -antlp 


##9419-9421端口 Master-Servers
##9422  chunnkserver连接端口
##9425   CGI Server

2.2 Deployment on the slave side

2.2.1 demo2

[root@demo1 ~]# cd /etc/yum.repos.d/
[root@demo1 yum.repos.d]# ls
MooseFS.repo  redhat.repo  rhel7.6.repo
[root@demo1 yum.repos.d]# scp MooseFS.repo server2:/etc/yum.repos.d/
ssh: Could not resolve hostname server2: Name or service not known
lost connection
[root@demo1 yum.repos.d]# scp MooseFS.repo demo2:/etc/yum.repos.d/
[root@demo1 yum.repos.d]# scp MooseFS.repo demo3:/etc/yum.repos.d/
[root@demo2 ~]# yum install moosefs-chunkserver   ##安装冷备份，两个slave端
[root@demo3 ~]# yum install moosefs-chunkserver   ##安装冷备份
[root@demo3 ~]# vim /etc/hosts   ##做解析

##server2添加一块虚拟磁盘，当作分布式存储
[root@demo2 mfs]# fdisk /dev/vdb   ##默认分一块就可以
[root@demo2 mfs]# mkfs.xfs /dev/vdb1   ##格式化
[root@demo2 mfs]# mkdir /mnt/chunk1
[root@demo2 mfs]# mount /dev/vdb1 /mnt/chunk1/   ##挂载
[root@demo2 mfs]# blkid
/dev/vdb1: UUID="e94e65fa-35cd-4a40-9ef8-fa6ef42ef128" TYPE="xfs" 
[root@demo2 mfs]# vim /etc/fstab 
[root@demo2 mfs]# cat /etc/fstab     ##永久挂载
UUID="e94e65fa-35cd-4a40-9ef8-fa6ef42ef128" /mnt/chunk1   xfs     defaults 0 0 
[root@demo2 mfs]# mount -a   ##刷新
[root@demo2 mfs]# id mfs   ##设置权限
uid=997(mfs) gid=995(mfs) groups=995(mfs)
[root@demo2 mfs]# chown mfs.mfs /mnt/chunk1/

##启动服务启动服务
[root@demo2 mfs]# systemctl start  moosefs-chunkserver

##查看demo2是否链接到demo1这个master上
[root@demo2 ~]# cd /etc/mfs/
[root@demo2 mfs]# ls
mfschunkserver.cfg  mfschunkserver.cfg.sample  mfshdd.cfg  mfshdd.cfg.sample
[root@demo2 mfs]# vim mfshdd.cfg   ##设置挂载目录
[root@demo2 mfs]# grep -v ^# mfshdd.cfg
/mnt/chunk1 
[root@demo2 mfs]# systemctl restart moosefs-chunkserver   ##重启服务

2.2.2 demo3

[root@demo3 ~]# cd /etc/mfs/
[root@demo3 mfs]# ls
mfschunkserver.cfg  mfschunkserver.cfg.sample  mfshdd.cfg  mfshdd.cfg.sample
[root@demo3 mfs]# mkdir /mnt/chunk2
[root@demo3 mfs]# vim mfshdd.cfg
[root@demo3 mfs]# grep -v ^# mfshdd.cfg
/mnt/chunk2
[root@demo3 mfs]# chown mfs.mfs /mnt/chunk2
[root@demo3 mfs]# df -h /
Filesystem             Size  Used Avail Use% Mounted on
/dev/mapper/rhel-root   17G  1.2G   16G   7% /
[root@demo3 mfs]# systemctl start moosefs-chunkserver

The cpu pressure on the master side comes from the file size of the distributed system, and the
memory pressure comes from the number of files

2.3 Real machine as client

[root@westos ~]# cd /etc/yum.repos.d/
[root@westos yum.repos.d]# ls
aliyun.repo  google-chrome.repo  redhat.repo  rhel8.2.repo
[root@westos yum.repos.d]# curl "http://ppa.moosefs.com/MooseFS-3-el8.repo" > /etc/yum.repos.d/MooseFS.repo
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   168  100   168    0     0     93      0  0:00:01  0:00:01 --:--:--    93
[root@westos yum.repos.d]# vim MooseFS.repo 
[root@westos yum.repos.d]# cat MooseFS.repo 
[MooseFS]
name=MooseFS $releasever - $basearch
baseurl=http://ppa.moosefs.com/moosefs-3/yum/el8
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
enabled=1
[root@westos yum.repos.d]# yum install moosefs-client
[root@westos ~]# cd /etc/mfs/
[root@westos mfs]# ls
mfsmount.cfg  mfsmount.cfg.sample
[root@westos mfs]# vim mfsmount.cfg
[root@westos mfs]# grep -v ^# mfsmount.cfg
/mnt/mfs    #添加挂载的目录，前提是这个挂载目录必须是空的
[root@westos mfs]# vim /etc/hosts  ##做解析

[root@westos ~]# mkdir /mnt/mfs
[root@westos ~]# mfsmount
mfsmaster accepted connection with parameters: read-write,restricted_ip,admin ; root mapped to root:root
[root@westos ~]# ll /etc/mfs/mfsmount.cfg
-rw-r--r--. 1 root root 400 Mar 19 11:29 /etc/mfs/mfsmount.cfg
[root@westos ~]# df
Filesystem            1K-blocks      Used Available Use% Mounted on
devtmpfs                7772476         0   7772476   0% /dev
tmpfs                   7801380    191128   7610252   3% /dev/shm
tmpfs                   7801380     18548   7782832   1% /run
tmpfs                   7801380         0   7801380   0% /sys/fs/cgroup
/dev/mapper/rhel-root 148006288  95042268  52964020  65% /
/dev/nvme0n1p8          1038336    228892    809444  23% /boot
/dev/loop1              4391278   4391278         0 100% /var/www/html/rhel7.6
/dev/loop0              8238560   8238560         0 100% /var/www/html/rhel8.2
/dev/nvme0n1p1            98304     67874     30430  70% /boot/efi
tmpfs                   1560276        20   1560256   1% /run/user/42
tmpfs                   1560276        40   1560236   1% /run/user/0
/dev/sda1             976727036 447430532 529296504  46% /run/media/root/My Passport
mfs#mfsmaster:9421     28285952   1749568  26536384   7% /mnt/mfs   ##挂载成功

[root@westos mfs]# ls
dir1  dir2
[root@westos mfs]# mfsgetgoal dir1
dir1: 2
[root@westos mfs]# mfsgetgoal dir2
dir2: 2
[root@westos mfs]# mfssetgoal -r 1 dir1
dir1:
 inodes with goal changed:                       1
 inodes with goal not changed:                   0
 inodes with permission denied:                  0
[root@westos mfs]# mfsgetgoal dir1
dir1: 1
[root@westos mfs]# cd dir1/
[root@westos dir1]# cp /etc/passwd .
[root@westos dir1]# mfsfileinfo passwd 
passwd:
	chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
		copy 1: 172.25.13.3:9422 (status:VALID)
[root@westos dir1]# cd ../dir2/
[root@westos dir2]# cp /etc/fstab  .
[root@westos dir2]# ls
fstab
[root@westos dir2]# mfsfileinfo fstab 
fstab:
	chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
		copy 1: 172.25.13.2:9422 (status:VALID)
		copy 2: 172.25.13.3:9422 (status:VALID)

3. storage class

3.1 Add a server4 host slave and a server5

##server4
[root@server4 ~]# vim /etc/hosts    ##做解析
[root@server4 ~]# yum install -y moosefs-chunkserver   ##安装软件
[root@server4 ~]# mkdir /mnt/chunk3
[root@server4 ~]# cd /etc/mfs/
[root@server4 mfs]# ls
mfschunkserver.cfg  mfschunkserver.cfg.sample  mfshdd.cfg  mfshdd.cfg.sample
[root@server4 mfs]# vim mfshdd.cfg
[root@server4 mfs]# grep -v ^# mfshdd.cfg
/mnt/chunk3
[root@server4 mfs]# chown mfs.mfs /mnt/chunk3
[root@server4 mfs]# systemctl start moosefs-chunkserver
##server5,和server4一样

3.2 Specify storage class tags to store data

3.2.1 Set label

[root@westos dir2]# ls
fstab
[root@westos dir2]# dd if=/dev/zero of=bigfile bs=1M count=100
100+0 records in
100+0 records out
104857600 bytes (105 MB, 100 MiB) copied, 0.228773 s, 458 MB/s
[root@westos dir2]# ls
bigfile  fstab
[root@westos dir2]# mfsfileinfo bigfile    ##查看分布信息


##server2设置标签
[root@server2 mfs]# pwd
/etc/mfs
[root@server2 mfs]# ls
mfschunkserver.cfg  mfschunkserver.cfg.sample  mfshdd.cfg  mfshdd.cfg.sample
[root@server2 mfs]# vim mfschunkserver.cfg    ##设置标签A
[root@server2 mfs]# systemctl reload moosefs-chunkserver.service 

## server3设置标签B
[root@serve3 ~]# cd /etc/mfs/
[root@serve3 mfs]# ls
mfschunkserver.cfg  mfschunkserver.cfg.sample  mfshdd.cfg  mfshdd.cfg.sample
[root@serve3 mfs]# vim mfschunkserver.cfg
[root@serve3 mfs]# systemctl reload moosefs-chunkserver.service 
##server4设置标签为A
[root@server4 ~]# cd /etc/mfs/
[root@server4 mfs]# ls
mfschunkserver.cfg  mfschunkserver.cfg.sample  mfshdd.cfg  mfshdd.cfg.sample
[root@server4 mfs]# vim mfschunkserver.cfg
[root@server4 mfs]# systemctl reload moosefs-chunkserver.service 

3.2.2 Using tags

[root@westos ~]# cd /mnt/mfs  
[root@westos mfs]# mfsscadmin create 2A sclass1   ##创建标签
[root@westos mfs]# pwd
/mnt/mfs
[root@westos mfs]# ls
dir1  dir2
[root@westos mfs]# cd dir2/
[root@westos dir2]# ls
bigfile  fstab
[root@westos dir2]# mfssetsclass sclass1 fstab    ##设置标签
fstab: storage class: 'sclass1'
[root@westos dir2]# mfsfileinfo fstab 
fstab:
	chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
		copy 1: 172.25.13.2:9422 (status:VALID)
		copy 2: 172.25.13.4:9422 (status:VALID)

3.3 CKA settings

##server2主机设置标签A S H(S表示固态存储，H表示机械硬盘存储)
##server3主机设置标签B S H
##server4主机设置标签B S 
##server5主机设置标签B H

[root@westos dir2]# mfsscadmin create -C 2AS -K AS,BS -A AH,BH -d 30 sclass2  ##C创建，K保持，A打包，d表示打包之后的过期时间
storage class make sclass2: ok
[root@westos dir2]# mfssetsclass sclass2 fstab 
fstab: storage class: 'sclass2'
[root@westos dir2]# mfsfileinfo fstab 
fstab:
	chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
		copy 1: 172.25.13.2:9422 (status:VALID)
		copy 2: 172.25.13.4:9422 (status:VALID)

3.4 Recover data

[root@westos ~]# cd /mnt/mfs/
[root@westos mfs]# ls
dir1  dir2
[root@westos dir2]# mfsgettrashtime fstab   ##查看删除之后可以在多久的时间之内恢复
fstab: 86400
[root@westos mfs]# cd dir1/
[root@westos dir1]# ls
passwd
[root@westos dir1]# rm -fr passwd 
[root@westos dir1]# ls
[root@westos dir1]# mfsgettrashtime passwd
passwd: realpath error on (/mnt/mfs/dir1/passwd): ENOENT (No such file or directory)
[root@westos ~]# mkdir /mnt/mfsmeta
[root@westos ~]# mfsmount -m /mnt/mfsmeta
[root@westos mnt]# cd /mnt/mfsmeta/
[root@westos mfsmeta]# ls
sustained  trash
[root@westos mfsmeta]# cd trash/
[root@westos trash]# find -name *passwd*
./004/00000004|dir1|passwd
[root@westos trash]# cd 004/
[root@westos 004]# ls
'00000004|dir1|passwd'   undel              
[root@westos 004]# mv 00000004\|dir1\|passwd undel/
[root@westos 004]# ls
undel
[root@westos 004]# cd /mnt/mfs
[root@westos mfs]# ls
dir1  dir2
[root@westos mfs]# cd dir1/
[root@westos dir1]# ls
passwd
[root@westos dir1]# mfsfileinfo passwd 
passwd:
	chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
		copy 1: 172.25.13.2:9422 (status:VALID)