准备工作
环境信息
修改dns配置
设置阿里源
root@sh-saas-cs-manager-online-01:~# mv /etc/apt/sources.list /etc/apt/sources.list.bak
root@sh-saas-cs-manager-online-01:~# cat /etc/apt/sources.list
deb http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse
更新系统
root@sh-saas-cs-manager-online-01:~# apt-get update #更新源信息
root@sh-saas-cs-manager-online-01:~# apt-get upgrade #更新本机应用软件
root@sh-saas-cs-manager-online-01:~# ufw status
Status: inactive
root@sh-saas-cs-manager-online-01:~# ufw disable
Firewall stopped and disabled on system startup
安装jdk1.8
root@sh-saas-cs-manager-online-01:~# add-apt-repository ppa:openjdk-r/ppa
root@sh-saas-cs-manager-online-01:~# apt-get update
root@sh-saas-cs-manager-online-01:~# apt-get install openjdk-8-jdk
添加java环境变量
在 /etc/profile和~/.bashrc中
增加
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
root@sh-saas-cs-manager-online-01:~# java -version
openjdk version "1.8.0_162"
安装NTP
root@sh-saas-cs-manager-online-01:~# apt-get install ntp
确保在各 Ceph 节点上启动了 NTP 服务，并且要使用同一个 NTP 服务器
修改ntp.conf配置，更新为阿里云ntp服务器为优先同步服务器配置
root@sh-saas-cs-manager-online-01:~# cat /etc/ntp.conf
# ntp.conf
driftfile /var/lib/ntp/drift
pidfile /var/run/ntpd.pid
logfile /var/log/ntp.log
# Access Control Support
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
restrict 127.0.0.1
# local clock
server 127.127.1.0
fudge 127.127.1.0 stratum 10
server ntp1.aliyun.com iburst minpoll 4 maxpoll 10
restrict ntp1.aliyun.com nomodify notrap nopeer noquery
server ntp1.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp1.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp10.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp10.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp11.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp11.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp12.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp12.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp2.aliyun.com iburst minpoll 4 maxpoll 10
restrict ntp2.aliyun.com nomodify notrap nopeer noquery
server ntp2.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp2.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp3.aliyun.com iburst minpoll 4 maxpoll 10
restrict ntp3.aliyun.com nomodify notrap nopeer noquery
server ntp3.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp3.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp4.aliyun.com iburst minpoll 4 maxpoll 10
restrict ntp4.aliyun.com nomodify notrap nopeer noquery
server ntp4.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp4.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp5.aliyun.com iburst minpoll 4 maxpoll 10
restrict ntp5.aliyun.com nomodify notrap nopeer noquery
server ntp5.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp5.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp6.aliyun.com iburst minpoll 4 maxpoll 10
restrict ntp6.aliyun.com nomodify notrap nopeer noquery
server ntp6.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp6.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp7.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp7.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp8.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp8.cloud.aliyuncs.com nomodify notrap nopeer noquery
server ntp9.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10
restrict ntp9.cloud.aliyuncs.com nomodify notrap nopeer noquery
root@sh-saas-cs-manager-online-01:~# /etc/init.d/ntp start
安装配置NFS
root@sh-saas-cs-manager-online-01:~# apt-get install nfs-kernel-server nfs-common -y #注：确保所有机器上都安装了nfs-common包
root@sh-saas-cs-manager-online-01:~# mkdir -p /data/{primary,secondary}
root@sh-saas-cs-manager-online-01:~# vim /etc/exports
/data/secondary *(rw,async,no_root_squash,no_subtree_check)
/data/primary *(rw,async,no_root_squash,no_subtree_check)
root@sh-saas-cs-manager-online-01:~# exportfs -a
root@sh-saas-cs-manager-online-01:~# showmount -e #查看共享目录
root@sh-saas-cs-manager-online-01:~# /etc/init.d/rpcbind restart # 重启rpcbind
root@sh-saas-cs-manager-online-01:~# /etc/init.d/nfs-kernel-server restart # 重启nfs
安装cloudstack
1. 添加cloudstack源(注：4.11包还没有，所以用4.10)
root@sh-saas-cs-manager-online-01:~# echo deb http://cloudstack.apt-get.eu/ubuntu trusty 4.10 | tee /etc/apt/sources.list.d/cloudstack.list
root@sh-saas-cs-manager-online-01:~# wget -O - http://cloudstack.apt-get.eu/release.asc | apt-key add -
添加cloudstack国内源
root@sh-saas-cs-manager-online-01:~# echo deb http://mirrors.cloudstack-china.com/mirror/ubuntu trusty 4.10 | tee /etc/apt/sources.list.d/cloudstack.list
root@sh-saas-cs-manager-online-01:~# wget -O - http://cloudstack.apt-get.eu/release.asc | sudo apt-key add -
安装cloudstack
root@sh-saas-cs-manager-online-01:~# sudo apt-get update
root@sh-saas-cs-manager-online-01:~# apt-get install cloudstack-management
# cloudstack-setup-databases cloud:<dbpassword>@<ip address mysql server> --deploy-as=root:<dbpassword> #mysql安装略，(cloudstack-setup-databases cloud:[email protected]—deploy-as=root:123456)
##
sudo apt-get install mysql-server libmysqlclient-dev
参考mysql安装文档
eg:
root@sh-saas-cs-manager-online-01:~# cloudstack-setup-databases cloud:[email protected] --deploy-as=root:Abc12345
如果以上mysql数据库不在本机，确保root@%用户有创建用户并授权的权限，可以在运行以下命令
mysql> grant all privileges on *.* to 'root'@'%'identified by 'Abc12345' with grant option;
If you are running the KVM hypervisor on the same machine with the Management Server, edit /etc/sudoers and add the following line:
Defaults:cloud !requiretty
导入系统虚拟机模板
2. 安装KVM模版
root@sh-saas-cs-manager-online-01:~# cd /data/secondary
root@sh-saas-cs-manager-online-01:~# wget http://cloudstack.apt-get.eu/systemvm/4.10/systemvm64template-4.10-4.10.0.0-kvm.qcow2.bz2
root@sh-saas-cs-manager-online-01:~# /usr/share/cloudstack-common/scripts/storage/secondary/cloud-install-sys-tmplt -m /data/secondary -f systemvm64template-4.10-4.10.0.0-kvm.qcow2.bz2 -h kvm -F
注：
1. kvm模版的版本对应cloudstack的版本号
2. /data/secondary给予777的权限，主要针对于模版文件
cloudstack4.10.x系统虚拟机默认用户名密码：root/password
安装完成, 访问
http://10.10.128.77:8080/client/
默认用户名和密码为 admin, password
#vim /etc/sudoers
增加
Defaults:cloud !requiretty
# cloudstack-setup-management
停止服务
# /etc/init.d/cloudstack-management stop
If the servlet container is Tomcat7 the argument –tomcat7 must be used.
安装配置KVM agent端安装
1. 安装kvm
注：KVM 需要有 CPU 的支持（Intel vmx 或 AMD svm），在安装 KVM 之前检查一下 CPU 是否提供了虚拟技术的支持：
root@sh-saas-cs-ceph-online-01:~# egrep '^flags.*(vmx|svm)' /proc/cpuinfo
有显示, 有显示则说明处理器具有VT功能
root@sh-saas-cs-ceph-online-01:~# apt-get install qemu-kvm qemu virt-manager virt-viewer libvirt-bin bridge-utils -y
检查kvm是否成功安装：
root@sh-saas-cs-ceph-online-01:~# kvm-ok
INFO: /dev/kvm exists
KVM acceleration can be used
或者
root@sh-saas-cs-ceph-online-01:~# lsmod | grep kvm
kvm_intel 167936 0
kvm 532480 1 kvm_intel
irqbypass 16384 1 kvm
以上有输出表示成功安装
2. 修改网卡配置(修改前)
root@sh-saas-cs-ceph-online-01:~# cat /etc/network/interfaces
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto eth0
iface eth0 inet manual
auto eth1
iface eth1 inet manual
auto bond1
iface bond1 inet static
address 10.16.144.2
netmask 255.255.240.0
gateway 10.16.144.1
bond_mode 4
bond_miimon 100
bond-ad-select 1
bond_downdelay 200
bond_updelay 200
bond_xmit_hash_policy layer2+3
bond-slaves none
up /sbin/ifenslave bond1 eth0 eth1
post-up route add -net 100.64.0.0 netmask 255.192.0.0 gw 10.16.144.1
up ifconfig eth0 up
up ifconfig eth1 up
down /sbin/ifenslave -d bond1 eth0 eth1
post-down ifconfig eth0 down
post-down ifconfig eth1 down
修改网卡配置(修改后)
root@sh-saas-cs-ceph-online-01:~# cat /etc/network/interfaces
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto eth0
iface eth0 inet manual
mtu 9600
bond-master bond1
auto eth1
iface eth1 inet manual
mtu 9600
bond-master bond1
auto bond1
iface bond1 inet manual
bond_mode 4
bond_miimon 100
bond_xmit_hash_policy layer3+4
bond_downdelay 200
bond_updelay 200
mtu 9600
post-up ifenslave bond1 eth0 eth1
pre-down ifenslave -d bond1 eth0 eth1
bond-slaves none
auto cloudbr0
iface cloudbr0 inet static
bridge_ports bond1
address 10.16.144.2
netmask 255.255.240.0
gateway 10.16.144.1
dns-nameservers 10.16.253.253
dns-nameservers 10.10.253.252
post-up route add -net 10.0.0.0 netmask 255.0.0.0 gw 10.16.144.1
post-up route add -net 172.16.0.0 netmask 255.240.0.0 gw 10.16.144.1
post-up route add -net 192.168.0.0 netmask 255.255.0.0 gw 10.16.144.1
post-up route add -net 100.64.0.0 netmask 255.192.0.0 gw 10.16.144.1
root@sh-saas-cs-ceph-online-01:~# ifdown eth0 && ifup eth0 && ifup cloudbr0 #重启网卡服务
3. 修改kvm配置
root@sh-saas-cs-ceph-online-01:~# vim /etc/libvirt/qemu.conf
vnc_listen=0.0.0.0 #取消前面的注释
新增以下内容（解决权限问题）
security_driver="none"
user="root"
group="root"
root@sh-saas-cs-ceph-online-01:~# cat /etc/libvirt/qemu.conf |egrep -v "^#|^$"
vnc_listen = "0.0.0.0"
security_driver="none"
user="root"
group="root"
root@sh-saas-cs-ceph-online-01:~# cat /etc/libvirt/libvirtd.conf |egrep -v "^#|^$"
listen_tls = 0
listen_tcp = 1
tcp_port = "16509"
mdns_adv = 0
unix_sock_group = "libvirtd"
unix_sock_ro_perms = "0777"
unix_sock_rw_perms = "0770"
auth_unix_ro = "none"
auth_unix_rw = "none"
auth_tcp = "none"
root@sh-saas-cs-ceph-online-01:~# vim /etc/cloudstack/agent/agent.properties #cloudstack管理服务器地址
host=10.10.128.77
root@sh-saas-cs-ceph-online-01:~# cloudstack-setup-agent
Welcome to the CloudStack Agent Setup:
Please input the Management Server Hostname/IP-Address:[10.10.128.77]
Please input the Zone Id:[default]
Please input the Pod Id:[default]
Please input the Cluster Id:[default]
Please choose which network used to create VM:[cloudbr0]
Starting to configure your system:
Configure Apparmor ... [OK]
Configure Network ... [OK]
Configure Libvirt ... [OK]
Configure Firewall ... [OK]
Configure Nfs ... [OK]
Configure cloudAgent ... [OK]
CloudStack Agent setup is done!
root@sh-saas-cs-ceph-online-01:~#
root@sh-saas-cs-ceph-online-01:~# /etc/init.d/libvirt-bin restart
安装cloudstack agent端安装
安装jdk1.8
root@sh-saas-cs-ceph-online-01:~# add-apt-repository ppa:openjdk-r/ppa
root@sh-saas-cs-ceph-online-01:~# apt-get update
root@sh-saas-cs-ceph-online-01:~# apt-get install openjdk-8-jdk
添加java环境变量
在 /etc/profile和~/.bashrc中
增加
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
root@sh-saas-cs-ceph-online-01:~# java -version
openjdk version "1.8.0_162"
添加cloudstack源站源
root@sh-saas-cs-ceph-online-01:~# echo deb http://cloudstack.apt-get.eu/ubuntu trusty 4.10 | tee /etc/apt/sources.list.d/cloudstack.list
root@sh-saas-cs-ceph-online-01:~# wget -O - http://cloudstack.apt-get.eu/release.asc | sudo apt-key add -
添加cloudstack国内源
root@sh-saas-cs-ceph-online-01:~# echo deb http://mirrors.cloudstack-china.com/mirror/ubuntu trusty 4.3 | tee /etc/apt/sources.list.d/cloudstack.list
root@sh-saas-cs-ceph-online-01:~# wget -O – 'http://cloudstack.apt-get.eu/release.asc' | apt-key add -
2. 安装cloudstack agent
root@sh-saas-cs-ceph-online-01:~# apt-get update
root@sh-saas-cs-ceph-online-01:~# sudo apt-get install cloudstack-agent -y
修改cloudstack agent配置
root@sh-saas-cs-ceph-online-01:~# vim /etc/cloudstack/agent/agent.properties #增加以下内容，cloudbr0表示网桥端口
public.network.device=cloudbr0
private.network.device=cloudbr0
root@sh-saas-cs-ceph-online-01:~# sudo /etc/init.d/cloudstack-agent restart
全局参数调整
secstorage.allowed.internal.sites ---> 10.16.144.0/8
network.throttling.rate ---> 0
vm.network.throttling.rate --->0
nfs主存储问题
部署好cloudstack需要删除nfs主存储，后续nfs主存储异常会导致整个机器物理机重启(切记)
jzGH1Sx0hjMznV6F
FWKKfDxT30NflxPg