Ceph 使用 TGT 创建 ISCSI target 服务器，让 Windows 可以通过 ISCSI 来使用 Ceph

编译 TGT

拉取源码

git clone https://github.com/fujita/tgt.git  /source_code/tgt

启动编译容器

docker run -ti --name tgt -v /source_code/tgt/:/tgt ubuntu

修改镜像源为阿里源

cat > /etc/apt/sources.list << EOF
#添加阿里源
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF

安装依赖包

apt update && apy install -y librbd-dev librados-dev

编译

cd /source_code/tgt && CEPH_RBD=1 make
mkdir -p /usr/lib/tgt/backing-store
cp -f /source_code/tgt/usr/tgt{d,adm,img} /source_code/tgt/bin
cp -f /source_code/tgt/usr/bs_rbd.so /source_code/tgt/bin/backing-store

制作 TGT 容器

这里偷个懒，不写 Dockerfile了

启动临时容器

docker run -ti --name tgt -v /source_code/tgt/:/tgt ubuntu

安装依赖包

cat > /etc/apt/sources.list << EOF
#添加阿里源
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF

apt update && apt install -y librbd1
rm -rf /var/lib/apt/lists/*

复制 TGT 二进制文件

mkdir -p /usr/lib/tgt/backing-store
cp -f /tgt/bin/tgt{d,adm,img} /sbin/
cp -f /tgt/bin/backing-store/* /usr/lib/tgt/backing-store

创建启动入口脚本

cat > /entrypoint.sh << EOF
#!/bin/bash

TGTADM=tgtadm
TGT_TID=1
TGT_LUNS=\${TGT_LUNS:-none}
TGT_BINDS=\${TGT_BINDS:-ALL}
TGT_IQNNAME=\${TGT_IQNNAME:-"iqn.2020-02.iscsi.tgt:default"}

if [[ "\${TGT_LUNS}" == "none" ]]; then
  echo "ERROR- could not find the lun devices, did you configure TGT luns?" >/dev/stderr
  echo "ERROR- use -e TGT_LUNS=\"rbd:rbd/dev-data rbd:rbd/dev-source\"" >/dev/stderr
  exit 1
fi

tgtd -f&

while [ ! -f /var/run/tgtd/socket.0.lock ]; do
    sleep 0.1s
done

\${TGTADM} --lld iscsi --mode target --op new --tid \${TGT_TID} --targetname \${TGT_IQNNAME}

TGT_LUNID=0
for TGT_LUN in \${TGT_LUNS}; do
  let TGT_LUNID++ # Start to number 1
  TGT_TYPE="\$(echo "\${TGT_LUN}"|sed 's/\(.*\):\(.*\)/\1/')"
  TGT_STORE="\$(echo "\${TGT_LUN}"|sed 's/\(.*\):\(.*\)/\2/')"
  \${TGTADM} --lld iscsi --mode logicalunit --op new --tid \${TGT_TID} --lun \${TGT_LUNID} --backing-store \${TGT_STORE} --bstype \${TGT_TYPE}
done

for TGT_BIND in \${TGT_BINDS}; do
  \${TGTADM} --lld iscsi --mode target --op bind --tid 1 -I \${TGT_BIND}
done

\${TGTADM} --lld iscsi --mode target --op show

trap 'exit -1;' INT

wait
EOF

chmod +x /entrypoint.sh

生成镜像并删除临时容器

docker commit --change='ENTRYPOINT ["/entrypoint.sh"]' tgt xxx/tgt
docker rm tgt

到这里 TGT 的 docker 镜像就做好了

启动

这里挂载两块硬盘，rbd/disk-dev_data 和 rbd/disk-dev_source.

rbd 是 RBD 块 所在 pool 的名字
disk-dev_data 和 disk-dev_source 是 RBD 块 的名字

docker run -ti -p 3260:3260 -v /etc/ceph/:/etc/ceph -e TGT_LUNS="rbd:rbd/disk-dev_data rbd:rbd/disk-dev_source" -e TGT_IQNNAME="--targetname iqn.2020-02.rbd:disk" ssoor/tgt