Ubuntu14.04.5 Ceph [jewel] 快速部署
The ceph-deploy tool operates out of a directory on an admin node. Any host with network connectivity and a modern python environment and ssh (such as Linux) should work.
|| 所有节点
域名 & 时间
# vim /etc/hosts
10.50.50.137 ceph-admin
10.50.50.134 ceph01
10.50.50.135 ceph02
10.50.50.136 ceph03
# apt-get install ntp -y
配置同一个用户 zeastion
# useradd -d /home/zeastion -m zeastion
# passwd zeastion
增加该用户 sudo 权限（免密码）
# echo "zeastion ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/zeastion
# chmod 0440 /etc/sudoers.d/zeastion
|| OSD 节点
创建 OSD 守护进程目录
注：以下命令在部署节点第6步后执行
- 节点 ceph02
# mkdir /var/local/osd0
# chown ceph:ceph /var/local/osd0/
- 节点 ceph03
# mkdir /var/local/osd1
# chown ceph:ceph /var/local/osd1/
|| 部署节点 ceph-admin
0- 清理环境（可选）
清理数据
# ceph-deploy purgedata ceph01 ceph02 ceph03
# ceph-deploy forgetkeys
若要连安装包一并清除
# ceph-deploy purge ceph01 ceph02 ceph03
1- 准备工作
添加源&安装
# wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
OK
# echo deb http://download.ceph.com/debian-jewel/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
deb http://download.ceph.com/debian-jewel/ trusty main
# apt-get update
# apt-get install ceph-deploy -y
切换用户
# su zeastion
无密码登陆
# ssh-keygen
# ssh-copy-id zeastion@ceph-admin
# ssh-copy-id zeastion@ceph01
# ssh-copy-id zeastion@ceph02
# ssh-copy-id zeastion@ceph03
测试
# ssh zeastion@ceph-admin
# ssh zeastion@ceph01
# ssh zeastion@ceph02
# ssh zeastion@ceph03
让 ceph-deploy 使用所建用户登陆 ceph 节点
# vim ~/.ssh/config
Host ceph-admin
Hostname ceph-admin
User zeastion
Host ceph01
Hostname ceph01
User zeastion
Host ceph02
Hostname ceph02
User zeastion
Host ceph03
Hostname ceph03
User zeastion
创建 ceph-deploy 目录，保存生成的配置文件和密钥对
# mkdir my-cluster && cd my-cluster
2- 初始化 MON 节点
# ceph-deploy new ceph01
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/zeastion/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.38): /usr/bin/ceph-deploy new ceph01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] func : <function new at 0x7f7beaa85de8>
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f7bea40a128>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] mon : ['ceph01']
[ceph_deploy.cli][INFO ] public_network : None
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster_network : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph01][DEBUG ] connected to host: ceph-admin
[ceph01][INFO ] Running command: ssh -CT -o BatchMode=yes ceph01
[ceph01][DEBUG ] connection detected need for sudo
[ceph01][DEBUG ] connected to host: ceph01
[ceph01][DEBUG ] detect platform information from remote host
[ceph01][DEBUG ] detect machine type
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: sudo /sbin/initctl version
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: sudo /bin/ip link show
[ceph01][INFO ] Running command: sudo /bin/ip addr show
[ceph01][DEBUG ] IP addresses found: [u'10.50.50.134']
[ceph_deploy.new][DEBUG ] Resolving host ceph01
[ceph_deploy.new][DEBUG ] Monitor ceph01 at 10.50.50.134
[ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph01']
[ceph_deploy.new][DEBUG ] Monitor addrs are ['10.50.50.134']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...
多出几个文件
# ls
ceph.conf ceph-deploy-ceph.log ceph.mon.keyring
3- 修改配置文件
目前只有两个 OSD 节点，把副本数改为 2
添加公共网 public network
# vim ceph.conf
[global]
fsid = f866ace3-5a12-4691-b171-95d57d419396
mon_initial_members = ceph01
mon_host = 10.50.50.134
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
osd pool default size = 2
public network = 10.50.50.0/24
注：由于网络问题，从 ceph 官方下载软件包速度很慢，替换成网易源
# export CEPH_DEPLOY_REPO_URL=http://mirrors.163.com/ceph/debian-jewel
# export CEPH_DEPLOY_GPG_URL=http://mirrors.163.com/ceph/keys/release.asc
4- 安装 ceph
# ceph-deploy install ceph-admin ceph01 ceph02 ceph03
5- 配置初始 MON ，收集所有密钥
# ceph-deploy mon create-initial
查看新增文件
# ls ~/my-cluster/
ceph.bootstrap-mds.keyring ceph.bootstrap-rgw.keyring ceph-deploy-ceph.log
ceph.bootstrap-mgr.keyring ceph.client.admin.keyring ceph.mon.keyring
ceph.bootstrap-osd.keyring ceph.conf release.asc
6- 部署 OSD 节点
准备
# ceph-deploy osd prepare ceph02:/var/local/osd0 ceph03:/var/local/osd1
激活
# ceph-deploy osd activate ceph02:/var/local/osd0 ceph03:/var/local/osd1
7- 同步配置文件 & admin 密钥
# ceph-deploy admin ceph-admin ceph01 ceph02 ceph03
8- 检查
# sudo chmod +r /etc/ceph/ceph.client.admin.keyring
# ceph health
HEALTH_OK
# ceph -s
cluster 5651d8c0-317f-404f-b22f-21f5496f1b07
health HEALTH_OK
monmap e1: 1 mons at {ceph01=10.50.50.134:6789/0}
election epoch 3, quorum 0 ceph01
osdmap e10: 2 osds: 2 up, 2 in
flags sortbitwise,require_jewel_osds
pgmap v20: 64 pgs, 1 pools, 0 bytes data, 0 objects
14201 MB used, 39176 MB / 53377 MB avail
64 active+clean
|| 集群扩展
测试在 ceph01 上添加 OSD & MDS 守护进程，在 ceph02 ceph03 上添加 MON 守护进程，
1- 把 ceph01 加入 OSD
- ceph01 节点
# mkdir /var/local/osd2
# chown ceph:ceph /var/local/osd2/
- ceph-admin 节点
准备
# ceph-deploy osd prepare ceph01:/var/local/osd2
激活
# ceph-deploy osd activate ceph01:/var/local/osd2
查看
# ceph -s
cluster 5651d8c0-317f-404f-b22f-21f5496f1b07
health HEALTH_OK
monmap e1: 1 mons at {ceph01=10.50.50.134:6789/0}
election epoch 3, quorum 0 ceph01
osdmap e16: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v82: 64 pgs, 1 pools, 0 bytes data, 0 objects
21304 MB used, 58762 MB / 80066 MB avail
64 active+clean
2- 在 ceph01 添加 MDS
# ceph-deploy mds create ceph01
3- 在 ceph01 添加 RGW
# ceph-deploy rgw create ceph01
ceph01 上的 RGW 进程默认监听 7480 端口
4- 把 ceph02 ceph03 加入 MON
集群中需要多个 MON，实现高可用（PASOX 算法）
# ceph-deploy mon add ceph02
# ceph-deploy mon add ceph03
# ceph -s
cluster 5651d8c0-317f-404f-b22f-21f5496f1b07
health HEALTH_OK
monmap e3: 3 mons at {ceph01=10.50.50.134:6789/0,ceph02=10.50.50.135:6789/0,ceph03=10.50.50.136:6789/0}
election epoch 6, quorum 0,1,2 ceph01,ceph02,ceph03
osdmap e16: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v146: 64 pgs, 1 pools, 0 bytes data, 0 objects
21329 MB used, 58737 MB / 80066 MB avail
64 active+clean
检查法定人数状态
# ceph quorum_status --format json-pretty
{
"election_epoch": 8,
"quorum": [
0,
1,
2
],
"quorum_names": [
"ceph01",
"ceph02",
"ceph03"
],
"quorum_leader_name": "ceph01",
"monmap": {
"epoch": 3,
"fsid": "5651d8c0-317f-404f-b22f-21f5496f1b07",
"modified": "2017-08-31 15:59:36.754149",
"created": "2017-08-31 09:50:05.582547",
"mons": [
{
"rank": 0,
"name": "ceph01",
"addr": "10.50.50.134:6789\/0"
},
{
"rank": 1,
"name": "ceph02",
"addr": "10.50.50.135:6789\/0"
},
{
"rank": 2,
"name": "ceph03",
"addr": "10.50.50.136:6789\/0"
}
]
}
}
|| 测试存入/检出对象数据
1- 创建存储池 ‘mydata’
# ceph osd pool create mydata 20
pool 'mydata' created
# ceph osd lspools
0 rbd,1 .rgw.root,2 default.rgw.control,3 default.rgw.data.root,4 default.rgw.gc,5 default.rgw.log,6 mydata,
2- 创建一个对象并存储到 mydata pool
# echo myfirsttest > testfile.txt
# rados put test-object-1 testfile.txt --pool=mydata
检查
# rados -p mydata ls
test-object-1
定位
# ceph osd map mydata test-object-1
osdmap e34 pool 'mydata' (6) object 'test-object-1' -> pg 7.74dc35e2 (7.2) -> up ([1,2], p1) acting ([1,2], p1)
3- 删除
# rados rm test-object-1 --pool=mydata
