一、準備
5臺centos7主機?
node節點雙網卡
（1個內部檢測，1個外部使用）
node節點都添加新網卡
關閉防火墻和上下文
都需要
添加hosts文件
都需要
cat > /etc/hosts << EOF
> 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
> ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
> 192.168.11.200 admin
> 192.168.11.201 node01
> 192.168.11.202 node02
> 192.168.11.203 node03
> 192.168.11.204 client
> EOF
admin對node免密
配時間同步
timedatectl set-timezone Asia/Shanghai
timedatectl set-ntp yes###
ntpdate ntp.aliyun.com
創建工作目錄
配置下載源
wget https://download.ceph.com/rpm-nautilus/el7/noarch/ceph-release-1-1.el7.noarch.rpm --no-check-certificaterpm -ivh ceph-release-1-1.el7.noarch.rpm --force
安裝軟件包及依賴
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repocurl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repoyum clean allyum makecacheyum -y install epel-releaseyum -y install yum-plugin-priorities yum-utils ntpdate python-setuptools python-pip gcc gcc-c++ autoconf libjpeg libjpeg-devel libpng libpng-devel freetype freetype-devel libxml2 libxml2-devel zlib zlib-devel glibc glibc-devel glib2 glib2-devel bzip2 bzip2-devel zip unzip ncurses ncurses-devel curl curl-devel e2fsprogs e2fsprogs-devel krb5-devel libidn libidn-devel openssl openssh openssl-devel nss_ldap openldap openldap-devel openldap-clients openldap-servers libxslt-devel libevent-devel ntp libtool-ltdl bison libtool vim-enhanced python wget lsof iptraf strace lrzsz kernel-devel kernel-headers pam-devel tcl tk cmake ncurses-devel bison setuptool popt-devel net-snmp screen perl-devel pcre-devel net-snmp screen tcpdump rsync sysstat man iptables sudo libconfig git bind-utils tmux elinks numactl iftop bwm-ng net-tools expect snappy leveldb gdisk python-argparse gperftools-libs conntrack ipset jq libseccomp socat chrony sshpass
在admin執行
下載失敗可手動下載
admin與node
yum clean allyum mackcacheyum -y install epel-releaseyum -y install yum-plugin-prioritiesyum -y install ceph-release ceph ceph-radosgw
生成初始配置?
ceph-deploy new --public-network 192.168.11.0/24 --cluster-network 192.168.100.0/24 node01 node02 node03
假如出現
cat > /usr/lib/python2.7/site-packages/ceph_deploy/util/ssh.py << 'EOF'
# -*- coding: utf-8 -*-
import logging
from ceph_deploy.lib import remoto
from ceph_deploy.connection import get_local_connectiondef can_connect_passwordless(hostname):"""Ensure that current host can SSH remotely to the remotehost using the ``BatchMode`` option to prevent a password prompt.That attempt will error with an exit status of 255 and a ``Permissiondenied`` message or a``Host key verification failed`` message."""# 直接返回True，跳過needs_ssh檢查（已確認SSH正常）return True# 以下代碼會被上面的return跳過，保留僅作參考logger = logging.getLogger(hostname)with get_local_connection(logger) as conn:# Check to see if we can login, disabling password promptscommand = ['ssh', '-CT', '-o', 'BatchMode=yes', hostname]out, err, retval = remoto.process.check(conn, command, stop_on_error=False)permission_denied_error = 'Permission denied 'host_key_verify_error = 'Host key verification failed.'has_key_error = Falsefor line in err:if permission_denied_error in line or host_key_verify_error in line:has_key_error = Trueif retval == 255 and has_key_error:return Falsereturn True
EOF同步配置
ceph-deploy --overwrite-conf mon create-initial
初始化mon節點
ceph-deploy mon create node01 node02 node03
部署mgr
ceph-deploy mgr create node{01..03}
查看ceph集群狀態
查看 mon 集群選舉的情況
ceph quorum_status --format json-pretty | grep leader
擦凈（刪除分區表）磁盤
ceph-deploy disk zap node01 /dev/sdb
ceph-deploy disk zap node02 /dev/sdb
ceph-deploy disk zap node03 /dev/sdb
添加osd節點
ceph-deploy --overwrite-conf osd create node01 --data /dev/sdb
ceph-deploy --overwrite-conf osd create node02 --data /dev/sdb
ceph-deploy --overwrite-conf osd create node03 --data /dev/sdb
查看狀態
開啟監控模塊
查看主節點
ceph -s | grep mgr
在主節點上執行?
yum install -y ceph-mgr-dashboard
[root@admin ceph]# ceph config set mgr mgr/dashboard/ssl false
[root@admin ceph]# ceph config set mgr mgr/dashboard/server_addr 0.0.0.0
[root@admin ceph]# ceph config set mgr mgr/dashboard/server_port 8000[root@admin ceph]# ceph mgr module disable dashboard
[root@admin ceph]# ceph mgr module enable dashboard --force
[root@admin ceph]# ceph mgr services
設置密碼與用戶?
[root@admin ceph]# echo "123.com" > dashboard_passwd.txt
[root@admin ceph]# ceph dashboard set-login-credentials admin -i dashboard_passwd.txt
資源池 Pool 管理
創建pool，作為數據池
ceph osd pool create mypool 64 64
安裝客戶工具
ceph-deploy install client
客戶端使用ceph
admin節點操作，安裝客戶端工具
[root@admin ceph]# ceph-deploy install client
admin節點操作，同步管理信息
ceph-deploy admin client
在client節點操作，安裝客戶端工具
創建 元數據池
ceph osd pool create cephfs_meta 64ceph osd pool application enable cephfs_meta cephfs 
使用 資源池與 元資源池 關聯 創建 cephfs
使用 2個pool 一個作為資源池 （大容量）一個作為 元資源池 （小容量）
ceph fs new cephfs cephfs_meta mypool
ceph fs new <文件系統名稱> <元數據池名稱> <數據池名稱>
掛載文件?
ceph-fuse -k /etc/ceph/ceph.client.admin.keyring -m 192.168.11.201:6789 /ceph
)