windows下基于docker單機部署ceph集群

1. 創建ceph專用網絡

docker network create --driver bridge --subnet 172.20.0.0/16 ceph-network

2. 查看是否創建成功（查看創建狀態）

docker network inspect ceph-network

3. 拉取鏡像：(鏡像源自行選擇)

docker pull ceph/daemon:latest-luminous
# 查看鏡像
docker images

4. 搭建 monitor 節點（監控集群狀態）

# 創建本地目錄
New-Item -ItemType Directory -Force -Path ./etc/ceph
New-Item -ItemType Directory -Force -Path ./var/lib/ceph
New-Item -ItemType Directory -Force -Path ./var/log/ceph# 運行容器
docker run -d --name ceph-mon --network ceph-network --ip 172.20.0.10 -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_IP=172.20.0.10 -e MON_NAME=ceph-mon -e CEPH_PUBLIC_NETWORK=172.20.0.0/16 -v ${PWD}/etc/ceph:/etc/ceph -v ${PWD}/var/lib/ceph/:/var/lib/ceph/ -v ${PWD}/var/log/ceph/:/var/log/ceph/ ceph/daemon:latest-luminous mon

5. 搭建 osd 節點（對象存儲節點）

# 執行命令獲取密鑰環
docker exec ceph-mon ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring# 修改配置文件，使用 PowerShell 追加內容到文件
$configPath = "${PWD}/etc/ceph/ceph.conf"
"osd max object name len = 256`nosd max object namespace len = 64" | Out-File -FilePath $configPath -Append -Encoding UTF8# 創建 OSD 本地目錄
New-Item -ItemType Directory -Force -Path ./var/lib/ceph/osd/1
New-Item -ItemType Directory -Force -Path ./var/lib/ceph/osd/2
New-Item -ItemType Directory -Force -Path ./var/lib/ceph/osd/3# 分別啟動三個容器來模擬集群
docker run -d --privileged=true --name ceph-osd-1 --network ceph-network --ip 172.20.0.11 -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=ceph-mon -e MON_IP=172.20.0.10 -e OSD_TYPE=directory -v ${PWD}/etc/ceph:/etc/ceph -v ${PWD}/var/lib/ceph/:/var/lib/ceph/ -v ${PWD}/var/lib/ceph/osd/1:/var/lib/ceph/osd ceph/daemon:latest-luminous osd
docker run -d --privileged=true --name ceph-osd-2 --network ceph-network --ip 172.20.0.12 -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=ceph-mon -e MON_IP=172.20.0.10 -e OSD_TYPE=directory -v ${PWD}/etc/ceph:/etc/ceph -v ${PWD}/var/lib/ceph/:/var/lib/ceph/ -v ${PWD}/var/lib/ceph/osd/2:/var/lib/ceph/osd ceph/daemon:latest-luminous osd
docker run -d --privileged=true --name ceph-osd-3 --network ceph-network --ip 172.20.0.13 -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=ceph-mon -e MON_IP=172.20.0.10 -e OSD_TYPE=directory -v ${PWD}/etc/ceph:/etc/ceph -v ${PWD}/var/lib/ceph/:/var/lib/ceph/ -v ${PWD}/var/lib/ceph/osd/3:/var/lib/ceph/osd ceph/daemon:latest-luminous osd

6. 搭建 mgr 節點（管理節點，可打開管理面板）

docker run -d --privileged=true --name ceph-mgr --network ceph-network --ip 172.20.0.14 -e CLUSTER=ceph -p 7000:7000 --pid=container:ceph-mon -v ${PWD}/etc/ceph:/etc/ceph -v ${PWD}/var/lib/ceph/:/var/lib/ceph/ ceph/daemon:latest-luminous mgr# 開啟管理界面
docker exec ceph-mgr ceph mgr module enable dashboard

7. 搭建 rgw 節點（ceph網關，可以搭建多個）

# 執行命令獲取密鑰環
docker exec ceph-mon ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring# 運行 RGW 容器
docker run -d --privileged=true --name ceph-rgw --network ceph-network --ip 172.20.0.15 -e CLUSTER=ceph -e RGW_NAME=ceph-rgw -p 7480:7480 -v ${PWD}/var/lib/ceph/:/var/lib/ceph/ -v ${PWD}/etc/ceph:/etc/ceph ceph/daemon:latest-luminous rgw

8. 檢查 ceph 狀態

docker exec ceph-mon ceph -s

集群已經部署完畢，下面是用于開發的步驟：

9. 測試添加 rgw 用戶，生成 access_key 和 secret_key 用于訪問

docker exec ceph-rgw radosgw-admin user create --uid="test" --display-name="test user"

生成結果如：

PS C:\WINDOWS\system32> docker exec ceph-rgw radosgw-admin user create --uid="test" --display-name="test user"
{"user_id": "test","display_name": "test user","email": "","suspended": 0,"max_buckets": 1000,"auid": 0,"subusers": [],"keys": [{"user": "test","access_key": "7367PT68J5OCWA84XXXXX","secret_key": "ieAuUzUJJ9Wa9SBTGSnloaIjS8eD8p9z2zXXXXX"}],"swift_keys": [],"caps": [],"op_mask": "read, write, delete","default_placement": "","placement_tags": [],"bucket_quota": {"enabled": false,"check_on_raw": false,"max_size": -1,"max_size_kb": 0,"max_objects": -1},"user_quota": {"enabled": false,"check_on_raw": false,"max_size": -1,"max_size_kb": 0,"max_objects": -1},"temp_url_keys": [],"type": "rgw"
}

開發使用：
可以直接通過代碼進行操作：

package cephimport ("gopkg.in/amz.v1/aws""gopkg.in/amz.v1/s3""log"cfg "filestore-server/config"
)var cephConn *s3.S3// GetCephConnection : 獲取ceph連接
func GetCephConnection() *s3.S3 {if cephConn != nil {return cephConn}// 1. 初始化ceph的一些信息auth := aws.Auth{AccessKey: cfg.Config.CephConfig.CephAccessKey,SecretKey: cfg.Config.CephConfig.CephSecretKey,}curRegion := aws.Region{Name:                 "default",EC2Endpoint:          cfg.Config.CephConfig.CephGWEndpoint,S3Endpoint:           cfg.Config.CephConfig.CephGWEndpoint,S3BucketEndpoint:     "",S3LocationConstraint: false,S3LowercaseBucket:    false,Sign:                 aws.SignV2,}// 2. 創建S3類型的連接return s3.New(auth, curRegion)
}// GetCephBucket : 獲取指定的bucket對象
func GetCephBucket(bucket string) *s3.Bucket {conn := GetCephConnection()b := conn.Bucket(bucket)// 檢查存儲桶是否存在_, err := b.List("", "", "", 0)if err != nil {if s3err, ok := err.(*s3.Error); ok && s3err.StatusCode == 404 {log.Printf("存儲桶 %s 不存在，嘗試創建...", bucket)// 創建存儲桶err = b.PutBucket(s3.PublicRead)if err != nil {log.Printf("創建存儲桶 %s 失敗: %v", bucket, err)return nil}log.Printf("存儲桶 %s 創建成功", bucket)} else {log.Printf("訪問存儲桶 %s 出錯: %v", bucket, err)return nil}}return b
}// PutObject : 上傳文件到ceph集群
func PutObject(bucket string, path string, data []byte) error {buck := GetCephBucket(bucket)return buck.Put(path, data, "octet-stream", s3.PublicRead)
}