• 兩臺服務器

1. Docker容器中使用GPU

• 必須確保已安裝并配置 NVIDIA Docker。你可以安裝 nvidia-docker 來確保 GPU 驅動能夠被 Docker 使用

#安裝 nvidia-docker：
sudo apt-get install nvidia-docker2#然后重啟 Docker：
sudo systemctl restart docker

2.下載vllm-openai鏡像

• 當前latest版本： v0.7.2

docker pull vllm/vllm-openai:latest

3. 啟動vllm-openai容器

• 會啟動一個node容器

3.1 啟動腳本

# master  服務器1執行
sudo bash run_cluster.sh \vllm/vllm-openai \  # 鏡像名稱172.16.3.38 \  # head服務器IP--head \   # 代表head/home/llm/ai/model \  # huggingface 模型路徑-v /home/llm/ai/model:/home/llm/ai/model \   # 宿主機本地模型映射到容器內-e GLOO_SOCKET_IFNAME=enp4s0 \  # 服務器IP對應的網卡名稱-e NCCL_SOCKET_IFNAME=enp4s0 \  # 服務器IP對應的網卡名稱-e VLLM_HOST_IP=172.16.3.38# worker  服務器2執行
bash run_cluster.sh \vllm/vllm-openai \172.16.3.38 \--worker \  # 代表 從服務器/home/llm/ai/model \-v /home/llm/ai/model:/home/llm/ai/model \-e GLOO_SOCKET_IFNAME=enp5s0 \-e NCCL_SOCKET_IFNAME=enp5s0 \-e VLLM_HOST_IP=172.16.3.37

• run_cluster.sh

#!/bin/bash# Check for minimum number of required arguments
if [ $# -lt 4 ]; thenecho "Usage: $0 docker_image head_node_address --head|--worker path_to_hf_home [additional_args...]"exit 1
fi# Assign the first three arguments and shift them away
DOCKER_IMAGE="$1"
HEAD_NODE_ADDRESS="$2"
NODE_TYPE="$3"  # Should be --head or --worker
PATH_TO_HF_HOME="$4"
shift 4# Additional arguments are passed directly to the Docker command
ADDITIONAL_ARGS=("$@")# Validate node type
if [ "${NODE_TYPE}" != "--head" ] && [ "${NODE_TYPE}" != "--worker" ]; thenecho "Error: Node type must be --head or --worker"exit 1
fi# Define a function to cleanup on EXIT signal
cleanup() {docker stop nodedocker rm node
}
trap cleanup EXIT# Command setup for head or worker node
RAY_START_CMD="ray start --block"
if [ "${NODE_TYPE}" == "--head" ]; thenRAY_START_CMD+=" --head --port=6379"
elseRAY_START_CMD+=" --address=${HEAD_NODE_ADDRESS}:6379"
fi# Run the docker command with the user specified parameters and additional arguments
docker run \--entrypoint /bin/bash \--network host \--name node \--shm-size 10.24g \--gpus all \-v "${PATH_TO_HF_HOME}:/root/.cache/huggingface" \"${ADDITIONAL_ARGS[@]}" \"${DOCKER_IMAGE}" -c "${RAY_START_CMD}"

4. 進入到容器內部加載模型

# 進入容器內部
docker exec -it node /bin/bash# tensor-parallel-size 每臺服務器顯卡數量
# pipeline-parallel-size 服務器數量
vllm serve /home/llm/ai/model/Qwen/Qwen1___5-1___8B-Chat --port 8080 --tensor-parallel-size 1 --pipeline-parallel-size 2  --dtype float16

5. 查看集群狀態

• 容器內執行 可以看到兩個節點
  ray status