File size: 1,264 Bytes
0ad4cbc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
export BASE_MODEL_PATH='' # Base model path
export ALIGNER_MODEL_PATH='' # Aligner model path
export BASE_PORT=8011 # Base port
export ALIGNER_PORT=8013 # Aligner port
echo $BASE_MODEL_PATH
echo $ALIGNER_MODEL_PATH
echo $BASE_PORT
echo $ALIGNER_PORT
CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve $BASE_MODEL_PATH --host 0.0.0.0 --port $BASE_PORT --max-model-len 2048 --tensor-parallel-size 4 --api-key jiayi --trust-remote-code --dtype auto --enforce-eager --swap-space 1 &
CUDA_VISIBLE_DEVICES=4,5,6,7 vllm serve $ALIGNER_MODEL_PATH --host 0.0.0.0 --port $ALIGNER_PORT --max-model-len 2048 --tensor-parallel-size 4 --api-key jiayi --trust-remote-code --dtype auto --enforce-eager --swap-space 1
# vllm serve /aifs4su/hansirui/yaodong/models/DeepSeek-R1 --host 0.0.0.0 --port 8009 --max-model-len 12800 --tensor-parallel-size 16 --api-key jiayi --trust-remote-code --dtype auto --enforce-eager --enable-reasoning --reasoning-parser deepseek_r1 --swap-space 1
echo 'Base Port:' $BASE_PORT
echo 'Aligner Port:' $ALIGNER_PORT
# CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 vllm serve /aifs4su/yaodong/spring_r1_model/QVQ-72B-Preview --enable-reasoning --reasoning-parser deepseek_r1 --host 0.0.0.0 --port 8009 --max-model-len 12000 --tensor-parallel-size 8 --api-key jiayi |