File size: 2,038 Bytes
bbfa6f6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
#!/bin/bash
# cd /mnt/bn/algo-masp-nas-2/xiangchen/repo/LLaVA
cd /opt/tiger/masp_models
pip install --upgrade pip
pip install -e .
echo "$PWD"
ports=(`echo $METIS_WORKER_0_PORT | tr ',' ' '`)
port=${ports[0]}
echo "total workers: ${ARNOLD_WORKER_NUM}"
echo "cur worker id: ${ARNOLD_ID}"
echo "gpus per worker: ${ARNOLD_WORKER_GPU}"
echo "master ip: ${METIS_WORKER_0_HOST}"
echo "master port: ${port}"
#export OMP_NUM_THREADS=8
#export NCCL_IB_DISABLE=0
#export NCCL_IB_GID_INDEX=3
#export NCCL_IB_HCA=${ARNOLD_RDMA_DEVICE}
#export NCCL_SOCKET_IFNAME=eth0
# export NCCL_DEBUG=INFO
env="$1"
cmd="$2"
echo $env
echo $cmd
deepspeed \
--num_nodes=$ARNOLD_WORKER_NUM \
--num_gpus=$ARNOLD_WORKER_GPU \
--master_port=$port \
--master_addr $METIS_WORKER_0_HOST \
llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path mistralai/Mistral-7B-Instruct-v0.1 \
--version plain \
--dataset_config /mnt/bn/algo-masp-nas-2/xiangchen/repo/LLaVA/llava/configs/pretrain_data.yaml \
--vision_tower google/siglip-large-patch16-256 \
--tune_mm_mlp_adapter True \
--mm_vision_select_layer -2 \
--mm_use_start_end True \
--mm_use_patch_token False \
--image_aspect_ratio pad \
--num_token_per_image 256 \
--num_query_token 256 \
--bf16 True \
--output_dir /mnt/bn/algo-masp-nas-2/xiangchen/model/masp_models/checkpoints/llava-pretrain-googlesiglip_projector \
--group_by_modality_length True \
--num_train_epochs 1 \
--per_device_train_batch_size 8 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 4 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 2000 \
--save_total_limit 1 \
--learning_rate 5e-4 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 4096 \
--gradient_checkpointing True \
--dataloader_num_workers 1 \
--lazy_preprocess True \
--report_to none |