|
#!/bin/bash |
|
|
|
a=$(echo $HOSTNAME | cut -c12-16) |
|
|
|
CONFIG=$1 |
|
JOB_NAME=${2:-"experiments"} |
|
GPUS=${3:-8} |
|
|
|
SRUN=${4:-'reserved'} |
|
|
|
GPUS_PER_NODE=${GPUS:-8} |
|
if [ $GPUS_PER_NODE -ge 8 ]; then |
|
GPUS_PER_NODE=8 |
|
fi |
|
CPUS_PER_TASK=${CPUS_PER_TASK:-4} |
|
SRUN_ARGS=${SRUN_ARGS:-""} |
|
|
|
PY_ARGS=${@:5} |
|
|
|
WORK_DIR=${CONFIG//configs/work_dirs} |
|
WORK_DIR=${WORK_DIR//.yaml//$JOB_NAME} |
|
echo $WORK_DIR |
|
mkdir -p $WORK_DIR |
|
mkdir -p data/temp |
|
|
|
now=$(date +"%Y%m%d_%H%M%S") |
|
|
|
a=$(echo $HOSTNAME | cut -c12-16) |
|
|
|
|
|
if [ $a == '140-0' ]; then |
|
export DATA_PATH='/mnt/lustre/share_data/zhujinguo' |
|
export LD_LIBRARY_PATH=/mnt/cache/zhujinguo/anaconda3/envs/py36/lib:$LD_LIBRARY_PATH |
|
export TORCH_EXTENSIONS_DIR='/mnt/lustre/zhujinguo/.cache/torch_extensions' |
|
export NO_NVRTC=0 |
|
partition='INTERN' |
|
CEPH_CONFIG='slurm_tools/petreloss_1400.config' |
|
SRUNreal=${SRUN} |
|
|
|
if [ ${SRUN} == 'vcspot' ]; then |
|
SRUNreal='spot --async' |
|
partition=VC |
|
elif [ ${SRUN} == 'vcauto' ]; then |
|
SRUNreal='auto --async' |
|
partition=VC |
|
elif [ ${SRUN} == 'vcreserved' ]; then |
|
SRUNreal='reserved' |
|
partition=VC |
|
elif [ ${SRUN} == 'spot' ]; then |
|
SRUNreal='spot --async' |
|
elif [ ${SRUN} == 'auto' ]; then |
|
SRUNreal='auto --async' |
|
|
|
fi |
|
|
|
elif [ $a == '142-4' ]; then |
|
|
|
export DATA_PATH='/mnt/lustre/share_data/zhujinguo' |
|
export LD_LIBRARY_PATH=/mnt/cache/zhujinguo/anaconda3/envs/py36/lib:$LD_LIBRARY_PATH |
|
export TORCH_EXTENSIONS_DIR='/mnt/lustre/zhujinguo/.cache/torch_extensions' |
|
export NO_NVRTC=0 |
|
partition='vc_research_5' |
|
CEPH_CONFIG='slurm_tools/petreloss_1424.config' |
|
|
|
SRUNreal=${SRUN} |
|
|
|
if [ ${SRUN} == 'vc4spot' ]; then |
|
SRUNreal='spot --async' |
|
partition=vc_research_4 |
|
elif [ ${SRUN} == 'vc4auto' ]; then |
|
SRUNreal='auto --async -x SH-IDC1-10-142-4-76' |
|
partition=vc_research_4 |
|
elif [ ${SRUN} == 'vc4reserved' ]; then |
|
SRUNreal='reserved' |
|
partition=vc_research_4 |
|
elif [ ${SRUN} == 'spot' ]; then |
|
SRUNreal='spot --async' |
|
elif [ ${SRUN} == 'auto' ]; then |
|
SRUNreal='auto --async' |
|
fi |
|
|
|
else |
|
echo only SH1424 and SH1400 supported now |
|
|
|
fi |
|
|
|
srun --partition=${partition} $SRUN_ARGS --quotatype=${SRUNreal} -o $WORK_DIR/phoenix-slurm-%j-$now.out \ |
|
--job-name=${JOB_NAME} -n$GPUS --gres=gpu:${GPUS_PER_NODE} \ |
|
--ntasks-per-node=${GPUS_PER_NODE} \ |
|
--kill-on-bad-exit=1 --cpus-per-task 12 \ |
|
python -u main.py --num-gpus $GPUS \ |
|
--config-file ${CONFIG} --init_method slurm --resume \ |
|
${PY_ARGS} OUTPUT_DIR $WORK_DIR DATALOADER.USE_CEPH True \ |
|
DATALOADER.TCS_CONF_PATH $CEPH_CONFIG SOLVER.CHECKPOINT_PERIOD 10000 SOLVER.CHECKPOINT_MAX_SAVE 1 \ |
|
${OTHERARGS} 2>&1 |
|
|
|
|
|
|
|
|
