Spaces:
Runtime error
Runtime error
Yixin Liu
commited on
Commit
·
8963c43
1
Parent(s):
0fa848f
upload
Browse files
config.py
CHANGED
@@ -93,7 +93,7 @@ function update_device_idx {
|
|
93 |
useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
94 |
utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
95 |
|
96 |
-
if [ $useage -ge $((
|
97 |
echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
|
98 |
sleep 3
|
99 |
continue
|
|
|
93 |
useage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
94 |
utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i ${available_devices[$current_device_idx]})
|
95 |
|
96 |
+
if [ $useage -ge $((total_gpu_memory-max_gpu_memory_gap)) ] || [ $utilization -ge $max_gpu_utilization ]; then
|
97 |
echo "device ${available_devices[$current_device_idx]} is fully booked, try next one"
|
98 |
sleep 3
|
99 |
continue
|