diff --git "a/slurm/1038247.0.err" "b/slurm/1038247.0.err" new file mode 100644--- /dev/null +++ "b/slurm/1038247.0.err" @@ -0,0 +1,1156 @@ +2025-07-01 08:49:26.897 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:26.897 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:26.906 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.199 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.200 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.208 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.263 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.263 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.268 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.268 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.271 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.277 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.277 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.277 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.285 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.318 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.318 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.321 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.321 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.321 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.321 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.321 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.321 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.324 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.324 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.327 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.329 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.330 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.330 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.332 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.396 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.397 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.406 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.457 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.457 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.457 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.457 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.466 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.466 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.516 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.516 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.533 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.533 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.536 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.536 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.541 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.562 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.562 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.562 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.562 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.563 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.563 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.564 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.564 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.565 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.565 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.570 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.571 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.572 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.572 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.573 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.611 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.611 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.612 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.612 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.614 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.614 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.620 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.620 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.620 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.621 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.623 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:27.962 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:27.962 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:27.971 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.089 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.089 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.089 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.089 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.090 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.090 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.092 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.092 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.095 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.095 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.097 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.098 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.099 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.099 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.099 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.101 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.104 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.107 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.153 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.153 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.162 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.186 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.186 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.188 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.189 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.195 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.197 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.202 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.203 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.211 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.259 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.259 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.262 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.262 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.262 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.262 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.264 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.264 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.267 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.270 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.271 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.273 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.283 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.284 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.293 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.333 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.333 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.342 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +2025-07-01 08:49:28.461 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.461 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.462 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.462 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.470 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.470 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.470 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.471 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.472 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.472 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.474 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.474 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.476 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:28.477 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:28.479 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.481 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.482 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:28.485 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +2025-07-01 08:49:29.307 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.307 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.316 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.344 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.344 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.346 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.346 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.353 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.354 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.357 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.357 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.366 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.383 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.383 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +2025-07-01 08:49:29.392 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.396 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.396 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.404 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.469 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.470 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.470 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:29.470 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:29.478 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:29.479 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +2025-07-01 08:49:30.224 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.224 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.225 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.225 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.226 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.226 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.233 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:30.234 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:30.234 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +2025-07-01 08:49:30.336 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.336 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.343 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.343 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.344 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.344 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.345 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:30.347 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.347 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.351 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:30.352 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:49:30.354 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:49:30.354 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:49:30.356 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +2025-07-01 08:49:30.362 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! + Fetching 21 files: 0%| | 0/21 [00:00 +[rank0]: train() +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/train/train.py", line 906, in train +[rank0]: trainer.train(resume_from_checkpoint=resume_from_checkpoint) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer.py", line 2122, in train +[rank0]: return inner_training_loop( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/train/llava_trainer.py", line 665, in _inner_training_loop +[rank0]: return super()._inner_training_loop(batch_size, *args, **kwargs) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer.py", line 2381, in _inner_training_loop +[rank0]: self.control = self.callback_handler.on_train_begin(args, self.state, self.control) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer_callback.py", line 468, in on_train_begin +[rank0]: return self.call_event("on_train_begin", args, state, control) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer_callback.py", line 518, in call_event +[rank0]: result = getattr(callback, event)( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 911, in on_train_begin +[rank0]: self.setup(args, state, model, **kwargs) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 838, in setup +[rank0]: self._wandb.init( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1623, in init +[rank0]: wandb._sentry.reraise(e) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/analytics/sentry.py", line 156, in reraise +[rank0]: raise exc.with_traceback(sys.exc_info()[2]) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1551, in init +[rank0]: wi.maybe_login(init_settings) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 191, in maybe_login +[rank0]: wandb_login._login( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_login.py", line 315, in _login +[rank0]: key, key_status = wlogin.prompt_api_key(referrer=referrer) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_login.py", line 243, in prompt_api_key +[rank0]: raise UsageError("api_key not configured (no-tty). call " + directive) +[rank0]: wandb.errors.errors.UsageError: api_key not configured (no-tty). call wandb.login(key=[your_api_key]) +W0701 08:52:50.516000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086291 closing signal SIGTERM +W0701 08:52:50.517000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086292 closing signal SIGTERM +W0701 08:52:50.519000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086293 closing signal SIGTERM +W0701 08:52:50.520000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086294 closing signal SIGTERM +W0701 08:52:50.521000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086295 closing signal SIGTERM +W0701 08:52:50.522000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086296 closing signal SIGTERM +W0701 08:52:50.524000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2086297 closing signal SIGTERM +E0701 08:52:54.370000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 0 (pid: 2086290) of binary: /lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/bin/python3 +Traceback (most recent call last): + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/bin/torchrun", line 10, in + sys.exit(main()) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper + return f(*args, **kwargs) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main + run(args) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run + elastic_launch( + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-07-01_08:52:50 + host : pool0-01504.cm.cluster + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 2086290) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642507 closing signal SIGTERM +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080465 closing signal SIGTERM +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860941 closing signal SIGTERM +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512471 closing signal SIGTERM +W0701 08:52:54.885000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.886000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896464 closing signal SIGTERM +W0701 08:52:54.884000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.885000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139986 closing signal SIGTERM +W0701 08:52:54.885000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:52:54.885000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139987 closing signal SIGTERM +W0701 08:52:54.885000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008516 closing signal SIGTERM +W0701 08:52:54.885000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642508 closing signal SIGTERM +W0701 08:52:54.886000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860942 closing signal SIGTERM +W0701 08:52:54.886000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512472 closing signal SIGTERM +W0701 08:52:54.886000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080466 closing signal SIGTERM +W0701 08:52:54.886000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008517 closing signal SIGTERM +W0701 08:52:54.887000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896465 closing signal SIGTERM +W0701 08:52:54.887000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642509 closing signal SIGTERM +W0701 08:52:54.887000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860943 closing signal SIGTERM +W0701 08:52:54.887000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080467 closing signal SIGTERM +W0701 08:52:54.888000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512473 closing signal SIGTERM +W0701 08:52:54.889000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896466 closing signal SIGTERM +W0701 08:52:54.888000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008518 closing signal SIGTERM +W0701 08:52:54.888000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642510 closing signal SIGTERM +W0701 08:52:54.888000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139988 closing signal SIGTERM +W0701 08:52:54.889000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860944 closing signal SIGTERM +W0701 08:52:54.890000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896467 closing signal SIGTERM +W0701 08:52:54.889000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008519 closing signal SIGTERM +W0701 08:52:54.889000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512474 closing signal SIGTERM +W0701 08:52:54.889000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080468 closing signal SIGTERM +W0701 08:52:54.889000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642511 closing signal SIGTERM +W0701 08:52:54.890000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860945 closing signal SIGTERM +W0701 08:52:54.890000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139989 closing signal SIGTERM +W0701 08:52:54.891000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896468 closing signal SIGTERM +W0701 08:52:54.890000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512475 closing signal SIGTERM +W0701 08:52:54.891000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080469 closing signal SIGTERM +W0701 08:52:54.891000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642512 closing signal SIGTERM +W0701 08:52:54.891000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139990 closing signal SIGTERM +W0701 08:52:54.893000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896469 closing signal SIGTERM +W0701 08:52:54.891000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008520 closing signal SIGTERM +W0701 08:52:54.892000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860946 closing signal SIGTERM +W0701 08:52:54.892000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080470 closing signal SIGTERM +W0701 08:52:54.892000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512476 closing signal SIGTERM +W0701 08:52:54.892000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139991 closing signal SIGTERM +W0701 08:52:54.893000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008521 closing signal SIGTERM +W0701 08:52:54.893000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642513 closing signal SIGTERM +W0701 08:52:54.894000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896470 closing signal SIGTERM +W0701 08:52:54.893000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860947 closing signal SIGTERM +W0701 08:52:54.894000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512477 closing signal SIGTERM +W0701 08:52:54.894000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080471 closing signal SIGTERM +W0701 08:52:54.894000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139992 closing signal SIGTERM +W0701 08:52:54.894000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 642514 closing signal SIGTERM +W0701 08:52:54.895000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008522 closing signal SIGTERM +W0701 08:52:54.897000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1896471 closing signal SIGTERM +W0701 08:52:54.896000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1080472 closing signal SIGTERM +W0701 08:52:54.896000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1860948 closing signal SIGTERM +W0701 08:52:54.896000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3008523 closing signal SIGTERM +W0701 08:52:54.896000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1512478 closing signal SIGTERM +W0701 08:52:54.897000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 139993 closing signal SIGTERM