diff --git "a/slurm/1038241.0.err" "b/slurm/1038241.0.err" new file mode 100644--- /dev/null +++ "b/slurm/1038241.0.err" @@ -0,0 +1,1156 @@ +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.309 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.310 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.320 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.433 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.434 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.444 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.499 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.500 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.509 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.510 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.625 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.626 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.635 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.661 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.662 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:47.672 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.363 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.364 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.374 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.932 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.933 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.933 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.933 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.933 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.933 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.933 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:48.942 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.108 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.109 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:44:49.119 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( + Fetching 21 files: 0%| | 0/21 [00:00 +[rank0]: train() +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/train/train.py", line 906, in train +[rank0]: trainer.train(resume_from_checkpoint=resume_from_checkpoint) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer.py", line 2122, in train +[rank0]: return inner_training_loop( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/train/llava_trainer.py", line 665, in _inner_training_loop +[rank0]: return super()._inner_training_loop(batch_size, *args, **kwargs) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer.py", line 2381, in _inner_training_loop +[rank0]: self.control = self.callback_handler.on_train_begin(args, self.state, self.control) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer_callback.py", line 468, in on_train_begin +[rank0]: return self.call_event("on_train_begin", args, state, control) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer_callback.py", line 518, in call_event +[rank0]: result = getattr(callback, event)( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 911, in on_train_begin +[rank0]: self.setup(args, state, model, **kwargs) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 838, in setup +[rank0]: self._wandb.init( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1623, in init +[rank0]: wandb._sentry.reraise(e) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/analytics/sentry.py", line 156, in reraise +[rank0]: raise exc.with_traceback(sys.exc_info()[2]) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1551, in init +[rank0]: wi.maybe_login(init_settings) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 191, in maybe_login +[rank0]: wandb_login._login( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_login.py", line 315, in _login +[rank0]: key, key_status = wlogin.prompt_api_key(referrer=referrer) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_login.py", line 243, in prompt_api_key +[rank0]: raise UsageError("api_key not configured (no-tty). call " + directive) +[rank0]: wandb.errors.errors.UsageError: api_key not configured (no-tty). call wandb.login(key=[your_api_key]) +W0701 08:48:31.378000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073409 closing signal SIGTERM +W0701 08:48:31.380000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073410 closing signal SIGTERM +W0701 08:48:31.381000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073411 closing signal SIGTERM +W0701 08:48:31.382000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073412 closing signal SIGTERM +W0701 08:48:31.383000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073413 closing signal SIGTERM +W0701 08:48:31.387000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073414 closing signal SIGTERM +W0701 08:48:31.387000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2073415 closing signal SIGTERM +E0701 08:48:38.726000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 0 (pid: 2073408) of binary: /lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/bin/python3 +Traceback (most recent call last): + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/bin/torchrun", line 10, in + sys.exit(main()) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper + return f(*args, **kwargs) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main + run(args) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run + elastic_launch( + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-07-01_08:48:31 + host : pool0-01504.cm.cluster + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 2073408) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +W0701 08:48:39.238000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.238000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498437 closing signal SIGTERM +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498438 closing signal SIGTERM +W0701 08:48:39.238000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066066 closing signal SIGTERM +W0701 08:48:39.238000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845860 closing signal SIGTERM +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994607 closing signal SIGTERM +W0701 08:48:39.240000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881736 closing signal SIGTERM +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126184 closing signal SIGTERM +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 08:48:39.239000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627617 closing signal SIGTERM +W0701 08:48:39.241000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881737 closing signal SIGTERM +W0701 08:48:39.240000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845861 closing signal SIGTERM +W0701 08:48:39.240000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066067 closing signal SIGTERM +W0701 08:48:39.240000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994608 closing signal SIGTERM +W0701 08:48:39.240000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126185 closing signal SIGTERM +W0701 08:48:39.240000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627618 closing signal SIGTERM +W0701 08:48:39.242000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881738 closing signal SIGTERM +W0701 08:48:39.241000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845862 closing signal SIGTERM +W0701 08:48:39.241000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498439 closing signal SIGTERM +W0701 08:48:39.241000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994609 closing signal SIGTERM +W0701 08:48:39.241000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066068 closing signal SIGTERM +W0701 08:48:39.242000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126186 closing signal SIGTERM +W0701 08:48:39.242000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627619 closing signal SIGTERM +W0701 08:48:39.242000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066069 closing signal SIGTERM +W0701 08:48:39.243000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498440 closing signal SIGTERM +W0701 08:48:39.244000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881739 closing signal SIGTERM +W0701 08:48:39.243000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994610 closing signal SIGTERM +W0701 08:48:39.243000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126187 closing signal SIGTERM +W0701 08:48:39.243000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845863 closing signal SIGTERM +W0701 08:48:39.243000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627620 closing signal SIGTERM +W0701 08:48:39.244000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498441 closing signal SIGTERM +W0701 08:48:39.244000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066070 closing signal SIGTERM +W0701 08:48:39.244000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126188 closing signal SIGTERM +W0701 08:48:39.246000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881740 closing signal SIGTERM +W0701 08:48:39.245000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845864 closing signal SIGTERM +W0701 08:48:39.245000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994611 closing signal SIGTERM +W0701 08:48:39.245000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066071 closing signal SIGTERM +W0701 08:48:39.246000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498442 closing signal SIGTERM +W0701 08:48:39.245000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627621 closing signal SIGTERM +W0701 08:48:39.246000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126189 closing signal SIGTERM +W0701 08:48:39.247000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881741 closing signal SIGTERM +W0701 08:48:39.246000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845865 closing signal SIGTERM +W0701 08:48:39.246000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994612 closing signal SIGTERM +W0701 08:48:39.247000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066072 closing signal SIGTERM +W0701 08:48:39.247000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498443 closing signal SIGTERM +W0701 08:48:39.247000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627622 closing signal SIGTERM +W0701 08:48:39.247000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126190 closing signal SIGTERM +W0701 08:48:39.248000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845866 closing signal SIGTERM +W0701 08:48:39.249000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881742 closing signal SIGTERM +W0701 08:48:39.248000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 126191 closing signal SIGTERM +W0701 08:48:39.248000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1498444 closing signal SIGTERM +W0701 08:48:39.248000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994613 closing signal SIGTERM +W0701 08:48:39.249000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627623 closing signal SIGTERM +W0701 08:48:39.249000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1066073 closing signal SIGTERM +W0701 08:48:39.250000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1881743 closing signal SIGTERM +W0701 08:48:39.249000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1845867 closing signal SIGTERM +W0701 08:48:39.249000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2994614 closing signal SIGTERM +W0701 08:48:39.251000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 627624 closing signal SIGTERM