diff --git "a/slurm/1038255.0.err" "b/slurm/1038255.0.err" new file mode 100644--- /dev/null +++ "b/slurm/1038255.0.err" @@ -0,0 +1,1156 @@ +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.524 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.525 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.534 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.543 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.543 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.543 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.544 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.554 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.605 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.605 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.605 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.605 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.605 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.606 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.607 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.616 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.736 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.736 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.736 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.737 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.739 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.739 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.739 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.739 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.747 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.747 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.747 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.747 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.747 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.747 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.756 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.756 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.971 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.972 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.981 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:36.982 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.281 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.282 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.291 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.627 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.627 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.627 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.627 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.627 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.627 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.628 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.638 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.790 | INFO | llava.data.builder:register_datasets:39 - Registering datasets from environment: 'nvidia,draco-oci-iad'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.791 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/nvidia.yaml'. +2025-07-01 08:58:37.800 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.800 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.801 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.801 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.801 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.801 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.801 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +2025-07-01 08:58:37.801 | INFO | llava.data.builder:register_datasets:44 - Registering datasets from: '/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/data/registry/datasets/draco-oci-iad.yaml'. +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +Did not find AutoResume SDK! +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/training_args.py:1559: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( + Fetching 21 files: 0%| | 0/21 [00:00 +[rank0]: train() +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/train/train.py", line 906, in train +[rank0]: trainer.train(resume_from_checkpoint=resume_from_checkpoint) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer.py", line 2122, in train +[rank0]: return inner_training_loop( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/llava/train/llava_trainer.py", line 665, in _inner_training_loop +[rank0]: return super()._inner_training_loop(batch_size, *args, **kwargs) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer.py", line 2381, in _inner_training_loop +[rank0]: self.control = self.callback_handler.on_train_begin(args, self.state, self.control) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer_callback.py", line 468, in on_train_begin +[rank0]: return self.call_event("on_train_begin", args, state, control) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/trainer_callback.py", line 518, in call_event +[rank0]: result = getattr(callback, event)( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 911, in on_train_begin +[rank0]: self.setup(args, state, model, **kwargs) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 838, in setup +[rank0]: self._wandb.init( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1623, in init +[rank0]: wandb._sentry.reraise(e) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/analytics/sentry.py", line 156, in reraise +[rank0]: raise exc.with_traceback(sys.exc_info()[2]) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1551, in init +[rank0]: wi.maybe_login(init_settings) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 191, in maybe_login +[rank0]: wandb_login._login( +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_login.py", line 315, in _login +[rank0]: key, key_status = wlogin.prompt_api_key(referrer=referrer) +[rank0]: File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/wandb/sdk/wandb_login.py", line 243, in prompt_api_key +[rank0]: raise UsageError("api_key not configured (no-tty). call " + directive) +[rank0]: wandb.errors.errors.UsageError: api_key not configured (no-tty). call wandb.login(key=[your_api_key]) +W0701 09:02:19.167000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938960 closing signal SIGTERM +W0701 09:02:19.168000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938961 closing signal SIGTERM +W0701 09:02:19.169000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938962 closing signal SIGTERM +W0701 09:02:19.171000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938963 closing signal SIGTERM +W0701 09:02:19.172000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938964 closing signal SIGTERM +W0701 09:02:19.173000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938965 closing signal SIGTERM +W0701 09:02:19.175000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3938966 closing signal SIGTERM +E0701 09:02:25.161000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 0 (pid: 3938959) of binary: /lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/bin/python3 +Traceback (most recent call last): + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/bin/torchrun", line 10, in + sys.exit(main()) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper + return f(*args, **kwargs) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main + run(args) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run + elastic_launch( + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/lustre/fs1/portfolios/nvr/projects/nvr_elm_llm/users/ligengz/workspace/vila-q/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-07-01_09:02:19 + host : pool0-01868.cm.cluster + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 3938959) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +W0701 09:02:25.681000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834153 closing signal SIGTERM +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.681000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974225 closing signal SIGTERM +W0701 09:02:25.681000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647819 closing signal SIGTERM +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.681000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890545 closing signal SIGTERM +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117654 closing signal SIGTERM +W0701 09:02:25.681000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001931 closing signal SIGTERM +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001932 closing signal SIGTERM +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/agent/server/api.py:741] Received Signals.SIGTERM death signal, shutting down workers +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545985 closing signal SIGTERM +W0701 09:02:25.684000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974226 closing signal SIGTERM +W0701 09:02:25.682000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647820 closing signal SIGTERM +W0701 09:02:25.683000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834154 closing signal SIGTERM +W0701 09:02:25.684000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117655 closing signal SIGTERM +W0701 09:02:25.683000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890546 closing signal SIGTERM +W0701 09:02:25.683000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545986 closing signal SIGTERM +W0701 09:02:25.684000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647821 closing signal SIGTERM +W0701 09:02:25.684000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834155 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974227 closing signal SIGTERM +W0701 09:02:25.684000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001933 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117656 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890547 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545987 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834156 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001934 closing signal SIGTERM +W0701 09:02:25.685000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647822 closing signal SIGTERM +W0701 09:02:25.687000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974228 closing signal SIGTERM +W0701 09:02:25.686000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545988 closing signal SIGTERM +W0701 09:02:25.687000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117657 closing signal SIGTERM +W0701 09:02:25.687000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890548 closing signal SIGTERM +W0701 09:02:25.687000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834157 closing signal SIGTERM +W0701 09:02:25.686000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001935 closing signal SIGTERM +W0701 09:02:25.688000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974229 closing signal SIGTERM +W0701 09:02:25.688000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117658 closing signal SIGTERM +W0701 09:02:25.688000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647823 closing signal SIGTERM +W0701 09:02:25.688000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890549 closing signal SIGTERM +W0701 09:02:25.688000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545989 closing signal SIGTERM +W0701 09:02:25.688000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001936 closing signal SIGTERM +W0701 09:02:25.689000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834158 closing signal SIGTERM +W0701 09:02:25.689000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545990 closing signal SIGTERM +W0701 09:02:25.690000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117659 closing signal SIGTERM +W0701 09:02:25.690000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974230 closing signal SIGTERM +W0701 09:02:25.689000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001937 closing signal SIGTERM +W0701 09:02:25.690000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834159 closing signal SIGTERM +W0701 09:02:25.689000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647824 closing signal SIGTERM +W0701 09:02:25.690000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890550 closing signal SIGTERM +W0701 09:02:25.691000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974234 closing signal SIGTERM +W0701 09:02:25.691000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545991 closing signal SIGTERM +W0701 09:02:25.691000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117660 closing signal SIGTERM +W0701 09:02:25.691000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3001938 closing signal SIGTERM +W0701 09:02:25.691000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647825 closing signal SIGTERM +W0701 09:02:25.691000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890551 closing signal SIGTERM +W0701 09:02:25.692000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1834160 closing signal SIGTERM +W0701 09:02:25.693000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 890552 closing signal SIGTERM +W0701 09:02:25.693000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 3117661 closing signal SIGTERM +W0701 09:02:25.693000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2545992 closing signal SIGTERM +W0701 09:02:25.694000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 1974235 closing signal SIGTERM +W0701 09:02:25.693000 23456244200576 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 647826 closing signal SIGTERM