Update app.py

app.py

@@ -5,7 +5,21 @@ from transformers import AutoTokenizer,GenerationConfig
 import torch
 from peft import PeftModel
 import spaces
+import torch
+
+num_gpus = torch.cuda.device_count()
+print(f"Number of available GPUs: {num_gpus}")
 
+# List details for each GPU
+for i in range(num_gpus):
+    print(f"GPU {i}: {torch.cuda.get_device_name(i)}")
+    print(f"  Total Memory: {torch.cuda.get_device_properties(i).total_memory / 1e9:.2f} GB")
+    print(f"  CUDA Capability: {torch.cuda.get_device_properties(i).major}.{torch.cuda.get_device_properties(i).minor}")
+for i in range(num_gpus):
+    print(f"GPU {i}: {torch.cuda.get_device_name(i)}")
+    print(f"  Allocated Memory: {torch.cuda.memory_allocated(i) / 1e9:.2f} GB")
+    print(f"  Cached Memory: {torch.cuda.memory_reserved(i) / 1e9:.2f} GB")
+    print(f"  Free Memory: {torch.cuda.get_device_properties(i).total_memory / 1e9 - torch.cuda.memory_reserved(i) / 1e9:.2f} GB")
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
@@ -74,7 +88,8 @@ def loadModel():
                     torch_dtype=torch.float16,
                     cache_dir=cache_dir,
                     device_map="auto",
-                    load_in_8bit=True
+                    load_in_8bit=True,
+                    load_in_8bit_fp32_cpu_offload=True
                 )
         #model = PeftModel.from_pretrained(
         #            model,