init

app.py

@@ -94,4 +94,4 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
     clear_button.click(
         clear_chat, inputs=[chatbot], outputs=[txt, chatbot])
 
-demo.launch(share=True)
+demo.launch()

config/infer.yaml

@@ -4,7 +4,7 @@ vicuna:
   model_path: '/home/user/app/vicuna-7b'
   # model_path: '/mnt/petrelfs/wangyiqin/vid_cap/ChatVID_huggingface/vicuna-7b'
   device: 'cuda'
-  num_gpus: 1
+  num_gpus: 'auto'
   max_gpu_memory: '24Gib'
   load_8bit: True
   conv_template: 

model/fastchat/serve/inference.py

@@ -80,6 +80,9 @@ def load_model(
         kwargs = {}
     elif device == "cuda":
         kwargs = {"torch_dtype": torch.float16}
+        if load_8bit:
+            kwargs = {"load_in_8bit": True}
+
         if num_gpus == "auto":
             kwargs["device_map"] = "auto"
         else:
@@ -134,8 +137,8 @@ def load_model(
         )
         raise_warning_for_old_weights(model_path, model)
 
-    if load_8bit:
-        compress_module(model, device)
+    # if load_8bit:
+    #     compress_module(model, device)
 
     if (device == "cuda" and num_gpus == 1) or device == "mps":
         model.to(device)

pre-requirements.txt

@@ -13,4 +13,5 @@ regex
 tqdm
 openai-whisper
 accelerate
+bitsandbytes
 sentencepiece