Update app.py

app.py

@@ -10,7 +10,11 @@ import secrets
 from pathlib import Path
 
 tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat-Int4", trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-VL-Chat-Int4", device_map="auto", trust_remote_code=True).eval()
+config = AutoConfig.from_pretrained("Qwen/Qwen-VL-Chat-Int4")
+#config.quantization_config["use_exllama"] = True
+config.quantization_config["disable_exllama"] = False
+config.quantization_config["exllama_config"] = {"version":2}
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-VL-Chat-Int4", device_map="auto", trust_remote_code=True, config=config).eval()
 
 BOX_TAG_PATTERN = r"<box>([\s\S]*?)</box>"
 PUNCTUATION = "！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."