try to get model working

transformerschat.py

@@ -5,13 +5,13 @@ import spaces
 
 
 def models():
-    return ["teknium/OpenHermes-2.5-Mistral-7B"]
+    return ["openhermes-2.5-mistral-7b.Q4_K_M.gguf"]
 
 
 def load():
-    torch.set_default_device("cuda")
-    model = AutoModelForCausalLM.from_pretrained(models()[0], torch_dtype="auto", trust_remote_code=True)
-    tokenizer = AutoTokenizer.from_pretrained(models()[0], trust_remote_code=True).to("cuda")
+    # torch.set_default_device("cuda")
+    model = AutoModelForCausalLM.from_pretrained("TheBloke/OpenHermes-2.5-Mistral-7B-GGUF", model_file="openhermes-2.5-mistral-7b.Q4_K_M.gguf", model_type="mistral", gpu_layers=50)
+    # tokenizer = AutoTokenizer.from_pretrained(models()[0], trust_remote_code=True).to("cuda")
     return (model, tokenizer)
 
 
@@ -30,8 +30,9 @@ def ask(_, system_prompt, pre_prompt, question):
         },
     ]
     logger.debug(f"<< openhermes << {question}")
-    inputs = tokenizer(question, return_tensors="pt", return_attention_mask=False)
-    outputs = model.generate(**inputs, max_length=200)
-    answer = tokenizer.batch_decode(outputs)[0]
+    # inputs = tokenizer(question, return_tensors="pt", return_attention_mask=False)
+    # outputs = model.generate(**inputs, max_length=200)
+    # answer = tokenizer.batch_decode(outputs)[0]
+    answer = model(question)
     logger.debug(f">> openhermes >> {answer}")
     return answer