Update app.py

app.py

@@ -1,33 +1,33 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
+from ctransformers import AutoModelForCausalLM
 import gradio as gr
 
 sys_message = """
 This model can generate untruths, lies or inappropriate things. Only for testing and validation. 
 """
 
-# Configuration settings for model generation (example)
-generation_config = {
-    "max_new_tokens": 2048,
-    "temperature": 0.50,
-    "top_p": 0.95,
-    "top_k": 30,
-    "repetition_penalty": 1.1,
-    "stop_token": '### Instruction:'
-}
-
-llm = AutoModel.from_pretrained("tevykuch/sftsl0th")
-# llm = AutoModelForCausalLM.from_pretrained("tevykuch/sl0th", hf=True)
-tokenizer = AutoTokenizer.from_pretrained(llm)
-
-def stream(prompt):
-    # Tokenize the prompt
-    inputs = tokenizer.encode(prompt, return_tensors="pt")
-    # Generate a response
-    output_ids = llm.generate(inputs, **generation_config)
-    # Decode the generated ids to a string
-    response = tokenizer.decode(output_ids[0], skip_special_tokens=True)
-    return response
 
+llm = AutoModelForCausalLM.from_pretrained("tevykuch/sftsl0th",
+model_type='mistral',
+max_new_tokens = 2048,
+threads = 2,
+temperature = 0.50,
+top_p = 0.95,
+top_k = 30,
+repetition_penalty = 1.1,
+stop=['### Instruction:']
+)
+
+def stream(prompt, UL):
+    system_prompt = 'You are a helpful chatbot. You only answer in Khmer. User is based in Cambodia. Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.'
+    E_INST = " "
+    system, user, assistant = "###Instruction", "###Input", "###Response"
+    prompt = f"{system}\n{system_prompt}{E_INST}\n{user}\n{prompt.strip()}{E_INST}\n{assistant}\n"
+
+    output = ""
+    for response in llm(prompt, stream=True):
+        output += response
+        yield output
+    return output
 
 
 chat_interface = gr.ChatInterface(