Update app.py

app.py

@@ -1,4 +1,4 @@
-from ctransformers import AutoModelForCausalLM
+from transformers import AutoModel, AutoTokenizers
 import gradio as gr
 
 sys_message = """
@@ -6,28 +6,29 @@ This model can generate untruths, lies or inappropriate things. Only for testing
 """
 
 
-llm = AutoModelForCausalLM.from_pretrained("tevykuch/sftsl0th",
-model_type='mistral',
-max_new_tokens = 2048,
-threads = 2,
-temperature = 0.50,
-top_p = 0.95,
-top_k = 30,
-repetition_penalty = 1.1,
-stop=['### Instruction:']
-)
+# Configuration settings for model generation (example)
+generation_config = {
+    "max_new_tokens": 2048,
+    "temperature": 0.50,
+    "top_p": 0.95,
+    "top_k": 30,
+    "repetition_penalty": 1.1,
+    "stop_token": '### Instruction:'
+}
+
+llm = AutoModel.from_pretrained("tevykuch/sftsl0th")
+# llm = AutoModelForCausalLM.from_pretrained("tevykuch/sl0th", hf=True)
+tokenizer = AutoTokenizer.from_pretrained(llm)
+
+def stream(prompt):
+    # Tokenize the prompt
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    # Generate a response
+    output_ids = llm.generate(inputs, **generation_config)
+    # Decode the generated ids to a string
+    response = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    return response
 
-def stream(prompt, UL):
-    system_prompt = 'You are a helpful chatbot. You only answer in Khmer. User is based in Cambodia. Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.'
-    E_INST = " "
-    system, user, assistant = "###Instruction", "###Input", "###Response"
-    prompt = f"{system}\n{system_prompt}{E_INST}\n{user}\n{prompt.strip()}{E_INST}\n{assistant}\n"
-
-    output = ""
-    for response in llm(prompt, stream=True):
-        output += response
-        yield output
-    return output
 
 
 chat_interface = gr.ChatInterface(