update

app.py

@@ -1,11 +1,29 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
+from unsloth import FastLanguageModel
+max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
+dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
+load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = "minhnguyen5293/lora_model", # YOUR MODEL YOU USED FOR TRAINING
+    max_seq_length = max_seq_length,
+    dtype = dtype,
+    load_in_4bit = load_in_4bit,
+)
+FastLanguageModel.for_inference(model) # Enable native 2x faster inference
 
+max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
+dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
+load_in_4bit = True
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = "lora_model", # YOUR MODEL YOU USED FOR TRAINING
+    max_seq_length = max_seq_length,
+    dtype = dtype,
+    load_in_4bit = load_in_4bit,
+)
+FastLanguageModel.for_inference(model) # Enable native 2x faster inference
 
 def respond(
     message,
@@ -25,19 +43,27 @@ def respond(
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
+    from transformers import TextIteratorStreamer
 
-    for message in client.chat_completion(
+    inputs = tokenizer.apply_chat_template(
         messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+        tokenize = True,
+        add_generation_prompt = True, # Must add for generation
+        return_tensors = "pt",
+    ).to("cuda")
+
+    text_streamer = TextIteratorStreamer(tokenizer, skip_prompt = True)
+    _ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 128,
+                      use_cache = True, temperature = 1.5, min_p = 0.1)
+
+
+    response = ""
 
-        response += token
-        yield response
+    for message in text_streamer:
+        # not append if message contain <|eot_id|>
+        if "<|eot_id|>" not in message:
+          response += message
+          yield response
 
 
 """
@@ -61,4 +87,4 @@ demo = gr.ChatInterface(
 
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()