yoooo?

app.py

@@ -1,12 +1,23 @@
+#pip install transformers flash_attn
 import gradio as gr
-from huggingface_hub import InferenceClient
+import spaces
+import torch
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("jpacifico/Chocolatine-14B-Instruct-DPO-v1.2")
+model = AutoModelForCausalLM.from_pretrained(
+    "jpacifico/Chocolatine-14B-Instruct-DPO-v1.2",
+    device_map="cuda",  
+    torch_dtype="auto",  
+    trust_remote_code=True,
+)
 
 
+@spaces.GPU
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -26,14 +37,21 @@ def respond(
     messages.append({"role": "user", "content": message})
 
     response = ""
+    
+    prompt = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(model.device)
+    # Generate text
+    # messages = model.generate(prompt, max_new_tokens=512)
+
+    messages = model.generate(
+        prompt,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.9,
+        num_return_sequences=1,
+        max_length=200
+    )
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
+    for message in messages:
         token = message.choices[0].delta.content
 
         response += token

requirements.txt

@@ -1 +1,3 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+transformers[torch]
+flash_attn