replace endpoint

rag.py

@@ -4,7 +4,7 @@ import requests
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 import torch
 from openai import OpenAI
-from huggingface_hub import snapshot_download
+from huggingface_hub import snapshot_download, InferenceClient
 
 from langchain_community.vectorstores import FAISS
 from langchain_community.embeddings import HuggingFaceEmbeddings
@@ -105,26 +105,40 @@ class RAG:
 
     def predict_completion(self, instruction, context, model_parameters):
 
-        client = OpenAI(
-                base_url=os.getenv("MODEL"),
-                api_key=os.getenv("HF_TOKEN")
-            )
-
-        query = f"Context:\n{context}\n\nQuestion:\n{instruction}"
+        # client = OpenAI(
+        #         base_url=os.getenv("MODEL"),
+        #         api_key=os.getenv("HF_TOKEN")
+        #     )
+
+        # query = f"Context:\n{context}\n\nQuestion:\n{instruction}"
+
+        # chat_completion = client.chat.completions.create(
+        #     model="tgi",
+        #     messages=[
+        #         {"role": "user", "content": instruction}
+        #     ],
+        #     temperature=model_parameters["temperature"],
+        #     max_tokens=model_parameters["max_new_tokens"],
+        #     stream=False,
+        #     stop=["<|im_end|>"],
+        #     extra_body = {
+        #         "presence_penalty": model_parameters["repetition_penalty"] - 2,
+        #         "do_sample": False
+        #     }
+        # )
+  
+        client = InferenceClient(api_key=os.getenv("HF_TOKEN"),model="meta-llama/Llama-3.3-70B-Instruct")
 
         chat_completion = client.chat.completions.create(
-            model="tgi",
             messages=[
                 {"role": "user", "content": instruction}
             ],
             temperature=model_parameters["temperature"],
             max_tokens=model_parameters["max_new_tokens"],
+            presence_penalty= model_parameters["repetition_penalty"] - 2,
+            top_p= 0.7,
             stream=False,
             stop=["<|im_end|>"],
-            extra_body = {
-                "presence_penalty": model_parameters["repetition_penalty"] - 2,
-                "do_sample": False
-            }
         )
 
         response = chat_completion.choices[0].message.content