lets see

app.py

@@ -6,7 +6,7 @@ from langchain.embeddings.huggingface import HuggingFaceEmbeddings
 import torch
 import sentence_transformers
 from langchain.vectorstores import Pinecone
-from langchain import HuggingFaceHub
+from langchain.llms.huggingface_text_gen_inference import HuggingFaceTextGenInference
 from langchain.prompts import PromptTemplate
 from langchain.chains import RetrievalQA
 
@@ -55,15 +55,19 @@ vectorstore = Pinecone(
 )
 
 
-hub = HuggingFaceHub(repo_id = "HuggingFaceH4/zephyr-7b-beta",huggingfacehub_api_token={os.environ.get('API_KEY')})
-print(hub)
-prompt = PromptTemplate(
-    input_variables=["question"],
-    template="Question: {question}\nAnswer:",
+API_URL = "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta"
+llm = HuggingFaceTextGenInference(
+    inference_server_url=API_URL,
+    max_new_tokens=512,
+    top_k=10,
+    top_p=0.95,
+    typical_p=0.95,
+    temperature=0.01,
+    repetition_penalty=1.03,
 )
 
 rag_pipeline = RetrievalQA.from_chain_type(
-    llm=hub, chain_type='stuff',
+    llm=llm, chain_type='stuff',
     retriever=vectorstore.as_retriever()
 )