Update chain_setup.py

chain_setup.py

@@ -1,23 +1,49 @@
 from langchain.chains import ConversationalRetrievalChain
-from langchain.llms import LlamaCpp
+from langchain.llms import HuggingFacePipeline
 from langchain.memory import ConversationBufferMemory
+import transformers
+import torch
+import os
 
 def load_llm():
-    model_path = "qwen2.5-7b-instruct-q4_k_m.gguf"  # path to your GGUF file
-    # Adjust parameters like n_ctx as needed
-    llm = LlamaCpp(model_path=model_path, n_ctx=2048)
-    return llm
+    model_id = "redhat6/Qwen2.5-7B-Instruct-Q4_K_M-GGUF"
+    tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
 
+    offload_folder = "offload"
+    os.makedirs(offload_folder, exist_ok=True)
+
+    model = transformers.AutoModelForCausalLM.from_pretrained(
+        model_id,
+        trust_remote_code=True,
+        device_map="auto",
+        offload_folder=offload_folder
+    )
+
+    pipe = transformers.pipeline(
+        "text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        max_new_tokens=512
+    )
+    return pipe
+    
 def build_conversational_chain(vectorstore):
+    """
+    Creates a ConversationalRetrievalChain using the HuggingFacePipeline based LLM 
+    and a ConversationBufferMemory for multi-turn Q&A.
+    """
     llm = load_llm()
+
     memory = ConversationBufferMemory(
         memory_key="chat_history",
         return_messages=True
     )
+
     qa_chain = ConversationalRetrievalChain.from_llm(
         llm=llm,
         retriever=vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 5}),
         memory=memory,
-        verbose=True
+        verbose=True  # optional: enables debug logs
     )
+
     return qa_chain