Update chain_setup.py

chain_setup.py

@@ -1,50 +1,23 @@
 from langchain.chains import ConversationalRetrievalChain
-from langchain.llms import HuggingFacePipeline
+from langchain.llms import LlamaCpp
 from langchain.memory import ConversationBufferMemory
-import transformers
-import torch
-import os
 
 def load_llm():
-    model_id = "redhat6/Qwen2.5-7B-Instruct-Q4_K_M-GGUF"  # Verify the exact model ID on Hugging Face Hub
-    tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
-    
-    # Create an offload folder if it doesn't exist
-    offload_folder = "offload"
-    os.makedirs(offload_folder, exist_ok=True)
-    
-    model = transformers.AutoModelForCausalLM.from_pretrained(
-        model_id,
-        trust_remote_code=True,
-        device_map="auto",  # Use "cpu" if no GPU is available
-        offload_folder=offload_folder
-    )
-    
-    pipe = transformers.pipeline(
-        "text-generation",
-        model=model,
-        tokenizer=tokenizer,
-        max_new_tokens=512  # Adjust as needed
-    )
-    return HuggingFacePipeline(pipeline=pipe)
-    
+    model_path = "qwen2.5-7b-instruct-q4_k_m.gguf"  # path to your GGUF file
+    # Adjust parameters like n_ctx as needed
+    llm = LlamaCpp(model_path=model_path, n_ctx=2048)
+    return llm
+
 def build_conversational_chain(vectorstore):
-    """
-    Creates a ConversationalRetrievalChain using the HuggingFacePipeline based LLM 
-    and a ConversationBufferMemory for multi-turn Q&A.
-    """
     llm = load_llm()
-
     memory = ConversationBufferMemory(
         memory_key="chat_history",
         return_messages=True
     )
-
     qa_chain = ConversationalRetrievalChain.from_llm(
         llm=llm,
         retriever=vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 5}),
         memory=memory,
-        verbose=True  # optional: enables debug logs
+        verbose=True
     )
-
     return qa_chain