Update dspy_qa.py

dspy_qa.py

@@ -4,11 +4,12 @@ import dspy
 from dsp.utils import deduplicate
 from dspy.retrieve.faiss_rm import FaissRM
 from langchain_community.document_loaders import PyPDFLoader
-from langchain_community.document_loaders import CSVLoader
-
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
-# os.environ["AZURE_OPENAI_API_KEY"] = ""
+api_base=os.getenv("AZURE_OPENAI_ENDPOINT")
+api_version=os.getenv("OPENAI_API_VERSION")
+
+
 
 class GenerateSearchQuery(dspy.Signature):
     """Write a simple search query that will help answer a complex question."""
@@ -32,9 +33,9 @@ class DocQA(dspy.Module):
     def __init__(self, file_path,passages_per_hop=3, max_hops=2):
         super().__init__()
         self.cache = "cache.json"
-        self.llm = dspy.AzureOpenAI(api_base="https://azureadople.openai.azure.com/",
-                                   api_version="2023-09-15-preview",
-                                   model="GPT-3")
+        self.llm = dspy.AzureOpenAI(api_base=api_base,
+                                   api_version=api_version,
+                                   model="GPT-4o")
 
         self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]
         self.retrieve = dspy.Retrieve(k=passages_per_hop)
@@ -44,14 +45,14 @@ class DocQA(dspy.Module):
         self.knowledge_base = self.create_knowledge_base(file_path)
 
     def load_documents(self, file_path):
-        print("file_path", file_path)
-        loader = CSVLoader(file_path)
+        # print("file_path", file_path)
+        loader = PyPDFLoader(file_path)
         documents = loader.load()
         return documents
 
     def split_documents(self, documents):
         text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=6000,
+            chunk_size=10000,
             chunk_overlap=0,
             length_function=len,
             is_separator_regex=False,
@@ -59,11 +60,11 @@ class DocQA(dspy.Module):
 
         docs = text_splitter.split_documents(documents)
         document_chunks = [page_content.page_content for page_content in docs]
-        print("input context Ready")
+        # print("input context Ready")
         return document_chunks
 
     def create_knowledge_base(self, file_path):
-        print("file_path", file_path)
+        # print("file_path", file_path)
         document = self.load_documents(file_path)
         split_documents = self.split_documents(document)
         knowledge_base = FaissRM(split_documents)
@@ -72,8 +73,8 @@ class DocQA(dspy.Module):
     def run(self,question):
         dspy.settings.configure(lm=self.llm, rm=self.knowledge_base)
 
-
         passages = self.retrieve(question).passages
+        print("passages", passages)
         context = deduplicate(passages)
 
         pred = self.generate_answer(context=context, question=question)