Optimised parsing

pdf2vectorstore.py

@@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
 from pdf2image import convert_from_path
 import pytesseract
 import pickle
+from concurrent.futures import ThreadPoolExecutor
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.document_loaders import UnstructuredFileLoader
@@ -18,14 +19,19 @@ def download_pdf(url, filename):
         for chunk in response.iter_content(chunk_size=8192):
             f.write(chunk)
 
+def extract_image_text(image):
+    return pytesseract.image_to_string(image)
+
 def extract_pdf_text(filename):
     print("Extracting text from pdf...")
     pytesseract.pytesseract.tesseract_cmd = 'tesseract'  
     images = convert_from_path(filename)
     text = ""
-    for image in images:
-        text += pytesseract.image_to_string(image)
+
+    with ThreadPoolExecutor() as executor:
+        text_parts = list(executor.map(extract_image_text, images))
     
+    text = "".join(text_parts)
     return text
 
 def get_arxiv_pdf_url(paper_link):