add: pdfplumber text loader

medrag_multi_modal/__init__.py

@@ -1,6 +1,6 @@
 from .document_loader import (
     ImageLoader,
-    MarkerTextLoader,
+    PDFPlumberTextLoader,
     PyMuPDF4LLMTextLoader,
     PyPDF2TextLoader,
     TextImageLoader,
@@ -10,7 +10,7 @@ from .retrieval import MultiModalRetriever
 __all__ = [
     "PyMuPDF4LLMTextLoader",
     "PyPDF2TextLoader",
-    "MarkerTextLoader",
+    "PDFPlumberTextLoader",
     "ImageLoader",
     "TextImageLoader",
     "MultiModalRetriever",

medrag_multi_modal/document_loader/__init__.py

@@ -1,11 +1,11 @@
 from .load_image import ImageLoader
 from .load_text_image import TextImageLoader
-from .text_loader import MarkerTextLoader, PyMuPDF4LLMTextLoader, PyPDF2TextLoader
+from .text_loader import PDFPlumberTextLoader, PyMuPDF4LLMTextLoader, PyPDF2TextLoader
 
 __all__ = [
     "PyMuPDF4LLMTextLoader",
     "PyPDF2TextLoader",
-    "MarkerTextLoader",
+    "PDFPlumberTextLoader",
     "ImageLoader",
     "TextImageLoader",
 ]

medrag_multi_modal/document_loader/text_loader/__init__.py

@@ -1,9 +1,9 @@
-from .marker_text_loader import MarkerTextLoader
+from .pdfplumber_text_loader import PDFPlumberTextLoader
 from .pymupdf4llm_text_loader import PyMuPDF4LLMTextLoader
 from .pypdf2_text_loader import PyPDF2TextLoader
 
 __all__ = [
     "PyMuPDF4LLMTextLoader",
     "PyPDF2TextLoader",
-    "MarkerTextLoader",
+    "PDFPlumberTextLoader",
 ]

medrag_multi_modal/document_loader/text_loader/pdfplumber_text_loader.py

@@ -0,0 +1,20 @@
+from typing import Dict
+
+import pdfplumber
+
+from .base_text_loader import BaseTextLoader
+
+
+class PDFPlumberTextLoader(BaseTextLoader):
+    async def _process_page(self, page_idx: int) -> Dict[str, str]:
+        with pdfplumber.open(self.document_file_path) as pdf:
+            page = pdf.pages[page_idx]
+            text = page.extract_text()
+
+        return {
+            "text": text,
+            "page_idx": page_idx,
+            "document_name": self.document_name,
+            "file_path": self.document_file_path,
+            "file_url": self.url,
+        }