mratanusarkar commited on
Commit
391b2f3
·
1 Parent(s): e0aff18

add: pypdf2 loader text loader

Browse files
medrag_multi_modal/__init__.py CHANGED
@@ -2,12 +2,14 @@ from .document_loader import (
2
  ImageLoader,
3
  MarkerTextLoader,
4
  PyMuPDF4LLMTextLoader,
 
5
  TextImageLoader,
6
  )
7
  from .retrieval import MultiModalRetriever
8
 
9
  __all__ = [
10
  "PyMuPDF4LLMTextLoader",
 
11
  "MarkerTextLoader",
12
  "ImageLoader",
13
  "TextImageLoader",
 
2
  ImageLoader,
3
  MarkerTextLoader,
4
  PyMuPDF4LLMTextLoader,
5
+ PyPDF2TextLoader,
6
  TextImageLoader,
7
  )
8
  from .retrieval import MultiModalRetriever
9
 
10
  __all__ = [
11
  "PyMuPDF4LLMTextLoader",
12
+ "PyPDF2TextLoader",
13
  "MarkerTextLoader",
14
  "ImageLoader",
15
  "TextImageLoader",
medrag_multi_modal/document_loader/__init__.py CHANGED
@@ -1,12 +1,10 @@
1
  from .load_image import ImageLoader
2
  from .load_text_image import TextImageLoader
3
- from .text_loader import (
4
- MarkerTextLoader,
5
- PyMuPDF4LLMTextLoader,
6
- )
7
 
8
  __all__ = [
9
  "PyMuPDF4LLMTextLoader",
 
10
  "MarkerTextLoader",
11
  "ImageLoader",
12
  "TextImageLoader",
 
1
  from .load_image import ImageLoader
2
  from .load_text_image import TextImageLoader
3
+ from .text_loader import MarkerTextLoader, PyMuPDF4LLMTextLoader, PyPDF2TextLoader
 
 
 
4
 
5
  __all__ = [
6
  "PyMuPDF4LLMTextLoader",
7
+ "PyPDF2TextLoader",
8
  "MarkerTextLoader",
9
  "ImageLoader",
10
  "TextImageLoader",
medrag_multi_modal/document_loader/text_loader/__init__.py CHANGED
@@ -1,7 +1,9 @@
1
  from .marker_text_loader import MarkerTextLoader
2
  from .pymupdf4llm_text_loader import PyMuPDF4LLMTextLoader
 
3
 
4
  __all__ = [
5
  "PyMuPDF4LLMTextLoader",
 
6
  "MarkerTextLoader",
7
  ]
 
1
  from .marker_text_loader import MarkerTextLoader
2
  from .pymupdf4llm_text_loader import PyMuPDF4LLMTextLoader
3
+ from .pypdf2_text_loader import PyPDF2TextLoader
4
 
5
  __all__ = [
6
  "PyMuPDF4LLMTextLoader",
7
+ "PyPDF2TextLoader",
8
  "MarkerTextLoader",
9
  ]
medrag_multi_modal/document_loader/text_loader/pypdf2_text_loader.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict
2
+
3
+ import PyPDF2
4
+
5
+ from .base_text_loader import BaseTextLoader
6
+
7
+
8
+ class PyPDF2TextLoader(BaseTextLoader):
9
+ async def _process_page(self, page_idx: int) -> Dict[str, str]:
10
+ with open(self.document_file_path, "rb") as file:
11
+ pdf_reader = PyPDF2.PdfReader(file)
12
+ page = pdf_reader.pages[page_idx]
13
+ text = page.extract_text()
14
+
15
+ return {
16
+ "text": text,
17
+ "page_idx": page_idx,
18
+ "document_name": self.document_name,
19
+ "file_path": self.document_file_path,
20
+ "file_url": self.url,
21
+ }