Spaces:
Sleeping
Sleeping
Commit
·
391b2f3
1
Parent(s):
e0aff18
add: pypdf2 loader text loader
Browse files
medrag_multi_modal/__init__.py
CHANGED
@@ -2,12 +2,14 @@ from .document_loader import (
|
|
2 |
ImageLoader,
|
3 |
MarkerTextLoader,
|
4 |
PyMuPDF4LLMTextLoader,
|
|
|
5 |
TextImageLoader,
|
6 |
)
|
7 |
from .retrieval import MultiModalRetriever
|
8 |
|
9 |
__all__ = [
|
10 |
"PyMuPDF4LLMTextLoader",
|
|
|
11 |
"MarkerTextLoader",
|
12 |
"ImageLoader",
|
13 |
"TextImageLoader",
|
|
|
2 |
ImageLoader,
|
3 |
MarkerTextLoader,
|
4 |
PyMuPDF4LLMTextLoader,
|
5 |
+
PyPDF2TextLoader,
|
6 |
TextImageLoader,
|
7 |
)
|
8 |
from .retrieval import MultiModalRetriever
|
9 |
|
10 |
__all__ = [
|
11 |
"PyMuPDF4LLMTextLoader",
|
12 |
+
"PyPDF2TextLoader",
|
13 |
"MarkerTextLoader",
|
14 |
"ImageLoader",
|
15 |
"TextImageLoader",
|
medrag_multi_modal/document_loader/__init__.py
CHANGED
@@ -1,12 +1,10 @@
|
|
1 |
from .load_image import ImageLoader
|
2 |
from .load_text_image import TextImageLoader
|
3 |
-
from .text_loader import
|
4 |
-
MarkerTextLoader,
|
5 |
-
PyMuPDF4LLMTextLoader,
|
6 |
-
)
|
7 |
|
8 |
__all__ = [
|
9 |
"PyMuPDF4LLMTextLoader",
|
|
|
10 |
"MarkerTextLoader",
|
11 |
"ImageLoader",
|
12 |
"TextImageLoader",
|
|
|
1 |
from .load_image import ImageLoader
|
2 |
from .load_text_image import TextImageLoader
|
3 |
+
from .text_loader import MarkerTextLoader, PyMuPDF4LLMTextLoader, PyPDF2TextLoader
|
|
|
|
|
|
|
4 |
|
5 |
__all__ = [
|
6 |
"PyMuPDF4LLMTextLoader",
|
7 |
+
"PyPDF2TextLoader",
|
8 |
"MarkerTextLoader",
|
9 |
"ImageLoader",
|
10 |
"TextImageLoader",
|
medrag_multi_modal/document_loader/text_loader/__init__.py
CHANGED
@@ -1,7 +1,9 @@
|
|
1 |
from .marker_text_loader import MarkerTextLoader
|
2 |
from .pymupdf4llm_text_loader import PyMuPDF4LLMTextLoader
|
|
|
3 |
|
4 |
__all__ = [
|
5 |
"PyMuPDF4LLMTextLoader",
|
|
|
6 |
"MarkerTextLoader",
|
7 |
]
|
|
|
1 |
from .marker_text_loader import MarkerTextLoader
|
2 |
from .pymupdf4llm_text_loader import PyMuPDF4LLMTextLoader
|
3 |
+
from .pypdf2_text_loader import PyPDF2TextLoader
|
4 |
|
5 |
__all__ = [
|
6 |
"PyMuPDF4LLMTextLoader",
|
7 |
+
"PyPDF2TextLoader",
|
8 |
"MarkerTextLoader",
|
9 |
]
|
medrag_multi_modal/document_loader/text_loader/pypdf2_text_loader.py
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from typing import Dict
|
2 |
+
|
3 |
+
import PyPDF2
|
4 |
+
|
5 |
+
from .base_text_loader import BaseTextLoader
|
6 |
+
|
7 |
+
|
8 |
+
class PyPDF2TextLoader(BaseTextLoader):
|
9 |
+
async def _process_page(self, page_idx: int) -> Dict[str, str]:
|
10 |
+
with open(self.document_file_path, "rb") as file:
|
11 |
+
pdf_reader = PyPDF2.PdfReader(file)
|
12 |
+
page = pdf_reader.pages[page_idx]
|
13 |
+
text = page.extract_text()
|
14 |
+
|
15 |
+
return {
|
16 |
+
"text": text,
|
17 |
+
"page_idx": page_idx,
|
18 |
+
"document_name": self.document_name,
|
19 |
+
"file_path": self.document_file_path,
|
20 |
+
"file_url": self.url,
|
21 |
+
}
|