fix S3 bucket image upload
Browse files- mineru_single.py +10 -4
mineru_single.py
CHANGED
@@ -45,7 +45,7 @@ class Processor:
|
|
45 |
|
46 |
endpoint = os.getenv("S3_ENDPOINT", "").rstrip("/")
|
47 |
bucket = os.getenv("S3_BUCKET_NAME", "")
|
48 |
-
self.prefix =
|
49 |
|
50 |
logger.info("Processor initialized successfully")
|
51 |
except Exception as e:
|
@@ -92,7 +92,10 @@ class Processor:
|
|
92 |
logger.info("doc_analyze complete for key='%s'. Started extracting images...", key)
|
93 |
|
94 |
# Classify images and remove irrelevant ones
|
95 |
-
image_writer = ImageWriter(self.s3_writer)
|
|
|
|
|
|
|
96 |
pipe_result = inference.pipe_ocr_mode(image_writer, lang=self.language)
|
97 |
logger.info("OCR pipeline completed for key='%s'.", key)
|
98 |
|
@@ -109,8 +112,9 @@ class ImageWriter(DataWriter):
|
|
109 |
Receives each extracted image. Classifies it, uploads if relevant, or flags
|
110 |
it for removal if irrelevant.
|
111 |
"""
|
112 |
-
def __init__(self, s3_writer: S3Writer):
|
113 |
self.s3_writer = s3_writer
|
|
|
114 |
# self.svm_model = svm_model
|
115 |
self._redundant_images_paths = []
|
116 |
|
@@ -118,7 +122,9 @@ class ImageWriter(DataWriter):
|
|
118 |
"""
|
119 |
Called for each extracted image. If relevant, upload to S3; otherwise mark for removal.
|
120 |
"""
|
121 |
-
|
|
|
|
|
122 |
# label_str = self.svm_model.classify_image(data)
|
123 |
|
124 |
# if label_str == 1:
|
|
|
45 |
|
46 |
endpoint = os.getenv("S3_ENDPOINT", "").rstrip("/")
|
47 |
bucket = os.getenv("S3_BUCKET_NAME", "")
|
48 |
+
self.prefix = "/document-extracts/"
|
49 |
|
50 |
logger.info("Processor initialized successfully")
|
51 |
except Exception as e:
|
|
|
92 |
logger.info("doc_analyze complete for key='%s'. Started extracting images...", key)
|
93 |
|
94 |
# Classify images and remove irrelevant ones
|
95 |
+
# image_writer = ImageWriter(self.s3_writer)
|
96 |
+
image_base_path = f"{self.prefix}{key}/" # Construct base path
|
97 |
+
image_writer = ImageWriter(self.s3_writer, image_base_path) # Pass base path to ImageWriter
|
98 |
+
|
99 |
pipe_result = inference.pipe_ocr_mode(image_writer, lang=self.language)
|
100 |
logger.info("OCR pipeline completed for key='%s'.", key)
|
101 |
|
|
|
112 |
Receives each extracted image. Classifies it, uploads if relevant, or flags
|
113 |
it for removal if irrelevant.
|
114 |
"""
|
115 |
+
def __init__(self, s3_writer: S3Writer, base_path: str):
|
116 |
self.s3_writer = s3_writer
|
117 |
+
self.base_path = base_path
|
118 |
# self.svm_model = svm_model
|
119 |
self._redundant_images_paths = []
|
120 |
|
|
|
122 |
"""
|
123 |
Called for each extracted image. If relevant, upload to S3; otherwise mark for removal.
|
124 |
"""
|
125 |
+
# full_path = os.path.join(self.prefix, path)
|
126 |
+
full_path = f"{self.base_path}{path}"
|
127 |
+
self.s3_writer.write(full_path, data)
|
128 |
# label_str = self.svm_model.classify_image(data)
|
129 |
|
130 |
# if label_str == 1:
|