Spaces:
Sleeping
Sleeping
File size: 4,386 Bytes
56da2e5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
import os
import re
import shutil
import time
from pathlib import Path
from datetime import date
from cleantext import clean
from doctr.io import DocumentFile
from doctr.models import ocr_predictor
from spellchecker import SpellChecker
import nltk
nltk.download('punkt')
class Preprocessor:
"""Clase para preprocesar texto, realizar limpieza y correcciones."""
def __init__(self):
self.spell_checker = SpellChecker()
@staticmethod
def clean_text(text: str, lower: bool = False, lang: str = "en") -> str:
"""
Limpia texto de ruido y caracteres no deseados.
"""
return clean(
text,
fix_unicode=True,
to_ascii=True,
lower=lower,
no_line_breaks=True,
no_urls=True,
no_emails=True,
no_phone_numbers=True,
no_numbers=False,
no_digits=False,
no_currency_symbols=True,
no_punct=False,
lang=lang,
)
@staticmethod
def correct_spacing(text: str, exceptions=None) -> str:
"""
Corrige espacios alrededor de signos de puntuación y excepciones.
"""
if exceptions is None:
exceptions = ["e.g.", "i.e.", "etc.", "cf.", "vs.", "p."]
text = re.sub(r"\s+", " ", text)
text = re.sub(r'\s([?.!"](?:\s|$))', r"\1", text)
text = re.sub(r"\s,", r",", text)
for exception in exceptions:
text = text.replace(" ".join(exception.split()), exception)
return text.strip()
@staticmethod
def split_into_sentences(text: str) -> list:
"""
Divide texto en oraciones usando NLTK.
"""
from nltk.tokenize import sent_tokenize
return sent_tokenize(text)
def correct_spelling(self, text: str) -> str:
"""
Corrige la ortografía del texto dado.
"""
words = text.split()
corrected_words = [self.spell_checker.correction(word) for word in words]
return " ".join(corrected_words)
def preprocess_text(self, text: str) -> str:
"""
Limpia, corrige ortografía y ajusta espacios en texto.
"""
cleaned = self.clean_text(text)
corrected = self.correct_spelling(cleaned)
return self.correct_spacing(corrected)
class PDFProcessor:
"""Clase para procesar archivos PDF y convertirlos a texto."""
def __init__(self, max_pages=20):
self.ocr_model = ocr_predictor(pretrained=True)
self.max_pages = max_pages
def pdf_to_text(self, file_path: str) -> str:
"""
Convierte un archivo PDF a texto usando OCR.
"""
pdf_file = Path(file_path)
doc = DocumentFile.from_pdf(pdf_file)
if len(doc.pages) > self.max_pages:
doc.pages = doc.pages[:self.max_pages]
result = self.ocr_model(doc)
raw_text = "\n".join(
[block.text for page in result.pages for block in page.blocks]
)
return Preprocessor().preprocess_text(raw_text)
class FileHandler:
"""Clase para manejar archivos temporales y limpieza."""
@staticmethod
def save_temp_file(file_obj, temp_dir: Path = None) -> str:
"""
Guarda un archivo temporalmente y retorna su ruta.
"""
if temp_dir is None:
temp_dir = Path("temp")
temp_dir.mkdir(exist_ok=True)
file_path = Path(file_obj.name)
temp_path = temp_dir / file_path.name
with open(temp_path, "wb") as f:
f.write(file_obj.read())
return str(temp_path.resolve())
@staticmethod
def clear_temp_files(directory="temp", name_contains="RESULT_"):
"""
Limpia archivos temporales en el directorio especificado.
"""
temp_dir = Path(directory)
if not temp_dir.exists():
return
for file in temp_dir.iterdir():
if file.is_file() and name_contains in file.name:
file.unlink()
@staticmethod
def move_to_completed(from_dir: Path, filename: str, completed_dir="completed"):
"""
Mueve un archivo procesado a la carpeta 'completed'.
"""
completed_path = from_dir / completed_dir
completed_path.mkdir(exist_ok=True)
shutil.move(from_dir / filename, completed_path / filename)
|