Update app.py
Browse files
app.py
CHANGED
@@ -29,12 +29,21 @@ def extract_pdf_text_by_page(file) -> List[str]:
|
|
29 |
return [page.get_text("text") for page in doc]
|
30 |
|
31 |
def extract_pdf_text(file) -> str:
|
32 |
-
"""Extracts text from a PDF file using
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
|
39 |
def check_text_presence(full_text: str, search_terms: List[str]) -> Dict[str, bool]:
|
40 |
"""Checks for the presence of required terms in the text."""
|
@@ -94,32 +103,29 @@ def check_structure(full_text: str) -> Dict[str, bool]:
|
|
94 |
}
|
95 |
|
96 |
def check_language_issues(full_text: str) -> Dict[str, Any]:
|
97 |
-
"""Check for issues
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
|
|
|
|
|
|
113 |
}
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
return {
|
118 |
-
"issues_count": issues_count,
|
119 |
-
"issues_per_1000": issues_per_1000,
|
120 |
-
"failed": issues_per_1000 > 20,
|
121 |
-
"matches": serializable_matches
|
122 |
-
}
|
123 |
|
124 |
def check_language(full_text: str) -> Dict[str, Any]:
|
125 |
"""Check language quality."""
|
@@ -259,97 +265,107 @@ def find_text_instances(words, text):
|
|
259 |
instances.append(inst)
|
260 |
return instances
|
261 |
|
262 |
-
def highlight_issues_in_pdf(file,
|
263 |
-
"""
|
|
|
|
|
|
|
|
|
264 |
try:
|
265 |
-
|
266 |
-
|
267 |
-
|
268 |
-
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
273 |
-
words = page.get_text("words")
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
278 |
-
|
279 |
-
|
280 |
-
|
281 |
-
|
282 |
-
|
283 |
-
|
284 |
-
|
285 |
-
|
286 |
-
|
287 |
-
|
288 |
-
|
289 |
-
|
290 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
291 |
doc.close()
|
292 |
-
return annotated_pdf_bytes
|
293 |
|
|
|
|
|
|
|
|
|
|
|
|
|
294 |
except Exception as e:
|
295 |
-
print(f"
|
296 |
-
traceback.print_exc()
|
297 |
return b""
|
298 |
-
|
299 |
# ------------------------------
|
300 |
# Main Analysis Function
|
301 |
# ------------------------------
|
302 |
|
303 |
def analyze_pdf(file) -> Tuple[Dict[str, Any], bytes]:
|
304 |
-
"""
|
305 |
-
Analyze the uploaded PDF and return analysis results and annotated PDF bytes.
|
306 |
-
"""
|
307 |
try:
|
308 |
-
|
|
|
309 |
full_text = extract_pdf_text(file)
|
310 |
-
|
311 |
-
|
312 |
-
|
313 |
-
|
314 |
-
|
315 |
-
|
316 |
-
|
317 |
-
|
318 |
-
|
319 |
-
|
320 |
-
|
321 |
-
|
322 |
-
|
323 |
-
# Compile results
|
324 |
-
results = {
|
325 |
-
"metadata": metadata,
|
326 |
-
"disclosures": disclosures,
|
327 |
-
"figures_and_tables": figures_and_tables,
|
328 |
-
"figure_order": figure_order,
|
329 |
-
"references": references,
|
330 |
-
"reference_order": reference_order,
|
331 |
-
"reference_style": reference_style,
|
332 |
-
"structure": structure,
|
333 |
-
"language": language
|
334 |
-
}
|
335 |
-
|
336 |
-
# Handle annotations
|
337 |
-
inconsistent_refs = reference_style.get("inconsistent_refs", [])
|
338 |
-
language_matches = language.get("language_issues", {}).get("matches", [])
|
339 |
-
|
340 |
-
if inconsistent_refs or language_matches:
|
341 |
-
annotated_pdf_bytes = highlight_issues_in_pdf(file, inconsistent_refs, language_matches)
|
342 |
-
else:
|
343 |
-
annotated_pdf_bytes = None
|
344 |
-
|
345 |
-
return results, annotated_pdf_bytes
|
346 |
-
|
347 |
except Exception as e:
|
348 |
-
|
349 |
-
"error": str(e),
|
350 |
-
"traceback": traceback.format_exc()
|
351 |
-
}
|
352 |
-
return error_message, None
|
353 |
|
354 |
# ------------------------------
|
355 |
# Gradio Interface
|
|
|
29 |
return [page.get_text("text") for page in doc]
|
30 |
|
31 |
def extract_pdf_text(file) -> str:
|
32 |
+
"""Extracts full text from a PDF file using PyMuPDF."""
|
33 |
+
try:
|
34 |
+
# Open the PDF file
|
35 |
+
doc = fitz.open(stream=file.read(), filetype="pdf") if not isinstance(file, str) else fitz.open(file)
|
36 |
+
full_text = ""
|
37 |
+
for page_num, page in enumerate(doc, start=1):
|
38 |
+
text = page.get_text("text")
|
39 |
+
full_text += text + "\n"
|
40 |
+
print(f"Extracted text from page {page_num}: {len(text)} characters.")
|
41 |
+
doc.close()
|
42 |
+
print(f"Total extracted text length: {len(full_text)} characters.")
|
43 |
+
return full_text
|
44 |
+
except Exception as e:
|
45 |
+
print(f"Error extracting text from PDF: {e}")
|
46 |
+
return ""
|
47 |
|
48 |
def check_text_presence(full_text: str, search_terms: List[str]) -> Dict[str, bool]:
|
49 |
"""Checks for the presence of required terms in the text."""
|
|
|
103 |
}
|
104 |
|
105 |
def check_language_issues(full_text: str) -> Dict[str, Any]:
|
106 |
+
"""Check for language issues using LanguageTool."""
|
107 |
+
try:
|
108 |
+
language_tool = language_tool_python.LanguageTool('en-US')
|
109 |
+
matches = language_tool.check(full_text)
|
110 |
+
issues = []
|
111 |
+
for match in matches:
|
112 |
+
issues.append({
|
113 |
+
"message": match.message,
|
114 |
+
"context": match.context.strip(),
|
115 |
+
"suggestions": match.replacements[:3] if match.replacements else [],
|
116 |
+
"category": match.category,
|
117 |
+
"rule_id": match.ruleId,
|
118 |
+
"offset": match.offset,
|
119 |
+
"length": match.errorLength
|
120 |
+
})
|
121 |
+
print(f"Total language issues found: {len(issues)}")
|
122 |
+
return {
|
123 |
+
"total_issues": len(issues),
|
124 |
+
"issues": issues
|
125 |
}
|
126 |
+
except Exception as e:
|
127 |
+
print(f"Error checking language issues: {e}")
|
128 |
+
return {"error": str(e)}
|
|
|
|
|
|
|
|
|
|
|
|
|
129 |
|
130 |
def check_language(full_text: str) -> Dict[str, Any]:
|
131 |
"""Check language quality."""
|
|
|
265 |
instances.append(inst)
|
266 |
return instances
|
267 |
|
268 |
+
def highlight_issues_in_pdf(file, language_matches: List[Dict[str, Any]]) -> bytes:
|
269 |
+
"""
|
270 |
+
Highlights language issues in the PDF and returns the annotated PDF as bytes.
|
271 |
+
This function maps LanguageTool matches to specific words in the PDF
|
272 |
+
and highlights those words.
|
273 |
+
"""
|
274 |
try:
|
275 |
+
# Open the PDF
|
276 |
+
doc = fitz.open(stream=file.read(), filetype="pdf") if not isinstance(file, str) else fitz.open(file)
|
277 |
+
print(f"Opened PDF with {len(doc)} pages.")
|
278 |
+
|
279 |
+
# Extract words with positions from each page
|
280 |
+
word_list = [] # List of tuples: (page_number, word, x0, y0, x1, y1)
|
281 |
+
for page_number in range(len(doc)):
|
282 |
+
page = doc[page_number]
|
283 |
+
words = page.get_text("words") # List of tuples: (x0, y0, x1, y1, "word", block_no, line_no, word_no)
|
284 |
+
for w in words:
|
285 |
+
word_text = w[4]
|
286 |
+
# **Fix:** Insert a space before '[' to ensure "globally [2]" instead of "globally[2]"
|
287 |
+
if '[' in word_text:
|
288 |
+
word_text = word_text.replace('[', ' [')
|
289 |
+
word_list.append((page_number, word_text, w[0], w[1], w[2], w[3]))
|
290 |
+
print(f"Total words extracted: {len(word_list)}")
|
291 |
+
|
292 |
+
# Concatenate all words to form the full text
|
293 |
+
concatenated_text = " ".join([w[1] for w in word_list])
|
294 |
+
print(f"Concatenated text length: {len(concatenated_text)} characters.")
|
295 |
+
|
296 |
+
# Iterate over each language issue
|
297 |
+
for idx, issue in enumerate(language_matches, start=1):
|
298 |
+
offset = issue["offset"]
|
299 |
+
length = issue["length"]
|
300 |
+
error_text = concatenated_text[offset:offset+length]
|
301 |
+
print(f"\nIssue {idx}: '{error_text}' at offset {offset} with length {length}")
|
302 |
+
|
303 |
+
# Find the words that fall within the error span
|
304 |
+
current_pos = 0
|
305 |
+
target_words = []
|
306 |
+
for word in word_list:
|
307 |
+
word_text = word[1]
|
308 |
+
word_length = len(word_text) + 1 # +1 for the space
|
309 |
+
|
310 |
+
if current_pos + word_length > offset and current_pos < offset + length:
|
311 |
+
target_words.append(word)
|
312 |
+
current_pos += word_length
|
313 |
+
|
314 |
+
if not target_words:
|
315 |
+
print("No matching words found for this issue.")
|
316 |
+
continue
|
317 |
+
|
318 |
+
# Add highlight annotations to the target words
|
319 |
+
for target in target_words:
|
320 |
+
page_num, word_text, x0, y0, x1, y1 = target
|
321 |
+
page = doc[page_num]
|
322 |
+
# Define a rectangle around the word with some padding
|
323 |
+
rect = fitz.Rect(x0 - 1, y0 - 1, x1 + 1, y1 + 1)
|
324 |
+
# Add a highlight annotation
|
325 |
+
highlight = page.add_highlight_annot(rect)
|
326 |
+
highlight.set_colors(stroke=(1, 1, 0)) # Yellow color
|
327 |
+
highlight.update()
|
328 |
+
print(f"Highlighted '{word_text}' on page {page_num + 1} at position ({x0}, {y0}, {x1}, {y1})")
|
329 |
+
|
330 |
+
# Save annotated PDF to bytes
|
331 |
+
byte_stream = io.BytesIO()
|
332 |
+
doc.save(byte_stream)
|
333 |
+
annotated_pdf_bytes = byte_stream.getvalue()
|
334 |
doc.close()
|
|
|
335 |
|
336 |
+
# Save annotated PDF locally for verification
|
337 |
+
with open("annotated_temp.pdf", "wb") as f:
|
338 |
+
f.write(annotated_pdf_bytes)
|
339 |
+
print("Annotated PDF saved as 'annotated_temp.pdf' for manual verification.")
|
340 |
+
|
341 |
+
return annotated_pdf_bytes
|
342 |
except Exception as e:
|
343 |
+
print(f"Error in highlighting PDF: {e}")
|
|
|
344 |
return b""
|
|
|
345 |
# ------------------------------
|
346 |
# Main Analysis Function
|
347 |
# ------------------------------
|
348 |
|
349 |
def analyze_pdf(file) -> Tuple[Dict[str, Any], bytes]:
|
350 |
+
"""Analyzes the PDF for language issues and returns results and annotated PDF."""
|
|
|
|
|
351 |
try:
|
352 |
+
# Reset file pointer before reading
|
353 |
+
file.seek(0)
|
354 |
full_text = extract_pdf_text(file)
|
355 |
+
if not full_text:
|
356 |
+
return {"error": "Failed to extract text from PDF."}, None
|
357 |
+
|
358 |
+
language_issues = check_language_issues(full_text)
|
359 |
+
if "error" in language_issues:
|
360 |
+
return language_issues, None
|
361 |
+
|
362 |
+
issues = language_issues.get("issues", [])
|
363 |
+
# Reset file pointer before highlighting
|
364 |
+
file.seek(0)
|
365 |
+
annotated_pdf = highlight_issues_in_pdf(file, issues) if issues else None
|
366 |
+
return language_issues, annotated_pdf
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
367 |
except Exception as e:
|
368 |
+
return {"error": str(e)}, None
|
|
|
|
|
|
|
|
|
369 |
|
370 |
# ------------------------------
|
371 |
# Gradio Interface
|