samyak152002 commited on
Commit
e444d56
·
verified ·
1 Parent(s): d93d2aa

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +125 -109
app.py CHANGED
@@ -29,12 +29,21 @@ def extract_pdf_text_by_page(file) -> List[str]:
29
  return [page.get_text("text") for page in doc]
30
 
31
  def extract_pdf_text(file) -> str:
32
- """Extracts text from a PDF file using pdfminer."""
33
- if isinstance(file, str):
34
- with open(file, 'rb') as f:
35
- return extract_text(f, laparams=LAParams())
36
- else:
37
- return extract_text(file, laparams=LAParams())
 
 
 
 
 
 
 
 
 
38
 
39
  def check_text_presence(full_text: str, search_terms: List[str]) -> Dict[str, bool]:
40
  """Checks for the presence of required terms in the text."""
@@ -94,32 +103,29 @@ def check_structure(full_text: str) -> Dict[str, bool]:
94
  }
95
 
96
  def check_language_issues(full_text: str) -> Dict[str, Any]:
97
- """Check for issues with capitalization, hyphenation, punctuation, spacing, etc."""
98
- language_tool = language_tool_python.LanguageTool('en-US')
99
- matches = language_tool.check(full_text)
100
- word_count = len(full_text.split())
101
- issues_count = len(matches)
102
- issues_per_1000 = (issues_count / word_count) * 1000 if word_count else 0
103
-
104
- serializable_matches = [
105
- {
106
- "message": match.message,
107
- "replacements": match.replacements,
108
- "offset": match.offset,
109
- "errorLength": match.errorLength,
110
- "category": match.category,
111
- "ruleIssueType": match.ruleIssueType,
112
- "sentence": match.sentence
 
 
 
113
  }
114
- for match in matches
115
- ]
116
-
117
- return {
118
- "issues_count": issues_count,
119
- "issues_per_1000": issues_per_1000,
120
- "failed": issues_per_1000 > 20,
121
- "matches": serializable_matches
122
- }
123
 
124
  def check_language(full_text: str) -> Dict[str, Any]:
125
  """Check language quality."""
@@ -259,97 +265,107 @@ def find_text_instances(words, text):
259
  instances.append(inst)
260
  return instances
261
 
262
- def highlight_issues_in_pdf(file, inconsistent_refs: List[Tuple[int, str, str]], language_matches: List[Dict[str, Any]]) -> bytes:
263
- """Highlight inconsistent references and add notes for language issues in a single PDF."""
 
 
 
 
264
  try:
265
- if isinstance(file, str):
266
- doc = fitz.open(file)
267
- else:
268
- doc = fitz.open(stream=file.read(), filetype="pdf")
269
-
270
- added_notes = set()
271
-
272
- for page_number, page in enumerate(doc, start=1):
273
- words = page.get_text("words")
274
-
275
- if inconsistent_refs:
276
- for ref_num, ref_text, ref_style in inconsistent_refs:
277
- annotation_text = f"Reference {ref_num}: Inconsistent style ({ref_style}). Should be consolidated to {ref_style}."
278
- highlight_text(page, words, ref_text, annotation_text)
279
-
280
- if language_matches:
281
- for match in language_matches:
282
- issue_text = match['sentence']
283
- error_message = f"{match['message']}\nSuggested correction: {match['replacements'][0] if match['replacements'] else 'No suggestion'}"
284
- issue_key = (issue_text, error_message)
285
-
286
- if issue_key not in added_notes:
287
- if highlight_text(page, words, issue_text, error_message):
288
- added_notes.add(issue_key)
289
-
290
- annotated_pdf_bytes = doc.write()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
291
  doc.close()
292
- return annotated_pdf_bytes
293
 
 
 
 
 
 
 
294
  except Exception as e:
295
- print(f"An error occurred while annotating the PDF: {str(e)}")
296
- traceback.print_exc()
297
  return b""
298
-
299
  # ------------------------------
300
  # Main Analysis Function
301
  # ------------------------------
302
 
303
  def analyze_pdf(file) -> Tuple[Dict[str, Any], bytes]:
304
- """
305
- Analyze the uploaded PDF and return analysis results and annotated PDF bytes.
306
- """
307
  try:
308
- pages_text = extract_pdf_text_by_page(file)
 
309
  full_text = extract_pdf_text(file)
310
- full_text = label_authors(full_text)
311
-
312
- # Perform analyses
313
- metadata = check_metadata(full_text)
314
- disclosures = check_disclosures(full_text)
315
- figures_and_tables = check_figures_and_tables(full_text)
316
- figure_order = check_figure_order(full_text)
317
- references = check_references(full_text)
318
- reference_order = check_reference_order(full_text)
319
- reference_style = check_reference_style(full_text)
320
- structure = check_structure(full_text)
321
- language = check_language(full_text)
322
-
323
- # Compile results
324
- results = {
325
- "metadata": metadata,
326
- "disclosures": disclosures,
327
- "figures_and_tables": figures_and_tables,
328
- "figure_order": figure_order,
329
- "references": references,
330
- "reference_order": reference_order,
331
- "reference_style": reference_style,
332
- "structure": structure,
333
- "language": language
334
- }
335
-
336
- # Handle annotations
337
- inconsistent_refs = reference_style.get("inconsistent_refs", [])
338
- language_matches = language.get("language_issues", {}).get("matches", [])
339
-
340
- if inconsistent_refs or language_matches:
341
- annotated_pdf_bytes = highlight_issues_in_pdf(file, inconsistent_refs, language_matches)
342
- else:
343
- annotated_pdf_bytes = None
344
-
345
- return results, annotated_pdf_bytes
346
-
347
  except Exception as e:
348
- error_message = {
349
- "error": str(e),
350
- "traceback": traceback.format_exc()
351
- }
352
- return error_message, None
353
 
354
  # ------------------------------
355
  # Gradio Interface
 
29
  return [page.get_text("text") for page in doc]
30
 
31
  def extract_pdf_text(file) -> str:
32
+ """Extracts full text from a PDF file using PyMuPDF."""
33
+ try:
34
+ # Open the PDF file
35
+ doc = fitz.open(stream=file.read(), filetype="pdf") if not isinstance(file, str) else fitz.open(file)
36
+ full_text = ""
37
+ for page_num, page in enumerate(doc, start=1):
38
+ text = page.get_text("text")
39
+ full_text += text + "\n"
40
+ print(f"Extracted text from page {page_num}: {len(text)} characters.")
41
+ doc.close()
42
+ print(f"Total extracted text length: {len(full_text)} characters.")
43
+ return full_text
44
+ except Exception as e:
45
+ print(f"Error extracting text from PDF: {e}")
46
+ return ""
47
 
48
  def check_text_presence(full_text: str, search_terms: List[str]) -> Dict[str, bool]:
49
  """Checks for the presence of required terms in the text."""
 
103
  }
104
 
105
  def check_language_issues(full_text: str) -> Dict[str, Any]:
106
+ """Check for language issues using LanguageTool."""
107
+ try:
108
+ language_tool = language_tool_python.LanguageTool('en-US')
109
+ matches = language_tool.check(full_text)
110
+ issues = []
111
+ for match in matches:
112
+ issues.append({
113
+ "message": match.message,
114
+ "context": match.context.strip(),
115
+ "suggestions": match.replacements[:3] if match.replacements else [],
116
+ "category": match.category,
117
+ "rule_id": match.ruleId,
118
+ "offset": match.offset,
119
+ "length": match.errorLength
120
+ })
121
+ print(f"Total language issues found: {len(issues)}")
122
+ return {
123
+ "total_issues": len(issues),
124
+ "issues": issues
125
  }
126
+ except Exception as e:
127
+ print(f"Error checking language issues: {e}")
128
+ return {"error": str(e)}
 
 
 
 
 
 
129
 
130
  def check_language(full_text: str) -> Dict[str, Any]:
131
  """Check language quality."""
 
265
  instances.append(inst)
266
  return instances
267
 
268
+ def highlight_issues_in_pdf(file, language_matches: List[Dict[str, Any]]) -> bytes:
269
+ """
270
+ Highlights language issues in the PDF and returns the annotated PDF as bytes.
271
+ This function maps LanguageTool matches to specific words in the PDF
272
+ and highlights those words.
273
+ """
274
  try:
275
+ # Open the PDF
276
+ doc = fitz.open(stream=file.read(), filetype="pdf") if not isinstance(file, str) else fitz.open(file)
277
+ print(f"Opened PDF with {len(doc)} pages.")
278
+
279
+ # Extract words with positions from each page
280
+ word_list = [] # List of tuples: (page_number, word, x0, y0, x1, y1)
281
+ for page_number in range(len(doc)):
282
+ page = doc[page_number]
283
+ words = page.get_text("words") # List of tuples: (x0, y0, x1, y1, "word", block_no, line_no, word_no)
284
+ for w in words:
285
+ word_text = w[4]
286
+ # **Fix:** Insert a space before '[' to ensure "globally [2]" instead of "globally[2]"
287
+ if '[' in word_text:
288
+ word_text = word_text.replace('[', ' [')
289
+ word_list.append((page_number, word_text, w[0], w[1], w[2], w[3]))
290
+ print(f"Total words extracted: {len(word_list)}")
291
+
292
+ # Concatenate all words to form the full text
293
+ concatenated_text = " ".join([w[1] for w in word_list])
294
+ print(f"Concatenated text length: {len(concatenated_text)} characters.")
295
+
296
+ # Iterate over each language issue
297
+ for idx, issue in enumerate(language_matches, start=1):
298
+ offset = issue["offset"]
299
+ length = issue["length"]
300
+ error_text = concatenated_text[offset:offset+length]
301
+ print(f"\nIssue {idx}: '{error_text}' at offset {offset} with length {length}")
302
+
303
+ # Find the words that fall within the error span
304
+ current_pos = 0
305
+ target_words = []
306
+ for word in word_list:
307
+ word_text = word[1]
308
+ word_length = len(word_text) + 1 # +1 for the space
309
+
310
+ if current_pos + word_length > offset and current_pos < offset + length:
311
+ target_words.append(word)
312
+ current_pos += word_length
313
+
314
+ if not target_words:
315
+ print("No matching words found for this issue.")
316
+ continue
317
+
318
+ # Add highlight annotations to the target words
319
+ for target in target_words:
320
+ page_num, word_text, x0, y0, x1, y1 = target
321
+ page = doc[page_num]
322
+ # Define a rectangle around the word with some padding
323
+ rect = fitz.Rect(x0 - 1, y0 - 1, x1 + 1, y1 + 1)
324
+ # Add a highlight annotation
325
+ highlight = page.add_highlight_annot(rect)
326
+ highlight.set_colors(stroke=(1, 1, 0)) # Yellow color
327
+ highlight.update()
328
+ print(f"Highlighted '{word_text}' on page {page_num + 1} at position ({x0}, {y0}, {x1}, {y1})")
329
+
330
+ # Save annotated PDF to bytes
331
+ byte_stream = io.BytesIO()
332
+ doc.save(byte_stream)
333
+ annotated_pdf_bytes = byte_stream.getvalue()
334
  doc.close()
 
335
 
336
+ # Save annotated PDF locally for verification
337
+ with open("annotated_temp.pdf", "wb") as f:
338
+ f.write(annotated_pdf_bytes)
339
+ print("Annotated PDF saved as 'annotated_temp.pdf' for manual verification.")
340
+
341
+ return annotated_pdf_bytes
342
  except Exception as e:
343
+ print(f"Error in highlighting PDF: {e}")
 
344
  return b""
 
345
  # ------------------------------
346
  # Main Analysis Function
347
  # ------------------------------
348
 
349
  def analyze_pdf(file) -> Tuple[Dict[str, Any], bytes]:
350
+ """Analyzes the PDF for language issues and returns results and annotated PDF."""
 
 
351
  try:
352
+ # Reset file pointer before reading
353
+ file.seek(0)
354
  full_text = extract_pdf_text(file)
355
+ if not full_text:
356
+ return {"error": "Failed to extract text from PDF."}, None
357
+
358
+ language_issues = check_language_issues(full_text)
359
+ if "error" in language_issues:
360
+ return language_issues, None
361
+
362
+ issues = language_issues.get("issues", [])
363
+ # Reset file pointer before highlighting
364
+ file.seek(0)
365
+ annotated_pdf = highlight_issues_in_pdf(file, issues) if issues else None
366
+ return language_issues, annotated_pdf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
367
  except Exception as e:
368
+ return {"error": str(e)}, None
 
 
 
 
369
 
370
  # ------------------------------
371
  # Gradio Interface