Update app.py

app.py

@@ -27,6 +27,9 @@ from scrapy import signals
 from scrapy.signalmanager import dispatcher
 from scrapy.utils.log import configure_logging
 from newspaper import Article
+from io import BytesIO
+from PyPDF2 import PdfReader
+import logging
 
 
 
@@ -119,10 +122,29 @@ def scrape_with_scrapy(url, timeout=30):
 
 def scrape_with_newspaper(url):
     logger.info(f"Starting to scrape with Newspaper3k: {url}")
-    article = Article(url)
-    article.download()
-    article.parse()
-    return article.text
+    try:
+        # Check if the URL is a PDF
+        response = requests.get(url)
+        content_type = response.headers.get('Content-Type', '').lower()
+        
+        if 'application/pdf' in content_type:
+            # Handle PDF
+            logger.info(f"Detected PDF file: {url}")
+            pdf_file = BytesIO(response.content)
+            pdf_reader = PdfReader(pdf_file)
+            text = ""
+            for page in pdf_reader.pages:
+                text += page.extract_text() + "\n"
+            return text.strip()
+        else:
+            # Handle regular web page
+            article = Article(url)
+            article.download()
+            article.parse()
+            return article.text
+    except Exception as e:
+        logger.error(f"Error scraping {url} with Newspaper3k: {e}")
+        return ""
 
 def scrape_with_bs4(url, session, max_chars=None):
     try: