app.py

import fitz  # PyMuPDF
import gradio as gr
import requests
from bs4 import BeautifulSoup
import urllib.parse
import random
import os
from dotenv import load_dotenv
import shutil
import tempfile
load_dotenv()  # Load environment variables from .env file
# Now replace the hard-coded token with the environment variable
HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
def clear_cache():
   try:
       # Clear Gradio cache
       cache_dir = tempfile.gettempdir()
       shutil.rmtree(os.path.join(cache_dir, "gradio"), ignore_errors=True)
       # Clear any custom cache you might have
       # For example, if you're caching PDF files or search results:
       if os.path.exists("output_summary.pdf"):
           os.remove("output_summary.pdf")
       # Add any other cache clearing operations here
       print("Cache cleared successfully.")
       return "Cache cleared successfully."
   except Exception as e:
       print(f"Error clearing cache: {e}")
       return f"Error clearing cache: {e}"
_useragent_list = [
   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
]
# Function to extract visible text from HTML content of a webpage
def extract_text_from_webpage(html):
   print("Extracting text from webpage...")
   soup = BeautifulSoup(html, 'html.parser')
   for script in soup(["script", "style"]):
       script.extract()  # Remove scripts and styles
   text = soup.get_text()
   lines = (line.strip() for line in text.splitlines())
   chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
   text = '\n'.join(chunk for chunk in chunks if chunk)
   print(f"Extracted text length: {len(text)}")
   return text
# Function to perform a Google search and retrieve results
def google_search(term, num_results=5, lang="en", timeout=5, safe="active", ssl_verify=None):
   """Performs a Google search and returns the results."""
   print(f"Searching for term: {term}")
   escaped_term = urllib.parse.quote_plus(term)
   start = 0
   all_results = []
   max_chars_per_page = 8000  # Limit the number of characters from each webpage to stay under the token limit
   with requests.Session() as session:
       while start < num_results:
           print(f"Fetching search results starting from: {start}")
           try:
               # Choose a random user agent
               user_agent = random.choice(_useragent_list)
               headers = {
                   'User-Agent': user_agent
               }
               print(f"Using User-Agent: {headers['User-Agent']}")
               resp = session.get(
                   url="https://www.google.com/search",
                   headers=headers,
                   params={
                       "q": term,
                       "num": num_results - start,
                       "hl": lang,
                       "start": start,
                       "safe": safe,
                   },
                   timeout=timeout,
                   verify=ssl_verify,
               )
               resp.raise_for_status()
           except requests.exceptions.RequestException as e:
               print(f"Error fetching search results: {e}")
               break
           soup = BeautifulSoup(resp.text, "html.parser")
           result_block = soup.find_all("div", attrs={"class": "g"})
           if not result_block:
               print("No more results found.")
               break
           for result in result_block:
               link = result.find("a", href=True)
               if link:
                   link = link["href"]
                   print(f"Found link: {link}")
                   try:
                       webpage = session.get(link, headers=headers, timeout=timeout)
                       webpage.raise_for_status()
                       visible_text = extract_text_from_webpage(webpage.text)
                       if len(visible_text) > max_chars_per_page:
                           visible_text = visible_text[:max_chars_per_page] + "..."
                       all_results.append({"link": link, "text": visible_text})
                   except requests.exceptions.RequestException as e:
                       print(f"Error fetching or processing {link}: {e}")
                       all_results.append({"link": link, "text": None})
               else:
                   print("No link found in result.")
                   all_results.append({"link": None, "text": None})
           start += len(result_block)
   print(f"Total results fetched: {len(all_results)}")
   return all_results
# Function to format the prompt for the Hugging Face API
def format_prompt(query, search_results, instructions):
   formatted_results = ""
   for result in search_results:
       link = result["link"]
       text = result["text"]
       if link:
           formatted_results += f"URL: {link}\nContent: {text}\n{'-' * 80}\n"
       else:
           formatted_results += "No link found.\n" + '-' * 80 + '\n'
   prompt = f"{instructions}User Query: {query}\n\nWeb Search Results:\n{formatted_results}\n\nAssistant:"
   return prompt
# Function to generate text using Hugging Face API
def generate_text(input_text, temperature=0.7, repetition_penalty=1.0, top_p=0.9):
   print("Generating text using Hugging Face API...")
   endpoint = "https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.3"
   headers = {
       "Authorization": f"Bearer {HUGGINGFACE_API_TOKEN}",  # Use the environment variable
       "Content-Type": "application/json"
   }
   data = {
       "inputs": input_text,
       "parameters": {
           "max_new_tokens": 8000,  # Adjust as needed
           "temperature": temperature,
           "repetition_penalty": repetition_penalty,
           "top_p": top_p
       }
   }
   try:
       response = requests.post(endpoint, headers=headers, json=data)
       response.raise_for_status()
       # Check if response is JSON
       try:
           json_data = response.json()
       except ValueError:
           print("Response is not JSON.")
           return None
       # Extract generated text from response JSON
       if isinstance(json_data, list):
           # Handle list response (if applicable for your use case)
           generated_text = json_data[0].get("generated_text") if json_data else None
       elif isinstance(json_data, dict):
           # Handle dictionary response
           generated_text = json_data.get("generated_text")
       else:
           print("Unexpected response format.")
           return None
       if generated_text is not None:
           print("Text generation complete using Hugging Face API.")
           print(f"Generated text: {generated_text}")  # Debugging line
           return generated_text
       else:
           print("Generated text not found in response.")
           return None
   except requests.exceptions.RequestException as e:
       print(f"Error generating text using Hugging Face API: {e}")
       return None
# Function to read and extract text from a PDF
def read_pdf(file_obj):
   with fitz.open(file_obj.name) as document:
       text = ""
       for page_num in range(document.page_count):
           page = document.load_page(page_num)
           text += page.get_text()
       return text
# Function to format the prompt with instructions for text generation
def format_prompt_with_instructions(text, instructions):
   prompt = f"{instructions}{text}\n\nAssistant:"
   return prompt
# Function to save text to a PDF
def save_text_to_pdf(text, output_path):
   print(f"Saving text to PDF at {output_path}...")
   doc = fitz.open()  # Create a new PDF document
   page = doc.new_page()  # Create a new page
   # Set the page margins
   margin = 50  # 50 points margin
   page_width = page.rect.width
   page_height = page.rect.height
   text_width = page_width - 2 * margin
   text_height = page_height - 2 * margin
   # Define font size and line spacing
   font_size = 9
   line_spacing = 1 * font_size
   max_lines_per_page = int(text_height // line_spacing)
   # Load a built-in font
   font = "helv"
   # Split the text into lines
   lines = text.split("\n")
   current_line = 0
   for line in lines:
       if current_line >= max_lines_per_page:
           page = doc.new_page()  # Add a new page
           current_line = 0
       rect = fitz.Rect(margin, margin + current_line * line_spacing, text_width, margin + (current_line + 1) * line_spacing)
       page.insert_textbox(rect, line, fontsize=font_size, fontname=font, align=fitz.TEXT_ALIGN_LEFT)
       current_line += 1
   doc.save(output_path)
   print(f"Text saved to PDF at {output_path}.")
# Function to handle user queries
def handle_query(query, is_read_pdf, instructions):
   print("Handling user query...")
   max_chars_per_chunk = 1000  # Adjust this value as needed to control chunk size
   if is_read_pdf:
       pdf_text = read_pdf(query)
       text_chunks = [pdf_text[i:i+max_chars_per_chunk] for i in range(0, len(pdf_text), max_chars_per_chunk)]
   else:
       search_results = google_search(query)
       text_chunks = []
       for result in search_results:
           if result["text"]:
               text_chunks.extend([result["text"][i:i+max_chars_per_chunk] for i in range(0, len(result["text"]), max_chars_per_chunk)])
   summaries = []
   for chunk in text_chunks:
       formatted_prompt = format_prompt_with_instructions(chunk, instructions)
       summary = generate_text(formatted_prompt)
       if summary:
           summaries.append(summary)
   combined_summary = " ".join(summaries)
   save_text_to_pdf(combined_summary, "output_summary.pdf")
   return combined_summary
def run_app():
   with gr.Blocks() as demo:
       gr.Markdown("# Web and PDF Summarizer")
       query = gr.Textbox(label="Enter your query or upload a PDF", placeholder="Enter query here")
       is_read_pdf = gr.Checkbox(label="Read PDF", value=False)
       instructions = gr.Textbox(label="Enter instructions", placeholder="Enter instructions here")
       output = gr.Textbox(label="Summary")
       clear_cache_btn = gr.Button("Clear Cache")
       clear_cache_btn.click(fn=clear_cache, outputs=output)
       generate_btn = gr.Button("Generate Summary")
       generate_btn.click(fn=handle_query, inputs=[query, is_read_pdf, instructions], outputs=output)
   demo.launch()
run_app()