app.py

import fitz  # PyMuPDF
import gradio as gr
import requests
from bs4 import BeautifulSoup
import urllib.parse
import random
import os
from dotenv import load_dotenv
import shutil
import tempfile
import re
import unicodedata
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.probability import FreqDist
import nltk
from datetime import datetime, timedelta

# Download necessary NLTK data
nltk.download('punkt')
nltk.download('stopwords')

load_dotenv()  # Load environment variables from .env file

# Now replace the hard-coded token with the environment variable
HUGGINGFACE_API_TOKEN = os.getenv("HUGGINGFACE_TOKEN")

def clear_cache():
    try:
        # Clear Gradio cache
        cache_dir = tempfile.gettempdir()
        shutil.rmtree(os.path.join(cache_dir, "gradio"), ignore_errors=True)
        
        # Clear any custom cache you might have
        # For example, if you're caching PDF files or search results:
        if os.path.exists("output_summary.pdf"):
            os.remove("output_summary.pdf")
        
        # Add any other cache clearing operations here
        
        print("Cache cleared successfully.")
        return "Cache cleared successfully."
    except Exception as e:
        print(f"Error clearing cache: {e}")
        return f"Error clearing cache: {e}"

PREDEFINED_QUERIES = {
    "Recent Earnings": {
        "query": "{company} recent quarterly earnings",
        "instructions": "Provide the most recent quarterly earnings data for {company}. Include revenue, net income, loan growth, deposit growth if any, EPS and asset quality. Specify the exact quarter and year."
    },
    "Recent News": {
        "query": "{company} recent news",
        "instructions": "Summarize the most recent significant news about {company}. Focus on events that could impact the company's financial performance or stock price."
    },
    "Credit Rating": {
        "query": "{company} current credit rating",
        "instructions": "Provide the most recent credit rating for {company}. Include the rating agency, the exact rating, and the date it was issued or last confirmed."
    },
    "Earnings Call Transcript": {
        "query": "{company} most recent earnings call transcript",
        "instructions": "Summarize key points from {company}'s most recent earnings call. Include date of the call, major financial highlights, and any significant forward-looking statements."
    }
}
_useragent_list = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
]

# Function to extract visible text from HTML content of a webpage
def extract_text_from_webpage(html):
    print("Extracting text from webpage...")
    soup = BeautifulSoup(html, 'html.parser')
    for script in soup(["script", "style"]):
        script.extract()  # Remove scripts and styles
    text = soup.get_text()
    lines = (line.strip() for line in text.splitlines())
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    text = '\n'.join(chunk for chunk in chunks if chunk)
    print(f"Extracted text length: {len(text)}")
    return text

# Function to perform a Google search and retrieve results
def google_search(term, num_results=5, lang="en", timeout=5, safe="active", ssl_verify=None, instructions="", days_back=365):
    print(f"Searching for term: {term}")
    
    # Calculate the date range
    end_date = datetime.now()
    start_date = end_date - timedelta(days=days_back)
    
    # Format dates as strings
    start_date_str = start_date.strftime("%Y-%m-%d")
    end_date_str = end_date.strftime("%Y-%m-%d")
    
    # Add the date range to the search term
    search_term = f"{term} after:{start_date_str} before:{end_date_str}"
    
    escaped_term = urllib.parse.quote_plus(search_term)
    start = 0
    all_results = []

    with requests.Session() as session:
        while len(all_results) < num_results:
            print(f"Fetching search results starting from: {start}")
            try:
                # Choose a random user agent
                user_agent = random.choice(_useragent_list)
                headers = {
                    'User-Agent': user_agent
                }
                print(f"Using User-Agent: {headers['User-Agent']}")

                resp = session.get(
                    url="https://www.google.com/search",
                    headers=headers,
                    params={
                        "q": search_term,
                        "num": num_results - start,
                        "hl": lang,
                        "start": start,
                        "safe": safe,
                    },
                    timeout=timeout,
                    verify=ssl_verify,
                )
                resp.raise_for_status()
            except requests.exceptions.RequestException as e:
                print(f"Error fetching search results: {e}")
                break

            soup = BeautifulSoup(resp.text, "html.parser")
            result_block = soup.find_all("div", attrs={"class": "g"})
            if not result_block:
                print("No more results found.")
                break
            keywords = term.split()  # Use the search term as keywords for filtering

            for result in result_block:
                if len(all_results) >= num_results:
                    break
                link = result.find("a", href=True)
                if link:
                    link = link["href"]
                    print(f"Found link: {link}")
                    try:
                        webpage = session.get(link, headers=headers, timeout=timeout)
                        webpage.raise_for_status()
                        visible_text = extract_text_from_webpage(webpage.text)
                        
                        # Summarize the webpage content
                        summary = summarize_webpage(link, visible_text, term, instructions)
                        
                        all_results.append({"link": link, "text": summary})
                    except requests.exceptions.RequestException as e:
                        print(f"Error fetching or processing {link}: {e}")
                        all_results.append({"link": link, "text": None})
                else:
                    print("No link found in result.")
                    all_results.append({"link": None, "text": None})

            start += len(result_block)

    print(f"Total results fetched: {len(all_results)}")
    return all_results

def google_news_search(term, num_results=5, lang="en", timeout=5, safe="active", ssl_verify=None, days_back=30):
    print(f"Searching Google News for term: {term}")
    
    # Calculate the date range
    end_date = datetime.now()
    start_date = end_date - timedelta(days=days_back)
    
    # Format dates as strings
    start_date_str = start_date.strftime("%Y-%m-%d")
    end_date_str = end_date.strftime("%Y-%m-%d")
    
    # Add the date range to the search term
    search_term = f"{term} after:{start_date_str} before:{end_date_str}"
    
    escaped_term = urllib.parse.quote_plus(search_term)
    start = 0
    all_results = []

    with requests.Session() as session:
        while len(all_results) < num_results:
            try:
                user_agent = random.choice(_useragent_list)
                headers = {
                    'User-Agent': user_agent
                }
                print(f"Using User-Agent: {headers['User-Agent']}")

                resp = session.get(
                    url="https://news.google.com/search",
                    headers=headers,
                    params={
                        "q": search_term,
                        "hl": lang,
                        "gl": "US",
                        "ceid": "US:en"
                    },
                    timeout=timeout,
                    verify=ssl_verify,
                )
                resp.raise_for_status()
            except requests.exceptions.RequestException as e:
                print(f"Error fetching search results: {e}")
                break

            soup = BeautifulSoup(resp.text, "html.parser")
            articles = soup.find_all("article")
            
            for article in articles:
                if len(all_results) >= num_results:
                    break
                link_element = article.find("a", attrs={"class": "WwrzSb"}) or article.find("a", href=True)
#                link_element = article.find("a", class_="WwrzSb")
                if link_element:
                    # Google News uses relative URLs, so we need to construct the full URL
                    relative_link = link_element['href']
                    full_link = f"https://news.google.com{relative_link[1:]}"  # Remove the leading '.'
                    
                    title = link_element.text
                    
                    try:
                        # Fetch the actual article
                        article_page = session.get(full_link, headers=headers, timeout=timeout)
                        article_page.raise_for_status()
                        article_content = extract_text_from_webpage(article_page.text)
                        
                        all_results.append({"link": full_link, "title": title, "text": article_content})
                    except requests.exceptions.RequestException as e:
                        print(f"Error fetching or processing {full_link}: {e}")
                        all_results.append({"link": full_link, "title": title, "text": None})
                else:
                    print("No link found in article.")

            if len(articles) == 0:
                print("No more results found.")
                break

            start += len(articles)

    print(f"Total news results fetched: {len(all_results)}")
    return all_results

def summarize_webpage(url, content, query, instructions, max_chars=1000):
    if content is None:
        return f"Unable to fetch or process content from {url}"

    # Extract keywords from the query
    keywords = query.split()

    # Apply full preprocessing pipeline
    preprocessed_text = preprocess_text(content)
    preprocessed_text = remove_boilerplate(preprocessed_text)
    filtered_text = keyword_filter(preprocessed_text, keywords)
    summarized_text = summarize_text(filtered_text, num_sentences=5)  # Adjust num_sentences as needed

    if not summarized_text:
        return f"No relevant content found for the query in {url}"

    # Format a prompt for this specific webpage
    webpage_prompt = f"""
    Instructions: {instructions}
    Query: {query}
    URL: {url}

    Filtered and summarized webpage content:
    {summarized_text}

    Based on the above filtered and summarized content, provide a concise summary that's directly relevant to the query. Focus on specific data, facts, or insights mentioned. Keep the summary under 200 words.

    Summary:
    """
    
    # Generate summary using the AI model
    summary = generate_text(webpage_prompt, temperature=0.3, repetition_penalty=1.2, top_p=0.9)
    
    # Truncate if necessary
    if summary and len(summary) > max_chars:
        summary = summary[:max_chars] + "..."
    
    return summary if summary else f"Unable to generate summary for {url}"

def preprocess_text(text):
    if text is None:
        return ""  # Return an empty string if input is None
    
    # Remove HTML tags
    text = BeautifulSoup(str(text), "html.parser").get_text()
    
    # Remove URLs
    text = re.sub(r'http\S+|www.\S+', '', text)
    
    # Remove special characters and digits
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    
    # Remove extra whitespace
    text = ' '.join(text.split())
    
    # Convert to lowercase
    text = text.lower()
    
    return text

def remove_boilerplate(text):
    # List of common boilerplate phrases to remove
    boilerplate = [
        "all rights reserved",
        "terms of service",
        "privacy policy",
        "cookie policy",
        "copyright ©",
        "follow us on social media"
    ]
    
    for phrase in boilerplate:
        text = text.replace(phrase, '')
    
    return text

def keyword_filter(text, keywords):
    sentences = sent_tokenize(text)
    filtered_sentences = [sentence for sentence in sentences if any(keyword.lower() in sentence.lower() for keyword in keywords)]
    return ' '.join(filtered_sentences)

def summarize_text(text, num_sentences=3):
    # Tokenize the text into words
    words = word_tokenize(text)
    
    # Remove stopwords
    stop_words = set(stopwords.words('english'))
    words = [word for word in words if word.lower() not in stop_words]
    
    # Calculate word frequencies
    freq_dist = FreqDist(words)
    
    # Score sentences based on word frequencies
    sentences = sent_tokenize(text)
    sentence_scores = {}
    for sentence in sentences:
        for word in word_tokenize(sentence.lower()):
            if word in freq_dist:
                if sentence not in sentence_scores:
                    sentence_scores[sentence] = freq_dist[word]
                else:
                    sentence_scores[sentence] += freq_dist[word]
    
    # Get the top N sentences with highest scores
    summary_sentences = sorted(sentence_scores, key=sentence_scores.get, reverse=True)[:num_sentences]
    
    # Sort the selected sentences in the order they appear in the original text
    summary_sentences = sorted(summary_sentences, key=text.index)
    
    return ' '.join(summary_sentences)

def preprocess_web_content(content, keywords):
    # Apply basic preprocessing
    preprocessed_text = preprocess_text(content)
    
    # Remove boilerplate
    preprocessed_text = remove_boilerplate(preprocessed_text)
    
    # Apply keyword filtering
    filtered_text = keyword_filter(preprocessed_text, keywords)
    
    # Summarize the text
    summarized_text = summarize_text(filtered_text)
    
    return summarized_text


# Function to format the prompt for the Hugging Face API
def format_prompt(query, search_results, instructions):
    formatted_results = ""
    for result in search_results:
        link = result["link"]
        summary = result["text"]
        if link and summary:
            formatted_results += f"URL: {link}\nSummary: {summary}\n{'-' * 80}\n"
        else:
            formatted_results += "No relevant information found.\n" + '-' * 80 + '\n'

    prompt = f"""Instructions: {instructions}
User Query: {query}

Summarized Web Search Results:
{formatted_results}

Based on the above summarized information from multiple sources, provide a comprehensive and factual response to the user's query. Include specific dates, numbers, and sources where available. If information is conflicting or unclear, mention this in your response. Do not make assumptions or provide information that is not supported by the summaries.

Assistant:"""
    return prompt

# Function to generate text using Hugging Face API
def generate_text(input_text, temperature=0.3, repetition_penalty=1.2, top_p=0.9):
    print("Generating text using Hugging Face API...")
    endpoint = "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3-8B-Instruct"
    headers = {
        "Authorization": f"Bearer {HUGGINGFACE_API_TOKEN}",
        "Content-Type": "application/json"
    }
    data = {
        "inputs": input_text,
        "parameters": {
            "max_new_tokens": 1000,  # Reduced to focus on more concise answers
            "temperature": temperature,
            "repetition_penalty": repetition_penalty,
            "top_p": top_p,
            "do_sample": True
        }
    }

    try:
        response = requests.post(endpoint, headers=headers, json=data)
        response.raise_for_status()

        # Check if response is JSON
        try:
            json_data = response.json()
        except ValueError:
            print("Response is not JSON.")
            return None

        # Extract generated text from response JSON
        if isinstance(json_data, list):
            # Handle list response (if applicable for your use case)
            generated_text = json_data[0].get("generated_text") if json_data else None
        elif isinstance(json_data, dict):
            # Handle dictionary response
            generated_text = json_data.get("generated_text")
        else:
            print("Unexpected response format.")
            return None

        if generated_text is not None:
            print("Text generation complete using Hugging Face API.")
            print(f"Generated text: {generated_text}")  # Debugging line
            return generated_text
        else:
            print("Generated text not found in response.")
            return None

    except requests.exceptions.RequestException as e:
        print(f"Error generating text using Hugging Face API: {e}")
        return None

# Function to read and extract text from a PDF
def read_pdf(file_obj):
    with fitz.open(file_obj.name) as document:
        text = ""
        for page_num in range(document.page_count):
            page = document.load_page(page_num)
            text += page.get_text()
        return text

# Function to format the prompt with instructions for text generation
def format_prompt_with_instructions(text, instructions):
    prompt = f"{instructions}{text}\n\nAssistant:"
    return prompt

# Function to save text to a PDF
def save_text_to_pdf(text, output_path):
    print(f"Saving text to PDF at {output_path}...")
    doc = fitz.open()  # Create a new PDF document
    page = doc.new_page()  # Create a new page

    # Set the page margins
    margin = 50  # 50 points margin
    page_width = page.rect.width
    page_height = page.rect.height
    text_width = page_width - 2 * margin
    text_height = page_height - 2 * margin

    # Define font size and line spacing
    font_size = 9
    line_spacing = 1 * font_size
    fontname = "times-roman"  # Use a supported font name

    # Process the text to handle line breaks and paragraphs
    paragraphs = text.split("\n")  # Split text into paragraphs
    y_position = margin

    for paragraph in paragraphs:
        words = paragraph.split()
        current_line = ""

        for word in words:
            word = str(word)  # Ensure word is treated as string
            # Calculate the length of the current line plus the new word
            current_line_length = fitz.get_text_length(current_line + " " + word, fontsize=font_size, fontname=fontname)
            if current_line_length <= text_width:
                current_line += " " + word
            else:
                page.insert_text(fitz.Point(margin, y_position), current_line.strip(), fontsize=font_size, fontname=fontname)
                y_position += line_spacing
                if y_position + line_spacing > page_height - margin:
                    page = doc.new_page()  # Add a new page if text exceeds page height
                    y_position = margin
                current_line = word

        # Add the last line of the paragraph
        page.insert_text(fitz.Point(margin, y_position), current_line.strip(), fontsize=font_size, fontname=fontname)
        y_position += line_spacing

        # Add extra space for new paragraph
        y_position += line_spacing
        if y_position + line_spacing > page_height - margin:
            page = doc.new_page()  # Add a new page if text exceeds page height
            y_position = margin

    doc.save(output_path)  # Save the PDF to the specified path
    print("PDF saved successfully.")

# Integrated function to perform web scraping, formatting, and text generation
def scrape_and_display(query, num_results, instructions, web_search=True, use_news=False, days_back=None, temperature=0.7, repetition_penalty=1.0, top_p=0.9):
    print(f"Scraping and displaying results for query: {query} with num_results: {num_results}")
    if web_search:
        if days_back is None:
            current_year = datetime.now().year
            days_back = 365 if current_year % 4 != 0 else 366  # Account for leap years

        if use_news:
            # For news, we might want to use a shorter time frame by default
            news_days_back = min(days_back, 30)  # Use at most 30 days for news
            search_results = google_news_search(query, num_results, days_back=news_days_back)
        else:
            search_results = google_search(query, num_results=num_results, instructions=instructions, days_back=days_back)
        
        
        # Summarize each result
        summarized_results = []
        for result in search_results:
            try:
                summary = summarize_webpage(result['link'], result.get('text'), query, instructions)
                summarized_results.append({"link": result['link'], "text": summary})
            except Exception as e:
                print(f"Error summarizing {result['link']}: {e}")
                summarized_results.append({"link": result['link'], "text": f"Error summarizing content: {str(e)}"})
        
        formatted_prompt = format_prompt(query, summarized_results, instructions)
        generated_summary = generate_text(formatted_prompt, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
    else:
        formatted_prompt = format_prompt_with_instructions(query, instructions)
        generated_summary = generate_text(formatted_prompt, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
    
    print("Scraping and display complete.")
    if generated_summary:
        assistant_index = generated_summary.find("Assistant:")
        if assistant_index != -1:
            generated_summary = generated_summary[assistant_index:]
        else:
            generated_summary = "Assistant: No response generated."
    print(f"Generated summary: {generated_summary}")
    return generated_summary

# Main Gradio interface function
def gradio_interface(query, use_dashboard, use_news, use_pdf, pdf, num_results, custom_instructions, temperature, repetition_penalty, top_p, clear_cache_flag):
    if clear_cache_flag:
        return clear_cache()
    
    if use_dashboard:
        results = []
        for query_type, query_info in PREDEFINED_QUERIES.items():
            formatted_query = query_info['query'].format(company=query)
            formatted_instructions = query_info['instructions'].format(company=query)
            result = scrape_and_display(formatted_query, num_results=num_results, instructions=formatted_instructions, web_search=True, use_news=(query_type == "Recent News"), temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
            results.append(f"**{query_type}**\n\n{result}\n\n")
        generated_summary = "\n".join(results)
    elif use_pdf and pdf is not None:
        pdf_text = read_pdf(pdf)
        generated_summary = scrape_and_display(pdf_text, num_results=0, instructions=custom_instructions, web_search=False, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
    else:
        generated_summary = scrape_and_display(query, num_results=num_results, instructions=custom_instructions, web_search=True, use_news=use_news, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
    
    output_pdf_path = "output_summary.pdf"
    save_text_to_pdf(generated_summary, output_pdf_path)
    
    return generated_summary, output_pdf_path

# Update the Gradio Interface
gr.Interface(
    fn=gradio_interface,
    inputs=[
        gr.Textbox(label="Company Name or Query"),
        gr.Checkbox(label="Use Dashboard"),
        gr.Checkbox(label="Use News Search"),  # New checkbox for news search
        gr.Checkbox(label="Use PDF"),
        gr.File(label="Upload PDF"),
        gr.Slider(minimum=1, maximum=20, value=5, step=1, label="Number of Results"),
        gr.Textbox(label="Custom Instructions"),
        gr.Slider(minimum=0.1, maximum=1.0, value=0.7, step=0.1, label="Temperature"),
        gr.Slider(minimum=0.1, maximum=2.0, value=1.0, step=0.1, label="Repetition Penalty"),
        gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.1, label="Top p"),
        gr.Checkbox(label="Clear Cache", visible=False)
    ],
    outputs=["text", gr.File(label="Generated PDF")],
    title="Financial Analyst AI Assistant",
    description="Enter a company name to get a financial dashboard, or enter a custom query. Use the news search option for recent articles. Optionally, upload a PDF for analysis. Adjust parameters as needed for optimal results.",
    allow_flagging="never"
).launch(share=True)