app.py

import gradio as gr
import pandas as pd
from dotenv import load_dotenv
from langchain_community.llms import CTransformers, HuggingFacePipeline, HuggingFaceHub
from langchain_core.prompts import PromptTemplate
from langchain.chains import LLMChain
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
from sentence_transformers import SentenceTransformer, util
from sklearn.cluster import KMeans
import nltk
import pandas as pd
import smtplib
import os
nltk.download('punkt')
from nltk.tokenize import word_tokenize
from nltk import tokenize
import numpy as np
import scipy.spatial
import csv


load_dotenv()

def generate_prompts(user_input):
    prompt_template = PromptTemplate(
        input_variables=["Question"],
        template=f"Just list 10 question prompts for {user_input} and don't put number before each of the prompts."
    )
    config = {'max_new_tokens': 64, 'temperature': 0.7, 'context_length': 64}
    llm = CTransformers(model="TheBloke/Mistral-7B-Instruct-v0.1-GGUF",
                        config=config)
    hub_chain = LLMChain(prompt = prompt_template, llm = llm)
    input_data = {"Question": user_input}

    generated_prompts = hub_chain.run(input_data)  
    questions_list = generated_prompts.split('\n') 
    

    formatted_questions = "\n".join(f"Question: {question}" for i, question in enumerate(questions_list) if question.strip())
    questions_list = formatted_questions.split("Question:")[1:]
    return questions_list

def answer_question(prompt):
    prompt_template = PromptTemplate(
        input_variables=["Question"],
        template=f"give one answer for {prompt} and do not consider the number behind it."
    )
    config = {'max_new_tokens': 64, 'temperature': 0.7, 'context_length': 64}
    llm = CTransformers(model="TheBloke/Llama-2-7B-Chat-GGML",
                        config=config)
    hub_chain = LLMChain(prompt = prompt_template, llm = llm)
    input_data = {"Question": prompt}
    generated_answer = hub_chain.run(input_data)  
    return generated_answer

def calculate_similarity(word, other_words, model, threshold=0.5):
    embeddings_word = model.encode([word])
    embeddings_other_words = model.encode(other_words)
    for i, embedding in enumerate(embeddings_other_words):
        similarity = 1 - scipy.spatial.distance.cosine(embeddings_word[0], embedding)
        if similarity > threshold and similarity < 0.85:
            return i, similarity
    return None, None


def highlight_similar_paragraphs_with_colors(paragraphs, similarity_threshold=0.75):
    model = SentenceTransformer('all-MiniLM-L6-v2')
    
    # Split each paragraph into sentences
    all_sentences = [tokenize.sent_tokenize(paragraph) for paragraph in paragraphs]
    
    # Initialize storage for highlighted sentences
    highlighted_sentences = [['' for sentence in para] for para in all_sentences]
    colors = ['yellow', 'lightgreen', 'lightblue', 'pink', 'lavender', 'salmon', 'peachpuff', 'powderblue', 'khaki', 'wheat']
    
    # Track which sentences belong to which paragraph
    sentence_to_paragraph_index = [idx for idx, para in enumerate(all_sentences) for sentence in para]
    
    # Encode all sentences into vectors
    flattened_sentences = [sentence for para in all_sentences for sentence in para]
    sentence_embeddings = model.encode(flattened_sentences)
    
    # Calculate cosine similarities between all pairs of sentences
    cosine_similarities = util.pytorch_cos_sim(sentence_embeddings, sentence_embeddings)
    
    # Iterate through each sentence pair and highlight if they are similar but from different paragraphs
    color_index = 0
    for i, embedding_i in enumerate(sentence_embeddings):
        for j, embedding_j in enumerate(sentence_embeddings):
            if i != j and cosine_similarities[i, j] > similarity_threshold and sentence_to_paragraph_index[i] != sentence_to_paragraph_index[j]:
                color = colors[color_index % len(colors)]
                if highlighted_sentences[sentence_to_paragraph_index[i]][i % len(all_sentences[sentence_to_paragraph_index[i]])] == '':
                    highlighted_sentences[sentence_to_paragraph_index[i]][i % len(all_sentences[sentence_to_paragraph_index[i]])] = ("<span style='color: "+  color  +"'>"+ flattened_sentences[i]+"</span>")
                if highlighted_sentences[sentence_to_paragraph_index[j]][j % len(all_sentences[sentence_to_paragraph_index[j]])] == '':
                    highlighted_sentences[sentence_to_paragraph_index[j]][j % len(all_sentences[sentence_to_paragraph_index[j]])] = ("<span style='color: "+  color  +"'>"+ flattened_sentences[j]+"</span>")
                color_index += 1  # Move to the next color

    # Combine sentences back into paragraphs
    highlighted_paragraphs = [' '.join(para) for para in highlighted_sentences]

    # Combine all paragraphs into one HTML string
    html_output = '<div>' + '<br/><br/>'.join(highlighted_paragraphs) + '</div>'
    return highlighted_paragraphs

    
def calculate_similarity_score(sentences):
    # Encode all sentences to get their embeddings
    model = SentenceTransformer('all-MiniLM-L6-v2')
    embeddings = model.encode(sentences)

    # Calculate average cosine similarity
    total_similarity = 0
    comparisons = 0
    for i in range(len(embeddings)):
        for j in range(i+1, len(embeddings)):
            # Cosine similarity between embeddings
            similarity = 1 - cosine(embeddings[i], embeddings[j])
            total_similarity += similarity
            comparisons += 1

    # Average similarity
    average_similarity = total_similarity / comparisons if comparisons > 0 else 0

    # Scale from [-1, 1] to [0, 100]
    score_out_of_100 = (average_similarity + 1) / 2 * 100
    return score_out_of_100

def answer_question1(prompt):
    prompt_template = PromptTemplate.from_template(
        input_variables=["Question"],
        template=f"give one answer for {prompt} and do not consider the number behind it."
    )
    config = {'max_new_tokens': 2048, 'temperature': 0.7, 'context_length': 4096}
    llm = CTransformers(model="TheBloke/Llama-2-7B-Chat-GGML",
                        config=config,
                        threads=os.cpu_count())
    hub_chain = LLMChain(prompt = prompt_template, llm = llm)
    input_data = {"Question": prompt}
    generated_answer = hub_chain.run(input_data)  
    return generated_answer

    
def process_inputs(llm, file, relevance, diversity):
    # Check if file is uploaded
    if file is not None:
        # Read questions from the uploaded Excel file
        try:
            df = pd.read_excel(file, engine='openpyxl')
        except Exception as e:
            return f"Failed to read Excel file: {e}", None

        # Ensure that there is only one column in the file
        if df.shape[1] != 1:
            return "The uploaded file must contain only one column of questions.", None

        # Extract the first column
        questions_list = df.iloc[:, 0]

        # Initialize lists to store the expanded data
        # Initialize lists to store the expanded data
        expanded_questions = []
        expanded_prompts = []
        expanded_answers = []

        # Generate prompts for each question and expand the data
        for question in questions_list:
            prompts = generate_prompts(question)
            expanded_questions.extend([question] * len(prompts))
            expanded_prompts.extend(prompts)

            # Generate answers for each prompt
            answers = [answer_question(prompt) for prompt in prompts]
            expanded_answers.extend(answers)

        # Combine the expanded data into a DataFrame
        output_df = pd.DataFrame({
            'Questions': expanded_questions,
            'Generated Prompts': expanded_prompts,
            'Answers': expanded_answers
        })

        # Save the DataFrame to a new Excel file
        output_file = "processed_questions.xlsx"
        output_df.to_excel(output_file, index=False)
    else:
        return "No questions provided.", None

    return "Processing complete. Download the file below.", output_file
    

text_list = []

def updateChoices(prompt):
    newChoices = generate_prompts(prompt)
    return gr.CheckboxGroup(choices=newChoices)

def setTextVisibility(cbg, model_name_input):

    sentences = [answer_question(text, model_name_input) for text in cbg]
    
    # Apply highlighting to all processed sentences, receiving one complete HTML string.
    highlighted_html = []
    highlighted_html = highlight_similar_paragraphs_with_colors(sentences, similarity_threshold=0.75)
    
    
    result = []
    # Iterate through each original 'cbg' sentence and pair it with the entire highlighted block.
    for idx, sentence in enumerate(highlighted_html):
        result.append("<p><strong>"+ cbg[idx] +"</strong></p><p>"+ sentence +"</p><br/>")

    score = round(calculate_similarity_score(highlighted_html))

    final_html = f"""<div>{result}<div style="text-align: center; font-size: 24px; font-weight: bold;">Similarity Score: {score}</div></div>"""


    return final_html
    

    # update_show = [gr.Textbox(visible=True, label=text, value=answer_question(text, model_name_input)) for text in cbg]
    # update_hide = [gr.Textbox(visible=False, label="") for _ in range(10-len(cbg))]
    # return update_show + update_hide

with gr.Blocks(theme=gr.themes.Soft()) as demo:
    
    with gr.Tab("Live Mode"):

        gr.Markdown ("## Live Mode Auditing LLMs")
        gr.Markdown("In Live Auditing Mode, you gain the ability to probe the LLM directly.")
        gr.Markdown("First, select the LLM you wish to audit. Then, enter your question. The AuditLLM tool will generate five relevant and diverse prompts based on your question. You can now select these prompts for auditing the LLMs. Examine the similarity scores in the answers generated from these prompts to assess the LLM's performance effectively") 
        with gr.Row():
            model_name_input = gr.Dropdown([("Llama", "TheBloke/Llama-2-7B-Chat-GGML"), ("Falcon", "TheBloke/Falcon-180B-Chat-GGUF"), ("Zephyr", "TheBloke/zephyr-quiklang-3b-4K-GGUF"),("Vicuna", "TheBloke/vicuna-33B-GGUF"),("Claude","TheBloke/claude2-alpaca-13B-GGUF"),("Alpaca","TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], label="Large Language Model")
        with gr.Row():
            prompt_input = gr.Textbox(label="Enter your question", placeholder="Enter Your Question")
        with gr.Row():
            generate_button = gr.Button("Generate", variant="primary", min_width=300)
        with gr.Column():
            cbg = gr.CheckboxGroup(choices=[], label="List of the prompts", interactive=True)
        
        generate_button.click(updateChoices, inputs=[prompt_input], outputs=[cbg])

        with gr.Row() as exec: 
            btnExec = gr.Button("Execute", variant="primary", min_width=200)


        with gr.Column() as texts:
            for i in range(10):
                text = gr.Textbox(label="_", visible=False)
                text_list.append(text)

        with gr.Column():
            html_result = gr.HTML("""<div style="color: red"></div>""")

        #btnExec.click(setTextVisibility, inputs=[cbg, model_name_input], outputs=text_list)
        btnExec.click(setTextVisibility, inputs=[cbg, model_name_input], outputs=html_result)
        gr.HTML("""
        <div style="text-align: center; font-size: 24px; font-weight: bold;">Similarity Score: </div>
                """)

        clear = gr.ClearButton(link = "http://127.0.0.1:7865") 

    with gr.Tab("Batch Mode"):
        
        gr.Markdown("## Batch Mode Auditing LLMs")
        gr.Markdown("In batch auditing mode, you have the capability to probe the LLM. To begin, you must first select the LLM you wish to audit and then input the questions you intend to explore. For each question submitted, the model will generate five prompts, each accompanied by its respective answers.")
        gr.Markdown("To tailor the generation of these five prompts from your original question, you can adjust the relevance and diversity scores. The relevance score determines how closely the generated prompts should align with the original question, while the diversity score dictates the variance among the prompts themselves.")
        gr.Markdown("Upon completion, please provide your email address. We will compile and send the answers to you promptly.")
            
        llm_dropdown = gr.Dropdown(choices=[
                ("Llama", "TheBloke/Llama-2-7B-Chat-GGML"), 
                ("Falcon", "TheBloke/Falcon-180B-Chat-GGUF"), 
                ("Zephyr", "TheBloke/zephyr-quiklang-3b-4K-GGUF"), 
                ("Vicuna", "TheBloke/vicuna-33B-GGUF"), 
                ("Claude", "TheBloke/claude2-alpaca-13B-GGUF"), 
                ("Alpaca", "TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], 
                label="Large Language Model")
        file_upload = gr.File(label="Upload an Excel File with Questions", file_types=[".xlsx"])
        with gr.Row():
            relevance_slider = gr.Slider(1, 100, value=70, label="Relevance", info="Choose between 0 and 100", interactive=True)
            diversity_slider = gr.Slider(1, 100, value=25, label="Diversity", info="Choose between 0 and 100", interactive=True) 

            
        submit_button = gr.Button("Submit", variant="primary", min_width=200)
        output_textbox = gr.Textbox(label="Output")
        download_button = gr.File(label="Download Processed File")

        def on_submit(llm, file, relevance, diversity):
            result, output_file = process_inputs(llm, file, relevance, diversity)
            return result, output_file

        submit_button.click(fn=on_submit, inputs=[llm_dropdown, file_upload, relevance_slider, diversity_slider], outputs=[output_textbox, download_button])

# Launch the Gradio app
demo.launch()