Update app.py

app.py

@@ -1,20 +1,12 @@
 import gradio as gr
 import os
 from typing import List, Dict
-from ragas import evaluate
-from ragas.metrics import (
-    answer_relevancy,
-    faithfulness,
-    context_recall,
-    context_precision,
-    answer_correctness,
-    answer_similarity
-)
+import numpy as np
 from datasets import load_dataset
 from langchain.text_splitter import (
     RecursiveCharacterTextSplitter,
     CharacterTextSplitter,
-    SemanticTextSplitter
+    TokenTextSplitter
 )
 from langchain_community.vectorstores import FAISS, Chroma, Qdrant
 from langchain_community.document_loaders import PyPDFLoader
@@ -22,13 +14,17 @@ from langchain.chains import ConversationalRetrievalChain
 from langchain_community.embeddings import HuggingFaceEmbeddings 
 from langchain_community.llms import HuggingFaceEndpoint
 from langchain.memory import ConversationBufferMemory
+from sentence_transformers import SentenceTransformer, util
 import torch
 
-# Constants
+# Constants and setup
 list_llm = ["meta-llama/Meta-Llama-3-8B-Instruct", "mistralai/Mistral-7B-Instruct-v0.2"]
 list_llm_simple = [os.path.basename(llm) for llm in list_llm]
 api_token = os.getenv("HF_TOKEN")
 
+# Initialize sentence transformer for evaluation
+sentence_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+
 # Text splitting strategies
 def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int = 64):
     splitters = {
@@ -40,14 +36,38 @@ def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int
             chunk_size=chunk_size,
             chunk_overlap=chunk_overlap
         ),
-        "semantic": SemanticTextSplitter(
-            embedding_function=HuggingFaceEmbeddings().embed_query,
+        "token": TokenTextSplitter(
             chunk_size=chunk_size,
             chunk_overlap=chunk_overlap
         )
     }
     return splitters.get(strategy)
 
+# Custom evaluation metrics
+def calculate_semantic_similarity(text1: str, text2: str) -> float:
+    embeddings1 = sentence_model.encode([text1], convert_to_tensor=True)
+    embeddings2 = sentence_model.encode([text2], convert_to_tensor=True)
+    similarity = util.pytorch_cos_sim(embeddings1, embeddings2)
+    return float(similarity[0][0])
+
+def evaluate_response(question: str, answer: str, ground_truth: str, contexts: List[str]) -> Dict[str, float]:
+    # Answer similarity with ground truth
+    answer_similarity = calculate_semantic_similarity(answer, ground_truth)
+    
+    # Context relevance - average similarity between question and contexts
+    context_scores = [calculate_semantic_similarity(question, ctx) for ctx in contexts]
+    context_relevance = np.mean(context_scores)
+    
+    # Answer relevance - similarity between question and answer
+    answer_relevance = calculate_semantic_similarity(question, answer)
+    
+    return {
+        "answer_similarity": answer_similarity,
+        "context_relevance": context_relevance,
+        "answer_relevance": answer_relevance,
+        "average_score": np.mean([answer_similarity, context_relevance, answer_relevance])
+    }
+
 # Load and split PDF document
 def load_doc(list_file_path: List[str], splitting_strategy: str = "recursive"):
     loaders = [PyPDFLoader(x) for x in list_file_path]
@@ -83,17 +103,15 @@ def create_db(splits, db_choice: str = "faiss"):
     }
     return db_creators[db_choice](splits, embeddings)
 
-# Updated evaluation functions
 def load_evaluation_dataset():
-    # Load example dataset from RAGAS
     dataset = load_dataset("explodinggradients/fiqa", split="test")
     return dataset
 
-def prepare_ragas_dataset(qa_chain, dataset):
+def evaluate_rag_pipeline(qa_chain, dataset):
     # Sample a few examples for evaluation
     eval_samples = dataset.select(range(5))
     
-    ragas_dataset = []
+    results = []
     for sample in eval_samples:
         question = sample["question"]
         
@@ -103,40 +121,23 @@ def prepare_ragas_dataset(qa_chain, dataset):
             "chat_history": []
         })
         
-        ragas_dataset.append({
-            "question": question,
-            "answer": response["answer"],
-            "contexts": [doc.page_content for doc in response["source_documents"]],
-            "ground_truth": sample["answer"]
-        })
-    
-    return ragas_dataset
-
-def evaluate_rag_pipeline(qa_chain, dataset):
-    ragas_dataset = prepare_ragas_dataset(qa_chain, dataset)
-    
-    # Run RAGAS evaluation
-    results = evaluate(
-        ragas_dataset,
-        metrics=[
-            context_precision,
-            faithfulness,
-            answer_relevancy,
-            context_recall,
-            answer_correctness,
-            answer_similarity
-        ]
-    )
+        # Evaluate response
+        eval_result = evaluate_response(
+            question=question,
+            answer=response["answer"],
+            ground_truth=sample["answer"],
+            contexts=[doc.page_content for doc in response["source_documents"]]
+        )
+        
+        results.append(eval_result)
     
-    # Convert results to a dictionary
-    return {
-        "context_precision": float(results["context_precision"]),
-        "faithfulness": float(results["faithfulness"]),
-        "answer_relevancy": float(results["answer_relevancy"]),
-        "context_recall": float(results["context_recall"]),
-        "answer_correctness": float(results["answer_correctness"]),
-        "answer_similarity": float(results["answer_similarity"])
+    # Calculate average scores across all samples
+    avg_results = {
+        metric: float(np.mean([r[metric] for r in results]))
+        for metric in results[0].keys()
     }
+    
+    return avg_results
 
 # Initialize langchain LLM chain
 def initialize_llmchain(llm_model, temperature, max_tokens, top_k, vector_db, progress=gr.Progress()):
@@ -174,14 +175,12 @@ def initialize_llmchain(llm_model, temperature, max_tokens, top_k, vector_db, pr
     )
     return qa_chain
 
-# Initialize database with chunking strategy and vector DB choice
 def initialize_database(list_file_obj, splitting_strategy, db_choice, progress=gr.Progress()):
     list_file_path = [x.name for x in list_file_obj if x is not None]
     doc_splits = load_doc(list_file_path, splitting_strategy)
     vector_db = create_db(doc_splits, db_choice)
     return vector_db, f"Database created using {splitting_strategy} splitting and {db_choice} vector database!"
 
-# Formatting chat history
 def format_chat_history(message, chat_history):
     formatted_chat_history = []
     for user_message, bot_message in chat_history:
@@ -189,7 +188,6 @@ def format_chat_history(message, chat_history):
         formatted_chat_history.append(f"Assistant: {bot_message}")
     return formatted_chat_history
 
-# Conversation function
 def conversation(qa_chain, message, history):
     formatted_chat_history = format_chat_history(message, history)
     response = qa_chain.invoke({
@@ -230,7 +228,7 @@ def demo():
                 
                 with gr.Row():
                     splitting_strategy = gr.Radio(
-                        ["recursive", "fixed", "semantic"],
+                        ["recursive", "fixed", "token"],
                         label="Text Splitting Strategy",
                         value="recursive"
                     )