search improved

course_search/search_system/rag_system.py

@@ -1,19 +1,12 @@
-from langchain_community.document_loaders import DataFrameLoader
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.chains import RetrievalQA
-from langchain_community.llms import HuggingFaceHub
+from sentence_transformers import SentenceTransformer
+import numpy as np
 import pandas as pd
+import faiss
 import logging
 from typing import List, Dict
-import os
-from dotenv import load_dotenv
 from pathlib import Path
-import numpy as np
-import faiss
-from sentence_transformers import SentenceTransformer
 
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 class RAGSystem:
@@ -142,4 +135,41 @@ class RAGSystem:
         
         except Exception as e:
             logger.error(f"Error in search_courses: {str(e)}")
-            raise
+            raise
+
+    def calculate_text_similarity(self, text1: str, text2: str) -> float:
+        """
+        Calculate text similarity between two strings using word overlap
+        
+        Args:
+            text1 (str): First text string
+            text2 (str): Second text string
+            
+        Returns:
+            float: Similarity score between 0 and 1
+        """
+        try:
+            # Convert to lowercase and split into words
+            text1 = str(text1).lower()
+            text2 = str(text2).lower()
+            
+            words1 = set(text1.split())
+            words2 = set(text2.split())
+            
+            if not words1 or not words2:
+                return 0.0
+            
+            # Calculate Jaccard similarity
+            intersection = len(words1.intersection(words2))
+            union = len(words1.union(words2))
+            
+            if union == 0:
+                return 0.0
+            
+            similarity = intersection / union
+            
+            return similarity
+            
+        except Exception as e:
+            logger.error(f"Error calculating text similarity: {str(e)}")
+            return 0.0