commit

.gitignore

@@ -0,0 +1,3 @@
+.env
+
+

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

.gradio/flagged/dataset1.csv

@@ -0,0 +1,43 @@
+What would you like to learn?,Number of Results,output,timestamp
+"LLM
+",5,"# Search Results
+
+### 1. Getting Started with Large Language Models
+**Description:**
+Getting Started With LLMs
+
+**Course Link:** [View Course](https://courses.analyticsvidhya.com/courses/getting-started-with-llms)
+
+---
+
+### 2. Framework to Choose the Right LLM for your Business
+**Description:**
+This course will guide you through the process of selecting the most suitable Large Language Model (LLM) for various business needs. By examining factors such as accuracy, cost, scalability, and integration, you will understand how different LLMs perform in specific scenarios, from customer support to healthcare and strategy development. The course emphasizes practical decision-making with real-world case studies, helping businesses navigate the rapidly evolving LLM landscape effectively.
+
+**Course Link:** [View Course](https://courses.analyticsvidhya.com/courses/choosing-the-right-LLM-for-your-business)
+
+---
+
+### 3. Building LLM Applications using Prompt Engineering - Free Course
+**Description:**
+Professionals: Individuals looking to deepen their knowledge and apply advanced LLM and prompt engineering techniques to solve complex problems across various domains.
+Aspiring Students: Individuals looking to deepen their knowledge and apply advanced LLM and prompt engineering techniques to solve complex problems across various domains.
+
+**Course Link:** [View Course](https://courses.analyticsvidhya.com/courses/building-llm-applications-using-prompt-engineering-free)
+
+---
+
+### 4. Getting Started with Large Language Models
+**Description:**
+Who Should Enroll: Professionals: Individuals looking to expand their skill set and leverage LLMs across different industries. Aspiring Students: For those setting out on their journey to master language data analysis and leave a mark in the tech world.
+
+**Course Link:** [View Course](https://courses.analyticsvidhya.com/courses/getting-started-with-llms)
+
+---
+
+### 5. Getting Started with Large Language Models
+**Description:**
+This course will help you gain a comprehensive understanding of Large Language Models (LLMs) and develop advanced natural language processing (NLP) applications using the PyTorch framework. With a carefully curated list of resources and exercises, this course is your guide to becoming an expert in LLMs. Master the techniques to build and fine-tune LLMs, and generate human-like text.
+
+**Course Link:** [View Course](https://courses.analyticsvidhya.com/courses/getting-started-with-llms)
+

README.md

@@ -1,11 +1,11 @@
 ---
-title: Course Search Av
+title: Course Search Analytics Vidya
 emoji: 🚀
 colorFrom: purple
 colorTo: pink
 sdk: gradio
 sdk_version: 5.8.0
-app_file: app.py
+app_file: course_search/app/run.py
 pinned: false
 license: other
 short_description: Course search assignment for Analytics Vidhya

course_search.txt

@@ -0,0 +1,27 @@
+Course Search System: My Implementation Journey
+
+Data Gathering
+I started by collecting course data from various sources like university websites and online platforms. I used web scraping techniques to extract information like course titles, descriptions, prerequisites, and more. I also integrated APIs from some institutions to directly access their course catalogs. Once I had the data, I cleaned and standardized it to ensure consistency.
+
+Choosing the Right Tools
+For processing the text data, I selected a powerful language model like BERT to understand the semantic meaning of course descriptions. This model converted the text into numerical representations (vectors) that computers could easily process.
+
+To efficiently store and search these vectors, I used a vector database called FAISS. It's designed for handling large datasets of vectors and quickly finding the most similar ones.
+
+Building the System
+I designed my system to be flexible and scalable. Here's a breakdown of its key components:
+
+Data Ingestion: This part collects and prepares the course data.
+Embedding: The language model processes the course descriptions and creates vectors.
+Vector Database: This stores the vectors for efficient searching.
+Search API: This allows users to query the system and get relevant results.
+User Interface: This is the front-end where users can interact with the system.
+I deployed the system on a cloud platform to ensure it's reliable and can handle increasing user demand. Each component runs in its own container, making it easy to manage and update.
+
+Challenges and Solutions
+
+Data Quality: Ensuring data consistency and accuracy was a big challenge. I addressed this by carefully cleaning and standardizing the data.
+Model Performance: Choosing the right language model was crucial. I experimented with different models and fine-tuned them to get the best results.
+Scalability: Handling a large number of courses required a scalable vector database. FAISS was a great choice for this, and I configured it to handle the load.
+User Experience: I focused on making the system user-friendly. I conducted user tests and made improvements to the interface and search algorithm.
+Overall, this project was a great learning experience. I'm proud of what I've accomplished and excited to see how it can help students find the right courses.

course_search/app/gradio_app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+import pandas as pd
+from pathlib import Path
+import logging
+from course_search.search_system.rag_system import RAGSystem
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class CourseSearchApp:
+    def __init__(self):
+        """Initialize the search application"""
+        try:
+            self.load_components()
+        except Exception as e:
+            logger.error(f"Initialization error: {str(e)}")
+            raise
+
+    def load_components(self):
+        """Initialize RAG system and load data"""
+        try:
+            # Construct path to data file
+            data_path = Path(__file__).parent.parent.parent / 'data' / 'courses_with_embeddings.pkl'
+            
+            if not data_path.exists():
+                raise FileNotFoundError(f"Data file not found at: {data_path}")
+            
+            # Load saved course data
+            df = pd.read_pickle(str(data_path))
+            logger.info(f"Loaded {len(df)} courses from {data_path}")
+            
+            # Initialize RAG system
+            self.rag_system = RAGSystem()
+            self.rag_system.load_and_process_data(df)
+            logger.info("Components loaded successfully")
+            
+        except Exception as e:
+            logger.error(f"Error loading components: {str(e)}")
+            raise
+
+    def search_courses(self, query: str, num_results: int) -> str:
+        """
+        Search for courses and format results for Gradio
+        """
+        try:
+            results = self.rag_system.search_courses(query, top_k=num_results)
+            
+            # Format results for display
+            markdown_output = "# Search Results\n\n"
+            for i, result in enumerate(results['results'], 1):
+                markdown_output += f"### {i}. {result['title']}\n"
+                markdown_output += f"**Description:**\n{result['description']}\n\n"
+                markdown_output += f"**Course Link:** [View Course]({result['url']})\n\n"
+                markdown_output += "---\n\n"
+                        
+            return markdown_output
+            
+        except Exception as e:
+            error_msg = f"Error during search: {str(e)}"
+            logger.error(error_msg)
+            return f"# Error\n\n{error_msg}"
+
+def create_gradio_interface():
+    """Create and configure Gradio interface"""
+    try:
+        app = CourseSearchApp()
+        
+        # Define the interface
+        iface = gr.Interface(
+            fn=app.search_courses,
+            inputs=[
+                gr.Textbox(
+                    label="What would you like to learn?",
+                    placeholder="e.g., machine learning for beginners, data visualization, python basics",
+                    lines=2
+                ),
+                gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    value=5,
+                    step=1,
+                    label="Number of Results"
+                )
+            ],
+            outputs=gr.Markdown(),
+            title="Analytics Vidhya Course Search",
+            description="""
+            Search through Analytics Vidhya's free courses using natural language!
+            Get personalized course recommendations and AI-generated responses to your queries.
+            """,
+            theme=gr.themes.Soft()
+        )
+        
+        return iface
+        
+    except Exception as e:
+        logger.error(f"Error creating Gradio interface: {str(e)}")
+        raise
+
+def main():
+    """Main function to run the Gradio app"""
+    try:
+        iface = create_gradio_interface()
+        iface.launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            share=True
+        )
+    except Exception as e:
+        logger.error(f"Error launching Gradio app: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    main()

course_search/app/main.py

@@ -0,0 +1,26 @@
+import sys
+import os
+# Add the project root directory to Python path
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(PROJECT_ROOT)
+
+from course_search.search_system.data_pipeline import DataPipeline
+import logging
+import argparse
+
+def main():
+    parser = argparse.ArgumentParser(description='Run the course scraping and embedding pipeline')
+    parser.add_argument('--output', type=str, default='data/courses_with_embeddings.pkl',
+                      help='Path to save the processed data')
+    args = parser.parse_args()
+
+    # Initialize and run pipeline
+    pipeline = DataPipeline()
+    df = pipeline.run_pipeline(save_path=args.output)
+    
+    print(f"\nProcessed {len(df)} courses")
+    print(f"Data saved to {args.output}")
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

course_search/app/run.py

@@ -0,0 +1,65 @@
+import sys
+import os
+from pathlib import Path
+import subprocess
+import logging
+
+project_root = Path(__file__).parent.parent.parent
+
+if str(project_root) not in sys.path:
+    sys.path.append(str(project_root))
+        
+from course_search.search_system.data_pipeline import DataPipeline
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def setup_paths():
+    """Setup necessary paths and directories"""
+    project_root = Path(__file__).parent.parent.parent
+    
+    if str(project_root) not in sys.path:
+        sys.path.append(str(project_root))
+    
+    data_dir = project_root / 'data'
+    data_dir.mkdir(exist_ok=True)
+    
+    return project_root, data_dir
+
+def main():
+    try:
+        # Setup paths
+        project_root, data_dir = setup_paths()
+        
+        # Run data pipeline
+        logger.info("Running data pipeline...")
+        pipeline = DataPipeline()
+        pipeline.run_pipeline(save_path=str(data_dir / 'courses_with_embeddings.pkl'))
+        
+        # Run Gradio app
+        logger.info("Starting Gradio app...")
+        gradio_path = Path(__file__).parent / 'gradio_app.py'
+        
+        if not gradio_path.exists():
+            raise FileNotFoundError(f"Gradio app not found at: {gradio_path}")
+            
+        # Change to project root directory before running
+        os.chdir(str(project_root))
+        
+        # Run Gradio with proper Python path
+        env = os.environ.copy()
+        env['PYTHONPATH'] = str(project_root)
+        
+        subprocess.run(
+            ['python', str(gradio_path)],
+            env=env,
+            check=True
+        )
+        
+    except Exception as e:
+        logger.error(f"Error running application: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    main()

course_search/scraper/course_scraper.py

@@ -0,0 +1,98 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+from typing import List, Dict
+import logging
+from urllib.parse import urljoin
+import time
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class CourseScraper:
+    def __init__(self):
+        self.base_url = "https://courses.analyticsvidhya.com/pages/all-free-courses"
+        self.headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+        }
+        
+    def get_course_links(self) -> List[str]:
+        try:
+            logger.info(f"Fetching course links from {self.base_url}")
+            response = requests.get(self.base_url, headers=self.headers)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.content, 'html.parser')
+            course_links = []
+
+            for a_tag in soup.find_all('a', class_='course-card'):
+                href = a_tag.get('href')
+                if href:
+                    full_url = urljoin(self.base_url, href)
+                    course_links.append(full_url)
+                    logger.debug(f"Found course link: {full_url}")
+
+            logger.info(f"Found {len(course_links)} course links")
+            return course_links
+        
+        except requests.RequestException as e:
+            logger.error(f"Error fetching course links: {str(e)}")
+            return []
+        
+    
+    def extract_course_info(self, url: str) -> Dict:
+        try:
+            logger.info(f"Extracting course info from {url}")
+            response = requests.get(url, headers=self.headers)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            course_info = {
+                'url': url,
+                'title': '',
+                'description': '',
+                'curriculum': ''
+            }
+
+            # Extract title
+            title_elem = soup.find('h1', class_='section__heading')
+            if title_elem:
+                course_info['title'] = title_elem.text.strip()
+
+            # Extract description
+            desc_elem = soup.find('div', class_='rich-text__container')
+            if desc_elem:
+                course_info['description'] = desc_elem.text.strip()
+
+            # Extract curriculum
+            curr_elem = soup.find('div', class_='course-curriculum__container')
+            if curr_elem:
+                course_info['curriculum'] = curr_elem.text.strip()
+
+            return course_info
+
+        except requests.RequestException as e:
+            logger.error(f"Error extracting course info from {url}: {str(e)}")
+            return None
+
+    
+    def scrape_all_courses(self) -> pd.DataFrame:
+        all_courses = []
+        course_links = self.get_course_links()
+        
+        for link in course_links:
+            try:
+                course_info = self.extract_course_info(link)
+                if course_info:
+                    all_courses.append(course_info)
+                # Add a small delay to be respectful to the server
+                time.sleep(1)
+            except Exception as e:
+                logger.error(f"Error processing {link}: {str(e)}")
+                continue
+        
+        df = pd.DataFrame(all_courses)
+        logger.info(f"Successfully scraped {len(df)} courses")
+        return df

course_search/search_system/data_pipeline.py

@@ -0,0 +1,47 @@
+import pandas as pd
+from typing import Optional
+from course_search.scraper.course_scraper import CourseScraper
+from course_search.search_system.embeddings import EmbeddingGenerator
+from course_search.search_system.vector_store import FAISSManager
+import logging
+
+logger = logging.getLogger(__name__)
+
+class DataPipeline:
+    def __init__(self):
+        self.scraper = CourseScraper()
+        self.embedding_generator = EmbeddingGenerator()
+        self.vector_store = FAISSManager()
+        
+    def run_pipeline(self, save_path: Optional[str] = None) -> pd.DataFrame:
+        """
+        Run the complete data pipeline: scraping, embedding generation, and vector storage
+        """
+        try:
+            # Step 1: Scrape courses
+            logger.info("Starting course scraping...")
+            df = self.scraper.scrape_all_courses()
+            logger.info(f"Scraped {len(df)} courses successfully")
+            
+            # Step 2: Generate embeddings
+            logger.info("Generating embeddings...")
+            df = self.embedding_generator.add_embeddings_to_df(
+                df, 
+                text_column='description'
+            )
+            logger.info("Embeddings generated successfully")
+            
+            # Step 3: Upload to FAISS
+            logger.info("Uploading to FAISS...")
+            self.vector_store.upsert_courses(df)
+            
+            # Step 4: Save data if path provided
+            if save_path:
+                logger.info(f"Saving data to {save_path}")
+                df.to_pickle(save_path)
+                
+            return df
+            
+        except Exception as e:
+            logger.error(f"Error in pipeline: {str(e)}")
+            raise

course_search/search_system/embeddings.py

@@ -0,0 +1,123 @@
+import numpy as np
+import pandas as pd
+from typing import List
+import logging
+from sentence_transformers import SentenceTransformer
+import os
+from pathlib import Path
+import pickle
+
+os.environ['TOKENIZERS_PARALLELISM'] = 'false'  # To avoid warnings
+
+logger = logging.getLogger(__name__)
+
+class EmbeddingGenerator:
+    def __init__(self, model_name: str = 'all-MiniLM-L6-v2', cache_dir: str = None):
+        try:
+            self.model_name = model_name
+            self.model = SentenceTransformer(model_name)
+            
+            # Setup cache directory
+            self.cache_dir = Path(cache_dir) if cache_dir else Path('data/embedding_cache')
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Cache file for embeddings
+            self.cache_file = self.cache_dir / f"embeddings_cache_{model_name.replace('/', '_')}.pkl"
+            
+            # Load existing cache if available
+            self.embedding_cache = self._load_cache()
+            
+            logger.info(f"Successfully loaded model: {model_name}")
+        except Exception as e:
+            logger.error(f"Error loading model: {str(e)}")
+            raise
+
+    def _load_cache(self) -> dict:
+        """Load embedding cache from file if it exists"""
+        try:
+            if self.cache_file.exists():
+                with open(self.cache_file, 'rb') as f:
+                    cache = pickle.load(f)
+                logger.info(f"Loaded {len(cache)} cached embeddings")
+                return cache
+            return {}
+        except Exception as e:
+            logger.warning(f"Error loading cache, starting fresh: {str(e)}")
+            return {}
+
+    def _save_cache(self):
+        """Save embedding cache to file"""
+        try:
+            with open(self.cache_file, 'wb') as f:
+                pickle.dump(self.embedding_cache, f)
+            logger.info(f"Saved {len(self.embedding_cache)} embeddings to cache")
+        except Exception as e:
+            logger.error(f"Error saving cache: {str(e)}")
+
+    def generate_embeddings(self, texts: pd.Series) -> np.ndarray:
+        try:
+            # Convert texts to list
+            text_list = texts.tolist()
+            
+            # Initialize array to store embeddings
+            all_embeddings = []
+            texts_to_embed = []
+            indices_to_embed = []
+
+            # Check cache for existing embeddings
+            for i, text in enumerate(text_list):
+                text_hash = hash(text)
+                if text_hash in self.embedding_cache:
+                    all_embeddings.append(self.embedding_cache[text_hash])
+                else:
+                    texts_to_embed.append(text)
+                    indices_to_embed.append(i)
+
+            # Generate embeddings only for new texts
+            if texts_to_embed:
+                logger.info(f"Generating embeddings for {len(texts_to_embed)} new texts")
+                new_embeddings = self.model.encode(
+                    texts_to_embed,
+                    show_progress_bar=True,
+                    convert_to_numpy=True
+                )
+
+                # Cache new embeddings
+                for text, embedding in zip(texts_to_embed, new_embeddings):
+                    text_hash = hash(text)
+                    self.embedding_cache[text_hash] = embedding
+                
+                # Save updated cache
+                self._save_cache()
+
+                # Insert new embeddings in correct positions
+                for idx, embedding in zip(indices_to_embed, new_embeddings):
+                    all_embeddings.insert(idx, embedding)
+            else:
+                logger.info("All embeddings found in cache")
+
+            return np.array(all_embeddings)
+            
+        except Exception as e:
+            logger.error(f"Error generating embeddings: {str(e)}")
+            raise
+
+    def add_embeddings_to_df(self, df: pd.DataFrame, text_column: str = 'description') -> pd.DataFrame:
+        try:
+            embeddings = self.generate_embeddings(df[text_column])
+            df['embeddings'] = list(embeddings)
+            return df
+        except Exception as e:
+            logger.error(f"Error adding embeddings to DataFrame: {str(e)}")
+            raise
+
+    def clear_cache(self):
+        """Clear the embedding cache"""
+        try:
+            self.embedding_cache = {}
+            if self.cache_file.exists():
+                self.cache_file.unlink()
+            logger.info("Embedding cache cleared")
+        except Exception as e:
+            logger.error(f"Error clearing cache: {str(e)}")
+            raise

course_search/search_system/rag_system.py

@@ -0,0 +1,110 @@
+from langchain_community.document_loaders import DataFrameLoader
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.chains import RetrievalQA
+from langchain_community.llms import HuggingFaceHub
+import pandas as pd
+import logging
+from typing import List, Dict
+import os
+from dotenv import load_dotenv
+
+logger = logging.getLogger(__name__)
+
+class RAGSystem:
+    def __init__(self):
+        """Initialize the RAG system with LangChain components"""
+        load_dotenv()
+        
+        # Initialize embedding model
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name="sentence-transformers/all-MiniLM-L6-v2"
+        )
+        
+        # Initialize text splitter for chunking
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=500,
+            chunk_overlap=50
+        )
+        
+        self.vector_store = None
+        self.qa_chain = None
+
+    def load_and_process_data(self, df: pd.DataFrame) -> None:
+        """
+        Load course data and create vector store
+        """
+        try:
+            # Prepare documents from DataFrame
+            loader = DataFrameLoader(
+                data_frame=df,
+                page_content_column="description"
+            )
+            documents = loader.load()
+            for doc, row in zip(documents, df.itertuples()):
+                doc.metadata = {
+                    "title": row.title,
+                    "url": row.url,
+                    # Add other metadata fields as needed
+                }
+            
+            # Split documents into chunks
+            splits = self.text_splitter.split_documents(documents)
+            
+            # Create vector store
+            self.vector_store = FAISS.from_documents(
+                splits,
+                self.embeddings
+            )
+            
+            # Initialize QA chain
+            llm = HuggingFaceHub(
+                repo_id="google/flan-t5-small",
+                huggingfacehub_api_token=os.getenv('HUGGINGFACE_API_TOKEN')
+            )
+            
+            self.qa_chain = RetrievalQA.from_chain_type(
+                llm=llm,
+                chain_type="stuff",
+                retriever=self.vector_store.as_retriever()
+            )
+            
+            logger.info("RAG system initialized successfully")
+            
+        except Exception as e:
+            logger.error(f"Error initializing RAG system: {str(e)}")
+            raise
+
+    def search_courses(self, query: str, top_k: int = 5) -> List[Dict]:
+        """
+        Search for courses using RAG
+        """
+        try:
+            if not self.vector_store:
+                raise ValueError("Vector store not initialized. Please load data first.")
+            
+            # Get relevant documents
+            docs = self.vector_store.similarity_search(query, k=top_k)
+            
+            # Generate answer using QA chain
+            answer = self.qa_chain.run(query)
+            
+            # Format results
+            results = []
+            for doc in docs:
+                results.append({
+                    'title': doc.metadata['title'],
+                    'description': doc.page_content,
+                    'url': doc.metadata['url'],
+                    'score': doc.metadata.get('score', 1.0)
+                })
+            
+            return {
+                'results': results,
+                'answer': answer
+            }
+            
+        except Exception as e:
+            logger.error(f"Error in course search: {str(e)}")
+            raise

course_search/search_system/vector_store.py

@@ -0,0 +1,65 @@
+import numpy as np
+import pandas as pd
+from typing import List, Dict
+import faiss
+import logging
+
+logger = logging.getLogger(__name__)
+
+class FAISSManager:
+    def __init__(self, dimension: int = 384):
+        """
+        Initialize FAISS index with error handling
+        """
+        try:
+            self.dimension = dimension
+            self.index = faiss.IndexFlatL2(dimension)
+            self.metadata = []
+            logger.info("FAISS index initialized successfully")
+        except Exception as e:
+            logger.error(f"Error initializing FAISS: {str(e)}")
+            raise RuntimeError(f"Failed to initialize FAISS: {str(e)}")
+
+    def upsert_courses(self, df: pd.DataFrame) -> None:
+        """
+        Add course embeddings to the FAISS index with error handling
+        """
+        try:
+            # Convert embeddings to numpy array
+            vectors = np.vstack([
+                emb.astype('float32') for emb in df['embeddings'].values
+            ])
+            
+            # Add vectors to index
+            self.index.add(vectors)
+            self.metadata.extend(df[['title', 'description', 'url']].to_dict('records'))
+            logger.info(f"Added {len(vectors)} vectors to FAISS index")
+            
+        except Exception as e:
+            logger.error(f"Error adding vectors to FAISS: {str(e)}")
+            raise RuntimeError(f"Failed to add vectors to FAISS: {str(e)}")
+
+    def search_courses(self, query_embedding: np.ndarray, top_k: int = 5) -> List[Dict]:
+        """
+        Search for similar courses using query embedding with error handling
+        """
+        try:
+            # Ensure query embedding is in correct format
+            query_embedding = query_embedding.astype('float32').reshape(1, -1)
+            
+            # Perform search
+            distances, indices = self.index.search(query_embedding, top_k)
+            
+            results = []
+            for i, idx in enumerate(indices[0]):
+                if idx == -1:
+                    continue
+                result = self.metadata[idx].copy()
+                result['score'] = float(distances[0][i])  # Convert to float for JSON serialization
+                results.append(result)
+            
+            return results
+            
+        except Exception as e:
+            logger.error(f"Error searching FAISS index: {str(e)}")
+            raise RuntimeError(f"Failed to search FAISS index: {str(e)}")

data/courses_with_embeddings.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c55c9ffb5774c19e7443e3f1c5b8a8c0add246c1516f05f80d29aa0433972f61
+size 139388

data/embedding_cache/embeddings_cache_all-MiniLM-L6-v2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5444508e05b63d662919fc358bfed9dea64094858d01a6df9b291a56525ea13d
+size 123513

instructions.txt

@@ -0,0 +1,49 @@
+This assignment involves building a **smart search tool** for Analytics Vidhya's free courses. Here's a breakdown of how you can approach it:
+
+### Steps to Complete the Assignment
+
+#### 1. **Data Collection**
+   - **Explore the Free Courses**: Visit the [Analytics Vidhya Free Courses](https://courses.analyticsvidhya.com/collections/courses) and collect course-related data such as:
+     - Course Titles
+     - Descriptions
+     - Curriculum
+   - **Tools for Data Collection**: Use Python libraries like `BeautifulSoup` (for web scraping) or APIs (if available) to extract the data.
+
+#### 2. **Smart Search System**
+   - **Frameworks**: Use **LangChain (0.3.x)** or **LlamaIndex (0.12.x)** for implementing a Retrieval-Augmented Generation (RAG) system.
+   - **Steps**:
+     1. **Generate Embeddings**:
+        - Choose a pre-trained embedding model like OpenAI's `text-embedding-ada-002` or Sentence Transformers.
+        - Convert the collected course data into embeddings.
+     2. **Vector Database**:
+        - Use a vector database such as Pinecone, Weaviate, or FAISS to store and query embeddings.
+     3. **Search Implementation**:
+        - Implement a natural language search interface where user queries are matched against the stored embeddings.
+        - Return the most relevant courses.
+
+#### 3. **Deployment on Huggingface Spaces**
+   - **Frameworks**: Use `Gradio` or `Streamlit` for creating an interactive UI.
+   - **Deployment Steps**:
+     1. Set up a Huggingface Spaces account.
+     2. Integrate your search tool into the interface.
+     3. Deploy the app and obtain a public URL.
+
+#### 4. **Document Your Approach**
+   - Write a report explaining:
+     - Data collection process.
+     - Embedding model and vector database choices.
+     - System architecture.
+     - Challenges faced and how you addressed them.
+
+---
+
+### Required Skills and Tools
+- **Web Scraping**: `requests`, `BeautifulSoup`
+- **Embedding Models**: `sentence-transformers`, OpenAI API
+- **Vector Database**: Pinecone, Weaviate, or FAISS
+- **Deployment**: `Gradio`, `Streamlit`, Huggingface Spaces
+- **Programming Language**: Python
+
+---
+
+If you need guidance on specific steps, feel free to ask!

tests/test_complete_pipeline.py

@@ -0,0 +1,47 @@
+import logging
+from course_search.search_system.embeddings import EmbeddingGenerator
+from course_search.search_system.vector_store import FAISSManager
+import pandas as pd
+import numpy as np
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def test_pipeline():
+    try:
+        # Test data
+        test_data = pd.DataFrame({
+            'title': ['Test Course 1', 'Test Course 2'],
+            'description': ['This is a test course about Python', 'This is a test course about ML'],
+            'url': ['http://test1.com', 'http://test2.com']
+        })
+
+        # Test embedding generation
+        logger.info("Testing embedding generation...")
+        embedding_gen = EmbeddingGenerator()
+        test_data = embedding_gen.add_embeddings_to_df(test_data)
+        assert 'embeddings' in test_data.columns
+        logger.info("Embedding generation successful!")
+
+        # Test FAISS storage
+        logger.info("Testing FAISS storage...")
+        vector_store = FAISSManager()
+        vector_store.upsert_courses(test_data)
+        logger.info("FAISS storage successful!")
+
+        # Test search
+        logger.info("Testing search...")
+        query = "Python programming"
+        query_embedding = embedding_gen.generate_embeddings(pd.Series([query]))[0]
+        results = vector_store.search_courses(query_embedding)
+        assert len(results) > 0
+        logger.info("Search successful!")
+
+        return True
+
+    except Exception as e:
+        logger.error(f"Pipeline test failed: {str(e)}")
+        return False
+
+if __name__ == "__main__":
+    test_pipeline()

tests/test_integration.py

@@ -0,0 +1,19 @@
+from course_search.search_system.data_pipeline import DataPipeline
+import logging
+
+def test_pipeline():
+    pipeline = DataPipeline()
+    
+    # Run pipeline and save results
+    df = pipeline.run_pipeline(save_path='data/courses_with_embeddings.pkl')
+    
+    # Print results
+    print("\nDataFrame Info:")
+    print(df.info())
+    print("\nSample Course:")
+    print(df.iloc[0][['title', 'description']].to_dict())
+    print("\nSample Embedding Shape:", df.iloc[0]['embeddings'].shape)
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    test_pipeline()

tests/test_scraper.py

@@ -0,0 +1,33 @@
+import logging
+from course_search.scraper.course_scraper import CourseScraper
+
+def test_scraper():
+    scraper = CourseScraper()
+    
+    # Test 1: Get course links
+    print("\nTesting get_course_links()...")
+    links = scraper.get_course_links()
+    print(f"Found {len(links)} links")
+    print("Sample links:")
+    for link in links[:3]:
+        print(f"- {link}")
+
+    # Test 2: Extract course info
+    if links:
+        print("\nTesting extract_course_info()...")
+        sample_course = scraper.extract_course_info(links[0])
+        print("Sample course info:")
+        for key, value in sample_course.items():
+            print(f"{key}: {value[:100]}...")
+
+    # Test 3: Scrape all courses
+    print("\nTesting scrape_all_courses()...")
+    df = scraper.scrape_all_courses()
+    print("\nDataFrame Info:")
+    print(df.info())
+    print("\nFirst few rows:")
+    print(df.head())
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    test_scraper()

tests/test_vector_store.py

@@ -0,0 +1,49 @@
+import numpy as np
+import faiss
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def test_faiss():
+    try:
+        # Create a small test index
+        dimension = 64
+        nb = 100
+        
+        # Generate random data
+        xb = np.random.random((nb, dimension)).astype('float32')
+        
+        # Create index
+        index = faiss.IndexFlatL2(dimension)
+        
+        # Add vectors
+        index.add(xb)
+        
+        # Test search
+        k = 5
+        xq = np.random.random((1, dimension)).astype('float32')
+        D, I = index.search(xq, k)
+        
+        logger.info("FAISS test successful!")
+        logger.info(f"Found {k} nearest neighbors")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"FAISS test failed: {str(e)}")
+        return False
+    
+import torch
+
+def test_torch():
+    try:
+        x = torch.rand(5, 3)
+        print("PyTorch is working correctly. Tensor:", x)
+    except Exception as e:
+        print("Error with PyTorch:", e)
+
+
+if __name__ == "__main__":
+    test_faiss()
+    test_torch()