Upload 3 files

README.md

@@ -1,13 +1,21 @@
----
-title: SmartSearchTool AnalyticsVidhya
-emoji: 🏆
-colorFrom: gray
-colorTo: pink
-sdk: streamlit
-sdk_version: 1.40.0
-app_file: app.py
-pinned: false
-short_description: A smart search tool to display relevant free courses
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Smart Search Tool for Analytics Vidhya Courses
+# Goal
+To create a smart search tool that enables users to find relevant free courses on Analytics Vidhya’s platform quickly.
+
+# Project Approach
+# Data Collection
+I began by scraping the free courses' titles and relevant metadata, such as course links and images, from Analytics Vidhya’s platform using BeautifulSoup.
+
+# Model Selection
+Originally, I used the Groq API for generating embeddings and conducting searches. However, I found the results less suitable, leading me to switch to a more refined solution using BERT (Bidirectional Encoder Representations from Transformers). I leveraged a pre-trained BERT model (bert-base-uncased from Hugging Face) for generating embeddings.
+
+# Relevance Matching
+To match user queries with relevant courses, I calculated cosine similarity between the user’s query embedding and the course title embeddings. This similarity score enables ranking courses based on relevance, ensuring the most suitable courses are shown first.
+
+# Interface
+The application uses both Streamlit and Shiny for flexible, user-friendly interfaces. These interfaces display course details dynamically, including title, image, link, and relevance score.Finally I can able to conclude that Shiny is more faster in retrieving the results and display those in more interactive way than StreamLit.
+
+# Deployment on Hugging Face Spaces
+I deployed the tool on Hugging Face Spaces, providing an accessible, visually appealing interface for public use, enhanced with custom CSS for style and responsiveness.
+
+BERT model : google-bert/bert-base-uncased

app_using_shiny.py

@@ -0,0 +1,205 @@
+from shiny import App, ui, render
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import torch
+from transformers import BertTokenizer, BertModel
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+# Step 1: Scrape the free courses from Analytics Vidhya
+url = "https://courses.analyticsvidhya.com/pages/all-free-courses"
+response = requests.get(url)
+soup = BeautifulSoup(response.content, 'html.parser')
+
+courses = []
+
+# Extracting course title, image, and course link
+for course_card in soup.find_all('header', class_='course-card__img-container'):
+    img_tag = course_card.find('img', class_='course-card__img')
+    
+    if img_tag:
+        title = img_tag.get('alt')
+        image_url = img_tag.get('src')
+        
+        link_tag = course_card.find_previous('a')
+        if link_tag:
+            course_link = link_tag.get('href')
+            if not course_link.startswith('http'):
+                course_link = 'https://courses.analyticsvidhya.com' + course_link
+
+            courses.append({
+                'title': title,
+                'image_url': image_url,
+                'course_link': course_link
+            })
+
+# Step 2: Create DataFrame
+df = pd.DataFrame(courses)
+
+# Load pre-trained BERT model and tokenizer
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+model = BertModel.from_pretrained('bert-base-uncased')
+
+# Function to generate embeddings using BERT
+def get_bert_embedding(text):
+    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)
+    with torch.no_grad():
+        outputs = model(**inputs)
+    return outputs.last_hidden_state.mean(dim=1).numpy()
+
+# Create embeddings for course titles
+df['embedding'] = df['title'].apply(lambda x: get_bert_embedding(x))
+
+# Function to perform search using BERT-based similarity
+def search_courses(query):
+    query_embedding = get_bert_embedding(query)
+    course_embeddings = np.vstack(df['embedding'].values)
+    
+    # Compute cosine similarity between query embedding and course embeddings
+    similarities = cosine_similarity(query_embedding, course_embeddings).flatten()
+    
+    # Add the similarity scores to the DataFrame
+    df['score'] = similarities
+    
+    # Sort by similarity score in descending order and return top results
+    top_results = df.sort_values(by='score', ascending=False).head(10)
+    return top_results[['title', 'image_url', 'course_link', 'score']].to_dict(orient='records')
+
+# Shiny UI and Server
+app_ui = ui.page_fluid(
+    ui.tags.style(
+        """
+        @import url('https://fonts.googleapis.com/css2?family=Poppins:wght@300;500;700&display=swap');
+
+        body {
+            font-family: 'Poppins', sans-serif;
+            background-color: #f4f6f9;
+        }
+
+        .container {
+            padding: 20px;
+        }
+        
+        h2 {
+            color: #ff6f61;
+            font-weight: 700;
+            text-align: center;
+        }
+
+        .result-container {
+            display: flex;
+            flex-wrap: wrap;
+            gap: 20px;
+            justify-content: center;
+        }
+        
+        .course-card {
+            background-color: #ffffff;
+            border-radius: 12px;
+            box-shadow: 0 4px 10px rgba(0, 0, 0, 0.15);
+            overflow: hidden;
+            width: calc(50% - 10px);
+            transition: transform 0.3s, box-shadow 0.3s;
+        }
+        
+        .course-card:hover {
+            transform: scale(1.05);
+            box-shadow: 0 6px 20px rgba(0, 0, 0, 0.2);
+        }
+
+        .course-image {
+            width: 100%;
+            height: 180px;
+            object-fit: cover;
+            border-top-left-radius: 12px;
+            border-top-right-radius: 12px;
+        }
+
+        .course-info {
+            padding: 15px;
+        }
+        
+        .course-info h3 {
+            font-size: 20px;
+            color: #333;
+            margin-top: 0;
+        }
+
+        .course-info p {
+            color: #666;
+            font-size: 16px;
+            margin-bottom: 10px;
+        }
+
+        .course-link {
+            background-color: #ff6f61;
+            color: white;
+            padding: 8px 12px;
+            text-decoration: none;
+            border-radius: 6px;
+            font-size: 15px;
+            display: inline-block;
+            margin-top: 10px;
+            transition: background-color 0.2s;
+        }
+
+        .course-link:hover {
+            background-color: #e85a50;
+        }
+
+        .no-results {
+            text-align: center;
+            color: #888;
+            font-style: italic;
+        }
+        """
+    ),
+    ui.h2("Analytics Vidhya Smart Course Search"),
+    ui.input_text("query", "Enter your search query", placeholder="e.g., machine learning, data science, python"),
+    ui.output_text("search_info"),
+    ui.output_ui("results")
+)
+
+def server(input, output, session):
+    @output
+    @render.ui
+    def results():
+        if not input.query():
+            return ui.p("Enter a search query to get started!", class_="no-results")
+        
+        # Perform the search
+        query = input.query()
+        results = search_courses(query)
+        
+        if results:
+            result_ui = []
+            for item in results:
+                course_title = item['title']
+                course_image = item['image_url']
+                course_link = item['course_link']
+                relevance_score = round(item['score'] * 100, 2)
+                
+                # Create course card UI
+                result_ui.append(
+                    ui.div(
+                        ui.img(src=course_image, class_="course-image"),
+                        ui.div(
+                            ui.h3(course_title),
+                            ui.p(f"Relevance: {relevance_score}%"),
+                            ui.a("View Course", href=course_link, target="_blank", class_="course-link"),
+                            class_="course-info"
+                        ),
+                        class_="course-card"
+                    )
+                )
+            return ui.div(*result_ui, class_="result-container")
+        else:
+            return ui.p("No results found.", class_="no-results")
+    
+    @output
+    @render.text
+    def search_info():
+        return f"Results for '{input.query()}'" if input.query() else "Search for courses by typing a query above."
+
+app = App(app_ui, server)

app_using_streamlit.py

@@ -0,0 +1,89 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import streamlit as st
+import torch
+from transformers import BertTokenizer, BertModel
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+# Step 1: Scrape the free courses from Analytics Vidhya
+url = "https://courses.analyticsvidhya.com/pages/all-free-courses"
+response = requests.get(url)
+soup = BeautifulSoup(response.content, 'html.parser')
+
+courses = []
+
+# Extracting course title, image, and course link
+for course_card in soup.find_all('header', class_='course-card__img-container'):
+    img_tag = course_card.find('img', class_='course-card__img')
+    
+    if img_tag:
+        title = img_tag.get('alt')
+        image_url = img_tag.get('src')
+        
+        link_tag = course_card.find_previous('a')
+        if link_tag:
+            course_link = link_tag.get('href')
+            if not course_link.startswith('http'):
+                course_link = 'https://courses.analyticsvidhya.com' + course_link
+
+            courses.append({
+                'title': title,
+                'image_url': image_url,
+                'course_link': course_link
+            })
+
+# Step 2: Create DataFrame
+df = pd.DataFrame(courses)
+
+# Load pre-trained BERT model and tokenizer
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+model = BertModel.from_pretrained('bert-base-uncased')
+
+# Function to generate embeddings using BERT
+def get_bert_embedding(text):
+    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)
+    with torch.no_grad():
+        outputs = model(**inputs)
+    return outputs.last_hidden_state.mean(dim=1).numpy()
+
+# Create embeddings for course titles
+df['embedding'] = df['title'].apply(lambda x: get_bert_embedding(x))
+
+# Function to perform search using BERT-based similarity
+def search_courses(query):
+    query_embedding = get_bert_embedding(query)
+    course_embeddings = np.vstack(df['embedding'].values)
+    
+    # Compute cosine similarity between query embedding and course embeddings
+    similarities = cosine_similarity(query_embedding, course_embeddings).flatten()
+    
+    # Add the similarity scores to the DataFrame
+    df['score'] = similarities
+    
+    # Sort by similarity score in descending order and return top results
+    top_results = df.sort_values(by='score', ascending=False).head(10)
+    return top_results[['title', 'image_url', 'course_link', 'score']].to_dict(orient='records')
+
+# Streamlit Interface
+st.title("Analytics Vidhya Smart Course Search")
+st.write("Find the most relevant courses from Analytics Vidhya based on your query.")
+
+query = st.text_input("Enter your search query", placeholder="e.g., machine learning, data science, python")
+
+if query:
+    results = search_courses(query)
+    if results:
+        for item in results:
+            course_title = item['title']
+            course_image = item['image_url']
+            course_link = item['course_link']
+            relevance_score = round(item['score'] * 100, 2)
+            
+            st.image(course_image, width=300)
+            st.markdown(f"### [{course_title}]({course_link})")
+            st.write(f"Relevance: {relevance_score}%")
+            st.markdown("---")
+    else:
+        st.write("No results found.")