import pandas as pd from sentence_transformers import SentenceTransformer, util from transformers import pipeline import torch import gradio as gr import os # Load the dataset csv_file_path = os.path.join(os.getcwd(), 'Analytics_Vidhya_Free_Course_data.csv') df = pd.read_csv(csv_file_path, encoding='Windows-1252') df.fillna('', inplace=True) # Load the pre-trained model for embeddings model = SentenceTransformer('multi-qa-mpnet-base-dot-v1') # Combine title and description to create a full text for each course df['full_text'] = df.iloc[:, 0] + " " + df.iloc[:, 1] + " " + df['Instructor Name'] + " " + df['Rating'].astype(str) + " " + df['Category'] # Convert full course texts into embeddings course_embeddings = model.encode(df['full_text'].tolist(), convert_to_tensor=True) # Load a model for text generation (e.g., BART) generator = pipeline('text2text-generation', model='facebook/bart-large-cnn') def expand_query(query): paraphraser = pipeline('text2text-generation', model='Vamsi/T5_Paraphrase_Paws') expanded_queries = paraphraser(query, num_return_sequences=3, max_length=50, do_sample=True) return [q['generated_text'] for q in expanded_queries] def generate_description(query): response = generator(query, max_length=100, num_return_sequences=1) return response[0]['generated_text'] def search_courses(query, level_filter=None, category_filter=None, top_k=3): expanded_queries = expand_query(query) all_similarities = [] for expanded_query in expanded_queries: query_embedding = model.encode(expanded_query, convert_to_tensor=True) similarities = util.pytorch_cos_sim(query_embedding, course_embeddings)[0] all_similarities.append(similarities) aggregated_similarities = torch.max(torch.stack(all_similarities), dim=0)[0] filtered_df = df.copy() if level_filter and level_filter != "Nil": filtered_df = filtered_df[filtered_df['Level of Difficulty'] == level_filter] if category_filter and category_filter != "NIL": filtered_df = filtered_df[filtered_df['Category'] == category_filter] if filtered_df.empty: return "
No matching courses found.
" filtered_similarities = aggregated_similarities[filtered_df.index] top_results = filtered_similarities.topk(k=min(top_k, len(filtered_similarities))) results = [] for idx in top_results.indices: idx = int(idx) course_title = filtered_df.iloc[idx]['Course Title'] course_description = filtered_df.iloc[idx, 1] course_url = filtered_df.iloc[idx, -1] generated_description = generate_description(course_title + " " + course_description) course_link = f'{course_title}' results.append(f"{course_link}