Upload 8 files

.env

@@ -0,0 +1 @@
+PINECONE_INDEX_NAME= embeddings

API.yml

@@ -0,0 +1,12 @@
+#openai:
+  #api_key: "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"
+pinecone_index_name : "embeddings"
+pinecone_api_key: "pcsk_68KQkG_JBFUYTFA5eoND98wVAHvjStQpZNRkKmVQNZUAP1JgfaMKZ17hCxJWWBybSgqdsw"
+pinecone_env: "aws-starter"
+together_ai_api_key: "456b4bf845157be3699b4a78d6651e9dcb32f98cd2e199c560bb5dfdcbe52a04"
+cohere_api_key: "po8LkIuh7RAjkQ7DNdOv6bmXGUvo8F9pMBEJjZVZ"
+#huggingface:
+  #model_name: "sentence-transformers/all-mpnet-base-v2"
+
+
+  

app.py

@@ -0,0 +1,287 @@
+import yaml
+from together import Together
+from langchain.llms.together import Together as TogetherLLM
+from langchain.prompts import PromptTemplate
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.schema.output_parser import StrOutputParser
+from pinecone import Pinecone
+import gradio as gr
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+
+API_FILE_PATH = r"C:\Users\abhay\Analytics Vidhya\API.yml"
+COURSES_FILE_PATH = r"C:\Users\abhay\Analytics Vidhya\courses.json"
+
+def load_api_keys(api_file_path):
+    """Loads API keys from a YAML file."""
+    with open(api_file_path, 'r') as f:
+        api_keys = yaml.safe_load(f)
+    return api_keys
+
+def generate_query_embedding(query, together_api_key):
+    """Generates embedding for the user query."""
+    client = Together(api_key=together_api_key)
+    response = client.embeddings.create(
+        model="WhereIsAI/UAE-Large-V1", input=query
+    )
+    return response.data[0].embedding
+
+def initialize_pinecone(pinecone_api_key):
+    """Initializes Pinecone with API key."""
+    return Pinecone(api_key=pinecone_api_key)
+
+def pinecone_similarity_search(pinecone_instance, index_name, query_embedding, top_k=5):
+    """Performs a similarity search in Pinecone."""
+    try:
+        index = pinecone_instance.Index(index_name)
+        results = index.query(vector=query_embedding, top_k=top_k, include_metadata=True)
+        if not results.matches:
+            return None
+        return results
+    except Exception as e:
+        print(f"Error during similarity search: {e}")
+        return None
+
+def create_prompt_template():
+    """Creates a prompt template for LLM."""
+    template = """You are a helpful AI course advisor. Based on the following context and query, suggest relevant courses.
+    For each course, explain:
+    1. Why it's relevant to the query
+    2. What the student will learn
+    3. Who should take this course
+    
+    If no relevant courses are found, suggest alternative search terms.
+
+    Context: {context}
+    User Query: {query}
+
+    Response: Let me help you find the perfect courses for your needs! 🎓
+    """
+    return PromptTemplate(template=template, input_variables=["context", "query"])
+
+def initialize_llm(together_api_key):
+    """Initializes Together LLM."""
+    return TogetherLLM(
+        model="mistralai/Mixtral-8x7B-Instruct-v0.1",
+        together_api_key=together_api_key,
+        temperature=0.3,
+        max_tokens=500
+    )
+
+def create_chain(llm, prompt):
+    """Creates a chain using the RunnableSequence approach."""
+    chain = (
+        {"context": RunnablePassthrough(), "query": RunnablePassthrough()}
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+    return chain
+
+def format_course_info(metadata):
+    """Formats course information with emojis and styling."""
+    return f"""
+📚 **Course Title:** {metadata.get('title', 'No title')}
+
+📝 **Description:** {metadata.get('text', 'No description')}
+
+🔗 **Course Link:** {metadata.get('course_link', 'No link')}
+
+👨‍🏫 **Instructor:** {metadata.get('instructor', 'Not specified')}
+
+⏱️ **Duration:** {metadata.get('duration', 'Not specified')}
+
+📊 **Level:** {metadata.get('difficulty_level', 'Not specified')}
+
+💰 **Price:** {metadata.get('price', 'Not specified')}
+"""
+
+def generate_llm_response(chain, query, retrieved_data):
+    """Generates an LLM response with formatted course information."""
+    try:
+        if not retrieved_data or not retrieved_data.matches:
+            return "🔍 I couldn't find any relevant courses matching your query. Please try different search terms."
+
+        context_parts = []
+        formatted_courses = []
+        
+        for match in retrieved_data.matches:
+            metadata = match.metadata
+            if metadata:
+                context_parts.append(
+                    f"Title: {metadata.get('title', 'No title')}\n"
+                    f"Description: {metadata.get('text', 'No description')}\n"
+                    f"Link: {metadata.get('course_link', 'No link')}"
+                )
+                formatted_courses.append(format_course_info(metadata))
+
+        if not context_parts:
+            return "⚠️ I found some matches but couldn't extract course information. Please try again."
+
+        context = "\n\n".join(context_parts)
+        llm_analysis = chain.invoke({"context": context, "query": query})
+
+        separator = "=" * 50
+        final_response = f"""
+{llm_analysis}
+
+🎯 Here are the detailed course listings:
+{separator}
+{''.join(formatted_courses)}
+"""
+        return final_response
+
+    except Exception as e:
+        print(f"Error generating response: {e}")
+        return "❌ I encountered an error while generating the response. Please try again."
+
+def create_gradio_interface(api_keys):
+    """Creates a custom Gradio interface with improved styling."""
+    # Initialize components
+    pinecone_instance = initialize_pinecone(api_keys["pinecone_api_key"])
+    llm = initialize_llm(api_keys["together_ai_api_key"])
+    prompt = create_prompt_template()
+    chain = create_chain(llm, prompt)
+
+    def process_query(query):
+        try:
+            query_embedding = generate_query_embedding(query, api_keys["together_ai_api_key"])
+            results = pinecone_similarity_search(
+                pinecone_instance, 
+                api_keys["pinecone_index_name"], 
+                query_embedding
+            )
+            response = generate_llm_response(chain, query, results)
+            return response
+        except Exception as e:
+            return f"❌ Error: {str(e)}"
+
+    # Custom CSS for better styling
+    custom_css = """
+    .gradio-container {
+        background-color: #f0f8ff;
+    }
+    .input-box {
+        border: 2px solid #2e86de;
+        border-radius: 10px;
+        padding: 15px;
+        margin: 10px 0;
+    }
+    .output-box {
+        background-color: #ffffff;
+        border: 2px solid #54a0ff;
+        border-radius: 10px;
+        padding: 20px;
+        margin: 10px 0;
+        box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    .heading {
+        color: #2e86de;
+        text-align: center;
+        margin-bottom: 20px;
+    }
+    .submit-btn {
+        background-color: #2e86de !important;
+        color: white !important;
+        border-radius: 8px !important;
+        padding: 10px 20px !important;
+        font-size: 16px !important;
+    }
+    .examples {
+        margin-top: 20px;
+        padding: 15px;
+        background-color: #f8f9fa;
+        border-radius: 10px;
+    }
+    """
+
+    # Create Gradio interface with custom theme
+    theme = gr.themes.Soft().set(
+        body_background_fill="#f0f8ff",
+        block_background_fill="#ffffff",
+        block_border_width="2px",
+        block_border_color="#2e86de",
+        block_radius="10px",
+        button_primary_background_fill="#2e86de",
+        button_primary_text_color="white",
+        input_background_fill="#ffffff",
+        input_border_color="#2e86de",
+        input_radius="8px",
+    )
+
+    with gr.Blocks(theme=theme, css=custom_css) as demo:
+        gr.Markdown(
+            """
+            # 🎓 Course Recommendation Assistant
+            
+            Welcome to your personalized course finder! Ask me about any topics you're interested in learning.
+            I'll help you discover the perfect courses from Analytics Vidhya's collection.
+            
+            ## 🌟 Features:
+            - 📚 Detailed course recommendations
+            - 🎯 Learning path suggestions
+            - 📊 Course difficulty levels
+            - 💰 Price information
+            """,
+            elem_classes=["heading"]
+        )
+        
+        with gr.Row():
+            with gr.Column():
+                query_input = gr.Textbox(
+                    label="What would you like to learn? 🤔",
+                    placeholder="e.g., 'machine learning for beginners' or 'advanced python courses'",
+                    lines=3,
+                    elem_classes=["input-box"]
+                )
+                submit_btn = gr.Button(
+                    "🔍 Find Courses",
+                    variant="primary",
+                    elem_classes=["submit-btn"]
+                )
+
+        with gr.Row():
+            output = gr.Markdown(
+                label="Recommendations 📚",
+                elem_classes=["output-box"]
+            )
+
+        with gr.Row(elem_classes=["examples"]):
+            gr.Examples(
+                examples=[
+                    ["I want to learn machine learning from scratch"],
+                    ["Advanced deep learning courses"],
+                    ["Data visualization tutorials"],
+                    ["Python programming for beginners"],
+                    ["Natural Language Processing courses"]
+                ],
+                inputs=query_input,
+                label="📝 Example Queries"
+            )
+
+        submit_btn.click(
+            fn=process_query,
+            inputs=query_input,
+            outputs=output
+        )
+
+    return demo
+
+def main():
+    try:
+        
+        api_keys = load_api_keys(API_FILE_PATH)
+        
+        
+        demo = create_gradio_interface(api_keys)
+        demo.launch(
+            share=True)
+
+    except Exception as e:
+        print(f"An error occurred during initialization: {str(e)}")
+
+if __name__ == "__main__":
+    main()

courses.json

@@ -0,0 +1,470 @@
+[
+    {
+        "title": "Frameworks for Effective Problem Solving",
+        "description": "18 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/zdlLMkFXQjuKiItB273o_Thumbnail%201.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/frameworks-for-effective-problem-solving"
+    },
+    {
+        "title": "Anyone can Build AI Agents - Free Course",
+        "description": "5 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/HDiuucWOTp2G0PDvjEQm_build%20your%20first%20agent%20using%20no%20code%20tools%20(1).jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/your-ultimate-guide-to-becoming-an-agentic-ai-expert-by-2025"
+    },
+    {
+        "title": "A Comprehensive Learning Path to Become a Data Analyst in 2025",
+        "description": "298 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/qdmHBCw7Qtqw2Y9g0vOE_Become%20a%20GenAI%20Expert%20in%202025.%20Thumbnail.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-to-become-a-data-analyst-in-2025"
+    },
+    {
+        "title": "Reimagining GenAI: Common Mistakes and Best Practices for Success",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/ky8mTd5ISIypAcrVJkdO_unnamed%20(4).jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/reimagining-genai-common-mistakes-and-best-practices-for-success"
+    },
+    {
+        "title": "Coding a ChatGPT-style Language Model from Scratch in PyTorch",
+        "description": "7 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/Q1tYD2LSlaLzL1XPZWgw_1Mastering%20Multilingual%20GenAI%20Open-Weights%20for%20Indic%20Languages%20copy.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/coding-a-chatgpt-style-language-model-from-scratch-in-pytorch"
+    },
+    {
+        "title": "Mastering Multilingual GenAI Open-Weights for Indic Languages",
+        "description": "9 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/beC0zMfQhOnHcT2FLR4g_unnamed%20(2).jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/mastering-multilingual-genai-open-weights-for-indic-languages"
+    },
+    {
+        "title": "Learning Autonomous Driving Behaviors with LLMs & RL",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/lrzfD3upSjb0iUOrJcOo_unnamed%20(1).jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/learning-autonomous-driving-behaviors-with-llms-and-rl"
+    },
+    {
+        "title": "GenAI Applied to Quantitative Finance: For Control Implementation",
+        "description": "5 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/0hkgoKCKTnKKxhQGAEIp_govind_av_httpss.mj.runER3kgo8rubs_based_on_this_theme_create_b2190701-9e82-47ea-b65a-c52d6de94e07_3.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/genai-applied-to-quantitative-finance-for-control-implementation"
+    },
+    {
+        "title": "Navigating LLM Tradeoffs: Techniques for Speed, Cost, Scale & Accuracy",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/SV5jeKd1Q4iFyzMlUSdw_Thumbnail.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/navigating-llm-tradeoffs-techniques-for-speed-cost-scale-and-accuracy"
+    },
+    {
+        "title": "Creating Problem-Solving Agents using GenAI for Action Composition",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/ih6JsBMgSCWbl9ofgpYg_Thumbnail%20.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/creating-problem-solving-agents-using-genai-for-action-composition"
+    },
+    {
+        "title": "Improving Real World RAG Systems: Key Challenges & Practical Solutions",
+        "description": "12 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/70WfAJhFRsyXXdLQcWIl_281762.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/improving-real-world-rag-systems-key-challenges"
+    },
+    {
+        "title": "Framework to Choose the Right LLM for your Business",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/A3lVcQcRRbKN9IBZANvV_cinematic-person-is-looking-customer-journey-map%201.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/choosing-the-right-LLM-for-your-business"
+    },
+    {
+        "title": "Building Smarter LLMs with Mamba and State Space Model",
+        "description": "14 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/FbdKUqgKSj2K80QgVl1z_THUMBNAIL%20(1).png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/building-smarter-llms-with-mamba-and-state-space-model"
+    },
+    {
+        "title": "Generative AI - A Way of Life - Free Course",
+        "description": "31 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/0hkgoKCKTnKKxhQGAEIp_govind_av_httpss.mj.runER3kgo8rubs_based_on_this_theme_create_b2190701-9e82-47ea-b65a-c52d6de94e07_3.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/genai-a-way-of-life"
+    },
+    {
+        "title": "Building LLM Applications using Prompt Engineering - Free Course",
+        "description": "18 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/GnpgN0uiTA1VO2MpQvTA_3.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/building-llm-applications-using-prompt-engineering-free"
+    },
+    {
+        "title": "Building Your First Computer Vision Model - Free Course",
+        "description": "7 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/cv8WEAZPRtGcSR8A2kBY_cv%20thumbnail.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/building-your-first-computer-vision-model"
+    },
+    {
+        "title": "Bagging and Boosting ML Algorithms - Free Course",
+        "description": "16 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/ZPH29GMrTey5gAaobwAH_Getting%20started%20with%20Decision%20Trees.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/bagging-boosting-ML-Algorithms"
+    },
+    {
+        "title": "MidJourney: From Inspiration to Implementation - Free Course",
+        "description": "5 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/NclAzDSwRwedON784y83_QZQvFaQFTFu4cFHZ0O5A_Midjourney%20(1).png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/midjourney_from_inspiration_to_implementation"
+    },
+    {
+        "title": "Understanding Linear Regression - Free Course",
+        "description": "10 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/vK6lGCCqSsWtESgG69K1_Foundational%20ML%20Algorithms.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/free-understanding-linear-regression"
+    },
+    {
+        "title": "The Working of Neural Networks - Free Course",
+        "description": "11 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/cu3qVXYTR7WEDj7eYQXS_unnamed.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/the-working-of-neural-networks"
+    },
+    {
+        "title": "The A to Z of Unsupervised ML - Free Course",
+        "description": "13 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/a8FaIfN8SyfgOEhgzcHx_Getting_Started_with_Neural_Networks_307d8d11-d90e-45eb-8885-7d8f59fadc48.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/free-unsupervised-ml-guide"
+    },
+    {
+        "title": "Building Your first RAG System using LlamaIndex - Free Course",
+        "description": "12 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/HSr06pXxRTeF2Lpp6VQl_DALL%C2%B7E%202024-07-03%2011.42.42%20-%20A%20visually%20striking%20representation%20of%20a%20Retrieval-Augmented%20Generation%20(RAG)%20system%20using%20LlamaIndex.%20The%20image%20should%20have%20a%20dark%20background%20with%20int.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/building-first-rag-systems-using-llamaindex"
+    },
+    {
+        "title": "Data Preprocessing on a Real-World Problem Statement - Free Course",
+        "description": "13 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/2BmYWOASeye7uIgp3mHr_Building%20your%20first%20ML%20model.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/data-preprocessing"
+    },
+    {
+        "title": "Exploring Stability.AI - Free Course",
+        "description": "7 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/imiRzsUqTwuQxv4wfyUb_mastering-stable-diffusion-thumbnail_2.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/exploring-stability-ai"
+    },
+    {
+        "title": "Building a Text Classification Model with Natural Language Processing - Free Course",
+        "description": "16 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/YfA6OJ6ATeiwbbvvzb5h_NLP%20Banner.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/free-building-textclassification-natural-language-processing"
+    },
+    {
+        "title": "Getting Started with Large Language Models",
+        "description": "18 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/TwSmk34ARlGMcQu4M8Co_A_Comprehensive_Learning_Path_to_Become_a_Data_Sc_265db306-1f93-496d-827e-0f1e534c9cf1.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/getting-started-with-llms"
+    },
+    {
+        "title": "Introduction to Generative  AI",
+        "description": "9 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/CNjAWH6TUaIQt8N5FX9Y_thumbnail%201.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/introduction-to-generative-ai"
+    },
+    {
+        "title": "Exploring Natural Language Processing (NLP) using Deep Learning",
+        "description": "79 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/vl3JHf3eS0Wp9XPLeDaL_8.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/exploring-natural-language-processing-nlp-using-deep-learning"
+    },
+    {
+        "title": "Getting Started with Deep Learning",
+        "description": "299 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/KybSy5ROKQAC4fyxyFlQ_6_.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/getting-started-with-deep-learning"
+    },
+    {
+        "title": "Nano Course: Dreambooth-Stable Diffusion for Custom Images",
+        "description": "13 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/sU6NzHaSiCPghfiOpQ52_Introduction_to_Natural_Language_Processing_b92e8456-3145-489b-97e7-51925a655b2c.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/nano-course-dreambooth-stable-diffusion-for-custom-images"
+    },
+    {
+        "title": "A Comprehensive Learning Path for Deep Learning in 2023",
+        "description": "72 Lessons",
+        "image_url": "https://files.cdn.thinkific.com/courses/course_card_image_000/580/9901576591074.original.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-for-deep-learning-in-2023"
+    },
+    {
+        "title": "A Comprehensive Learning Path to Become a Data Scientist in 2024",
+        "description": "389 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/1b0s4lDiQluveqaLqKj8_A_Comprehensive_Learning_Path_to_Become_a_Data_Sc_6e902c6d-46bf-4b8b-91cc-24329e68086c.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-to-become-a-data-scientist-in-twenty-twenty-four"
+    },
+    {
+        "title": "Nano Course: Building Large Language Models for Code",
+        "description": "10 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/OelAeGXrRuGrnkcJSdAO_OG.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/building-large-language-models-for-code"
+    },
+    {
+        "title": "Nano Course: Cutting Edge LLM Tricks",
+        "description": "12 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/OelAeGXrRuGrnkcJSdAO_OG.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/cutting-edge-llm-tricks"
+    },
+    {
+        "title": "Certified AI & ML BlackBelt+ Program",
+        "description": "No Description",
+        "image_url": "https://files.cdn.thinkific.com/bundles/bundle_card_image_000/048/808/1719037298.original.png",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/certified-ai-ml-blackbelt-plus"
+    },
+    {
+        "title": "Machine Learning Summer Training",
+        "description": "263 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/courses/1903742/zJ10PGDvQPKiGRGM73Dn_icon%282%29%20%281%29.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/machine-learning-summer-training"
+    },
+    {
+        "title": "AI Ethics by Fractal",
+        "description": "5 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/courses/1720521/9MuNm0LRmeWOiy5VjCDB_ai.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/ai-ethics-fractal"
+    },
+    {
+        "title": "A Comprehensive Learning Path to Become a Data Engineer in 2022",
+        "description": "204 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/courses/1667459/Rqbh8QTKShaGKJ6vNiRx_overview%20of%20data%20engineering.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-to-become-a-data-engineer-in-2022"
+    },
+    {
+        "title": "Certified Business Analytics Program",
+        "description": "No Description",
+        "image_url": "https://files.cdn.thinkific.com/bundles/bundle_card_image_000/048/438/1719206987.original.png",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/certified-business-analytics-program"
+    },
+    {
+        "title": "Certified Machine Learning Master's Program (MLMP)",
+        "description": "No Description",
+        "image_url": "https://files.cdn.thinkific.com/bundles/bundle_card_image_000/046/940/1719206858.original.png",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/certified-machine-learning-master-s-program-mlmp"
+    },
+    {
+        "title": "Certified Natural Language Processing Master\u2019s Program",
+        "description": "No Description",
+        "image_url": "https://import.cdn.thinkific.com/118220/moBOBv3RMCKDgEoIi3DK_nlp%20master_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/certified-natural-language-processing-master-s-program"
+    },
+    {
+        "title": "Certified Computer Vision Master's Program",
+        "description": "No Description",
+        "image_url": "https://import.cdn.thinkific.com/118220/BY3BuXPFRuaKZuKdYB5T_thumb_.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/certified-computer-vision-masters-program"
+    },
+    {
+        "title": "Applied Machine Learning - Beginner to Professional",
+        "description": "612 Lessons",
+        "image_url": "https://files.cdn.thinkific.com/courses/course_card_image_000/739/8911589353707.original.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/applied-machine-learning-beginner-to-professional"
+    },
+    {
+        "title": "Ace Data Science Interviews",
+        "description": "59 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/eZzXQDJSgyLlxFiQ5rbu_data_science_interview.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/ace-data-science-interviews"
+    },
+    {
+        "title": "Writing Powerful Data Science Articles",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/courses/1292824/YfJD5SeRLaXRxYgVb2uw_icon.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/writing-powerful-data-science-articles"
+    },
+    {
+        "title": "Machine Learning Certification Course for Beginners",
+        "description": "261 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/NjFxaJVSFeAfMSHYoG9Q_Machine_Learning_for_Beginners_b896980a-b1d1-41bc-9bab-d8d3176daaf8.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/Machine-Learning-Certification-Course-for-Beginners"
+    },
+    {
+        "title": "Data Science Career Conclave",
+        "description": "6 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/courses/1230486/udv57o17QrG3uuBO8BwD_1200x628-icon.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/data-science-career-conclave"
+    },
+    {
+        "title": "Top Data Science Projects for Analysts and Data Scientists",
+        "description": "36 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/oNPtWVT9CbWwip5Ogang_project_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/top-data-science-projects-for-analysts-and-data-scientists"
+    },
+    {
+        "title": "Getting Started with Git and GitHub for Data Science Professionals",
+        "description": "20 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/pbMFVOw2RPi9fFXSOuGP_github_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/getting-started-with-git-and-github-for-data-science-professionals"
+    },
+    {
+        "title": "Machine Learning Starter Program",
+        "description": "No Description",
+        "image_url": "https://import.cdn.thinkific.com/118220/OTJAEtMGQWaxxIWD0pWv_MLSP%20thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/machine-learning-starter-program"
+    },
+    {
+        "title": "Data Science Hacks, Tips and Tricks",
+        "description": "37 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/qaomVKyQPaj9MCRxO9dg_course%20thumb%20image.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/data-science-hacks-tips-and-tricks"
+    },
+    {
+        "title": "Introduction to Business Analytics",
+        "description": "33 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/uQspaf6ESc6KPX6jkVrx_Introduction_to_Business_Analytics_fcacd85c-25ff-44bf-a74d-ee3aa66e51d4.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/introduction-to-analytics"
+    },
+    {
+        "title": "Introduction to PyTorch for Deep Learning",
+        "description": "28 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/3Ec7NO7MTkqaEImQio94_pytorch_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/introduction-to-pytorch-for-deeplearning"
+    },
+    {
+        "title": "Introductory Data Science for Business Managers",
+        "description": "No Description",
+        "image_url": "https://import.cdn.thinkific.com/118220/oF8KoXuyQ7KrHIRMmTt5_Business_Manager.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/bundles/introductory-data-science-for-business-managers"
+    },
+    {
+        "title": "Introduction to Natural Language Processing",
+        "description": "21 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/X8bM7n4REC8ykgf53oSj_10.Introduction%20to%20AI%20_%20ML.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/Intro-to-NLP"
+    },
+    {
+        "title": "Getting started with Decision Trees",
+        "description": "20 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/Tgfv1fveTpWrv3ZFTt6r_Decision_Trees_ca3c729e-525b-4c8c-9f48-dec4bbb4faf3.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/getting-started-with-decision-trees"
+    },
+    {
+        "title": "Introduction to Python",
+        "description": "69 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/QAUsjF0qSxCI7fsIrzhn_Algorithm_in_Python_and_R_1b947457-056e-4bfb-9a6f-ce614ebfe3c4.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/introduction-to-data-science"
+    },
+    {
+        "title": "Loan Prediction Practice Problem (Using Python)",
+        "description": "16 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/pcauZ3ZqSWqrYXlYTd0w_Loan_Prediction_Practice_Problem_8dbf2aea-f3c8-42d4-bd27-87e738de5aac.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/loan-prediction-practice-problem-using-python"
+    },
+    {
+        "title": "Big Mart Sales Prediction Using R",
+        "description": "19 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/t1hnCAcdT22pr7EEWQpW_Sales%20Prediction_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/big-mart-sales-prediction-using-r"
+    },
+    {
+        "title": "Twitter Sentiment Analysis",
+        "description": "18 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/nwZ67Ta2R0SO6uI4mWBe_Twitter%20Sentiment%20Analysis.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/twitter-sentiment-analysis"
+    },
+    {
+        "title": "Pandas for Data Analysis in Python",
+        "description": "27 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/SPL9Cu7sQ5uihQ2mGYtO_Pandas_for_Data_Analysis_in_Python_7760df6c-f648-46d8-93f3-ba4e49d89a22.png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/pandas-for-data-analysis-in-python"
+    },
+    {
+        "title": "Support Vector Machine (SVM) in Python and R",
+        "description": "13 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/PhPoLyt7QKOucBS6VLf2_Support%20Vector%20Machine_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/support-vector-machine-svm-in-python-and-r"
+    },
+    {
+        "title": "Evaluation Metrics for Machine Learning Models",
+        "description": "25 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/Qb1PlcDoTEeoR1ZOOvUy_evaluation%20metrics_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/evaluation-metrics-for-machine-learning-models"
+    },
+    {
+        "title": "Fundamentals of Regression Analysis",
+        "description": "24 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/C5HN5LVDQsyS1hRojIyT_courses_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/Fundamentals-of-Regression-Analysis"
+    },
+    {
+        "title": "Getting Started with scikit-learn (sklearn) for Machine Learning",
+        "description": "27 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/d8R1MoSmisGRZk4UJWQF_thumb_.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/get-started-with-scikit-learn-sklearn"
+    },
+    {
+        "title": "Convolutional Neural Networks (CNN) from Scratch",
+        "description": "17 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/4yxTr5xuQy6snBYBcHP8_Convolutional%20Neural%20Networks_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/convolutional-neural-networks-cnn-from-scratch"
+    },
+    {
+        "title": "Dimensionality Reduction for Machine Learning",
+        "description": "23 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/Um1x4Xh7RIGzMIQuwCeM_course_thumb%20.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/dimensionality-reduction-for-machine-learning"
+    },
+    {
+        "title": "K-Nearest Neighbors (KNN) Algorithm in Python and R",
+        "description": "10 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/7FpGUA1gQLWbDB5oVsCz_thumb_.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/K-Nearest-Neighbors-KNN-Algorithm"
+    },
+    {
+        "title": "Ensemble Learning and Ensemble Learning Techniques",
+        "description": "37 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/doHSugITVCfoBUVdlxeH_Ensemble%20Learning-thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/ensemble-learning-and-ensemble-learning-techniques"
+    },
+    {
+        "title": "Linear Programming for Data Science Professionals",
+        "description": "20 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/TMBNBD4R5a1YdlMHJada_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/linear-programming"
+    },
+    {
+        "title": "Naive Bayes from Scratch",
+        "description": "19 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/tdU7ioeOQG4ZuHzhoJrQ_thumb_.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/naive-bayes"
+    },
+    {
+        "title": "Learn Swift for Data Science",
+        "description": "21 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/QT5BeWvTG2ITfNiyiCZQ_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/learn-swift-for-data-science"
+    },
+    {
+        "title": "Introduction to Web Scraping using Python",
+        "description": "13 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/AEt5RjqFTCeJJ8oIBn7s_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/introduction-to-web-scraping"
+    },
+    {
+        "title": "Tableau for Beginners",
+        "description": "17 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/UtXF4erQmuUVLtZz8TMA_tablu_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/tableau-for-beginners"
+    },
+    {
+        "title": "Getting Started with Neural Networks",
+        "description": "45 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/hujFPpK4S261x92WyEyc_natural%20Network.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/getting-started-with-neural-networks"
+    },
+    {
+        "title": "Introduction to AI & ML",
+        "description": "17 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/rlT6Q4O0SEafxqg4WmzL_Introduction_to_AI__ML_88145141-2c69-4ae8-9681-5d7201a38288%20(1).png",
+        "course_link": "https://courses.analyticsvidhya.com/courses/introduction-to-ai-ml"
+    },
+    {
+        "title": "Winning Data Science Hackathons - Learn from Elite Data Scientists",
+        "description": "7 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/7exjvWuRSNesvaXPPxHO_data-science.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/winning-data-science-hackathons-learn-from-elite-data-scientists"
+    },
+    {
+        "title": "Hypothesis Testing for Data Science and Analytics",
+        "description": "19 Lessons",
+        "image_url": "https://import.cdn.thinkific.com/118220/iO9QE66XTOm17Mz0fxzs_thumb.jpg",
+        "course_link": "https://courses.analyticsvidhya.com/courses/hypothesis-testing-for-data-science-and-analytics"
+    }
+]

create_embeddings_together

@@ -0,0 +1,129 @@
+import json
+from pinecone import Pinecone, ServerlessSpec
+import os
+from dotenv import load_dotenv
+import yaml
+from together import Together
+
+
+
+load_dotenv()
+
+# Define file paths as constants
+API_FILE_PATH = r"C:\Users\abhay\Analytics Vidhya\API.yml"
+COURSES_FILE_PATH = r"C:\Users\abhay\Analytics Vidhya\courses.json"
+
+def load_api_keys(api_file_path):
+    """Loads API keys from a YAML file."""
+    with open(api_file_path, 'r') as f:
+        api_keys = yaml.safe_load(f)
+    return api_keys
+
+
+def load_course_data(json_file_path):
+    """Loads course data from a JSON file."""
+    with open(json_file_path, 'r') as f:
+        course_data = json.load(f)
+    return course_data
+
+
+def prepare_for_embedding(course_data):
+    """Combines relevant course fields for embedding."""
+    prepared_data = []
+    for i, course in enumerate(course_data):
+        combined_text = f"Title: {course.get('title', '')}, Description: {course.get('description', '')}"
+        prepared_data.append(
+            {
+                "course_id": i,
+                "text": combined_text,
+                "course_link": course.get("course_link"),
+                "image_url": course.get("image_url"),
+                "title": course.get("title"),
+            }
+        )
+    return prepared_data
+
+# --- Generate Embeddings using Together AI Model ---
+def generate_embeddings(texts, together_api_key):
+    """Generates embeddings using Together AI model directly."""
+    client = Together(api_key=together_api_key)
+    embeddings = []
+    for text in texts:
+        response = client.embeddings.create(
+            model="WhereIsAI/UAE-Large-V1", input=text
+        )
+        embeddings.append(response.data[0].embedding)
+    return embeddings
+
+# --- Initialize Pinecone ---
+def initialize_pinecone(pinecone_api_key, pinecone_env):
+    """Initializes Pinecone with API key and environment."""
+    pc = Pinecone(api_key=pinecone_api_key)
+    return pc
+
+# --- Upsert Embeddings into Pinecone ---
+def upsert_to_pinecone(pinecone_instance, index_name, prepared_data, embeddings):
+    """Upserts vectors into a Pinecone index."""
+    index = pinecone_instance.Index(index_name)
+    vectors_to_upsert = []
+    for i, item in enumerate(prepared_data):
+        vector = embeddings[i]
+        metadata = {
+            "course_id": item["course_id"],
+            "text": item["text"],
+            "course_link": item["course_link"],
+            "image_url": item["image_url"],
+            "title": item["title"],
+        }
+        vectors_to_upsert.append((str(item["course_id"]), vector, metadata))
+    index.upsert(vectors=vectors_to_upsert)
+
+
+# --- Main Function ---
+def main():
+    try:
+        
+        api_keys = load_api_keys(API_FILE_PATH)
+        together_api_key = api_keys["together_ai_api_key"]
+        pinecone_api_key = api_keys["pinecone_api_key"]
+        pinecone_env = api_keys["pinecone_env"]
+
+        
+        course_data = load_course_data(COURSES_FILE_PATH)
+
+        
+        prepared_data = prepare_for_embedding(course_data)
+        texts_for_embedding = [item["text"] for item in prepared_data]
+
+        
+        print("Generating embeddings...")
+        embeddings = generate_embeddings(texts_for_embedding, together_api_key)
+
+        
+        print("Initializing Pinecone...")
+        pinecone_instance = initialize_pinecone(pinecone_api_key, pinecone_env)
+        
+        
+        index_name = os.getenv("PINECONE_INDEX_NAME") or api_keys.get("pinecone_index_name")
+        if not index_name:
+            raise ValueError("Pinecone index name not found in environment variables or API.yml")
+        
+        
+        if index_name not in pinecone_instance.list_indexes().names():
+            pinecone_instance.create_index(
+                name=index_name,
+                dimension=1024,  # Dimension for UAE-Large-V1
+                metric='cosine'
+            )
+        
+        # Upsert embeddings into Pinecone
+        print("Upserting embeddings to Pinecone...")
+        upsert_to_pinecone(pinecone_instance, index_name, prepared_data, embeddings)
+
+        print("Embeddings generated and upserted to Pinecone successfully!")
+        
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+
+if __name__ == "__main__":
+    main()

llm_retrieval_conversation_rerank.py

@@ -0,0 +1,239 @@
+import json
+import os
+from dotenv import load_dotenv
+import yaml
+from together import Together
+from langchain.llms.together import Together as TogetherLLM
+from langchain.prompts import PromptTemplate
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.schema.output_parser import StrOutputParser
+from pinecone import Pinecone
+from typing import List, Dict
+import cohere
+load_dotenv()
+
+
+API_FILE_PATH = r"C:\Users\abhay\Analytics Vidhya\API.yml"
+COURSES_FILE_PATH = r"C:\Users\abhay\Analytics Vidhya\courses.json"
+
+# Global list to store conversation history
+conversation_history: List[Dict[str, str]] = []
+
+def load_api_keys(api_file_path):
+    """Loads API keys from a YAML file."""
+    with open(api_file_path, 'r') as f:
+        api_keys = yaml.safe_load(f)
+    return api_keys
+
+def generate_query_embedding(query, together_api_key):
+    """Generates embedding for the user query."""
+    client = Together(api_key=together_api_key)
+    response = client.embeddings.create(
+        model="WhereIsAI/UAE-Large-V1", input=query
+    )
+    return response.data[0].embedding
+
+def initialize_pinecone(pinecone_api_key):
+    """Initializes Pinecone with API key."""
+    return Pinecone(api_key=pinecone_api_key)
+
+def pinecone_similarity_search(pinecone_instance, index_name, query_embedding, top_k=10):
+    """Performs a similarity search in Pinecone and increase top k for reranking."""
+    try:
+        index = pinecone_instance.Index(index_name)
+        results = index.query(vector=query_embedding, top_k=top_k, include_metadata=True)
+        if not results.matches:
+            return None
+        return results
+    except Exception as e:
+        print(f"Error during similarity search: {e}")
+        return None
+
+def create_prompt_template():
+    """Creates a prompt template for LLM."""
+    template = """You are a helpful AI assistant that provides information on courses. 
+    Based on the following context, conversation history, and new user query, 
+    suggest relevant courses and explain why they might be useful, or respond accordingly if the user query is unrelated. 
+    If no relevant courses are found, please indicate that.
+
+    Conversation History:
+    {conversation_history}
+
+    Context: {context}
+    User Query: {query}
+
+    Response: Let me help you find relevant courses based on your query.
+    """
+    return PromptTemplate(template=template, input_variables=["context", "query", "conversation_history"])
+
+def initialize_llm(together_api_key):
+    """Initializes Together LLM."""
+    return TogetherLLM(
+        model="mistralai/Mixtral-8x7B-Instruct-v0.1",
+        together_api_key=together_api_key,
+        temperature=0,
+        max_tokens=250
+    )
+
+def create_chain(llm, prompt):
+    """Creates a chain using the new RunnableSequence approach."""
+    chain = (
+        {"context": RunnablePassthrough(), "query": RunnablePassthrough(), "conversation_history": RunnablePassthrough()}
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+    return chain
+
+
+def initialize_cohere_client(cohere_api_key):
+    """Initializes the Cohere client."""
+    return cohere.ClientV2(api_key=cohere_api_key)
+
+
+def rerank_results(cohere_client, query, documents, top_n=3):
+    """Reranks documents using Cohere."""
+    try:
+        results = cohere_client.rerank(
+            query=query,
+            documents=documents,
+            top_n=top_n,
+            model="rerank-english-v3.0",
+        )
+        return results
+    except Exception as e:
+        print(f"Error reranking results: {e}")
+        return None
+
+def generate_llm_response(chain, query, retrieved_data, history, cohere_client):
+    """Generates an LLM response based on context and conversation history."""
+    try:
+        if not retrieved_data or not retrieved_data.matches:
+            return "I couldn't find any relevant courses matching your query. Please try a different search term."
+
+        # Prepare documents for reranking
+        documents = []
+        for match in retrieved_data.matches:
+            metadata = match.metadata
+            if metadata:
+                documents.append(
+                    { "text" :f"Title: {metadata.get('title', 'No title')}\nDescription: {metadata.get('text', 'No description')}\nLink: {metadata.get('course_link', 'No link')}"
+                    }
+                )
+
+        if not documents:
+            return "I found some matches but couldn't extract course information. Please try again."
+        
+         # Rerank the documents
+        reranked_results = rerank_results(cohere_client, query, documents)
+
+        if not reranked_results:
+              return "I couldn't rerank the results, please try again."
+        
+          # Prepare context from reranked results
+        context_parts = []
+        for result in reranked_results.results:
+            context_parts.append(documents[result.index]["text"])
+            
+        context = "\n\n".join(context_parts)
+            
+        # Format conversation history
+        formatted_history = "\n".join(f"User: {item['user']}\nAssistant: {item['assistant']}" for item in history) if history else "No previous conversation."
+
+        response = chain.invoke({"context": context, "query": query, "conversation_history":formatted_history})
+        return response
+
+    except Exception as e:
+        print(f"Error generating response: {e}")
+        return "I encountered an error while generating the response. Please try again."
+    
+
+def check_context_similarity(query_embedding, previous_query_embedding, threshold=0.7):
+    """Checks if the new query is related to the previous one."""
+    if not previous_query_embedding:
+        return False  # First query, no previous embedding to compare
+
+    from numpy import dot
+    from numpy.linalg import norm
+
+    cos_sim = dot(query_embedding, previous_query_embedding) / (norm(query_embedding) * norm(previous_query_embedding))
+    return cos_sim > threshold
+
+def main():
+    global conversation_history
+    previous_query_embedding = None
+
+    try:
+        
+        api_keys = load_api_keys(API_FILE_PATH)
+        together_api_key = api_keys["together_ai_api_key"]
+        pinecone_api_key = api_keys["pinecone_api_key"]
+        index_name = api_keys["pinecone_index_name"]
+        cohere_api_key = api_keys["cohere_api_key"]
+        print("Initializing services...")
+        
+        # Initialize Pinecone
+        pinecone_instance = initialize_pinecone(pinecone_api_key)
+
+        # Initialize Together LLM
+        llm = initialize_llm(together_api_key)
+
+         # Initialize Cohere client
+        cohere_client = initialize_cohere_client(cohere_api_key)
+
+
+        
+        prompt = create_prompt_template()
+
+        # Create chain
+        chain = create_chain(llm, prompt)
+
+        print("Ready to process queries!")
+
+        while True:
+           
+            user_query = input("\nEnter your query (or 'quit' to exit): ").strip()
+            
+            if user_query.lower() == 'quit':
+                break
+
+            if not user_query:
+                print("Please enter a valid query.")
+                continue
+
+            try:
+                print("Generating query embedding...")
+                query_embedding = generate_query_embedding(user_query, together_api_key)
+
+                # Check context similarity
+                if previous_query_embedding and check_context_similarity(query_embedding, previous_query_embedding):
+                     print("Continuing the previous conversation...")
+                else:
+                    print("Starting a new conversation...")
+                    conversation_history = []  # Clear history for a new conversation
+                
+                print("Searching for relevant courses...")
+                pinecone_results = pinecone_similarity_search(
+                    pinecone_instance, index_name, query_embedding
+                )
+
+                print("Generating response...")
+                llm_response = generate_llm_response(chain, user_query, pinecone_results, conversation_history, cohere_client)
+
+                print("\nResponse:")
+                print(llm_response)
+                print("\n" + "="*50)
+
+                # Update conversation history
+                conversation_history.append({"user": user_query, "assistant": llm_response})
+                previous_query_embedding = query_embedding  # Save for next turn
+               
+            except Exception as e:
+                print(f"Error processing query: {e}")
+                print("Please try again with a different query.")
+
+    except Exception as e:
+        print(f"An error occurred during initialization: {str(e)}")
+
+if __name__ == "__main__":
+    main()

requriments.txt

@@ -0,0 +1,8 @@
+together
+pinecone-client
+pyyaml
+python-dotenv
+langchain
+numpy
+cohere
+gradio

scrape_data.py

@@ -0,0 +1,80 @@
+import requests
+from bs4 import BeautifulSoup
+import json
+import os
+
+
+BASE_URL = "https://courses.analyticsvidhya.com/collections/courses?page="
+
+
+OUTPUT_FILE = os.path.join(os.path.dirname(__file__), "../data/courses.json")
+
+def scrape_courses():
+    courses = []
+
+    
+    for page in range(1, 10):
+        
+        URL = f"{BASE_URL}{page}"
+        print(f"Scraping URL: {URL}")
+
+        
+        response = requests.get(URL)
+        print(f"Response status: {response.status_code}")
+
+        # Check if request was successful
+        if response.status_code != 200:
+            print(f"Failed to fetch the webpage. Status code: {response.status_code}")
+            continue
+
+        
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        # Locate course containers
+        course_items = soup.find_all("li", class_="products__list-item")
+        print(f"Found {len(course_items)} course containers on page {page}.")
+
+        # Loop through each course container to extract details
+        for item in course_items:
+            # Extract course link
+            link_tag = item.find("a", class_="course-card")
+            course_link = link_tag.get("href", "#") if link_tag else "#"
+            if not course_link.startswith("http"):
+                course_link = f"https://courses.analyticsvidhya.com{course_link}"
+
+            # Extract course title
+            title_tag = link_tag.find("h3") if link_tag else None
+            title = title_tag.text.strip() if title_tag else "No Title"
+
+            # Extract course image
+            image_tag = link_tag.find("img", class_="course-card__img") if link_tag else None
+            image_url = image_tag.get("src", "No Image URL") if image_tag else "No Image URL"
+
+            # Extract course description
+            lesson_tag = link_tag.find("span", class_="course-card__lesson-count") if link_tag else None
+            description = lesson_tag.text.strip() if lesson_tag else "No Description"
+
+            # Add the extracted details to the list
+            courses.append({
+                "title": title,
+                "description": description,
+                "image_url": image_url,
+                "course_link": course_link,
+            })
+
+    # Debugging: Print the first few courses
+    print(f"Scraped {len(courses)} courses.")
+    for course in courses[:3]:
+        print(course)
+
+    # Ensure the directory for the output file exists
+    os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
+
+    # Save the course data to a JSON file
+    with open(OUTPUT_FILE, "w") as f:
+        json.dump(courses, f, indent=4)
+
+    print(f"Data saved to {os.path.abspath(OUTPUT_FILE)}")
+
+if __name__ == "__main__":
+    scrape_courses()